@larkup/workspace-server 0.0.0-stage → 0.2.1

package/src/lib/chat/tool-registry.ts

@@ -0,0 +1,6 @@
+/** Remove optional capabilities that are unavailable in the current runtime. */
+export function executableTools<T extends Record<string, unknown>>(candidates: T) {
+  return Object.fromEntries(
+    Object.entries(candidates).filter(([, definition]) => definition != null),
+  ) as { [K in keyof T as T[K] extends null | undefined ? never : K]: NonNullable<T[K]> };
+}

package/src/lib/chat/video-rag-routing.ts

@@ -0,0 +1,149 @@
+export type VideoEvidenceVerificationStatus =
+  'supported' | 'conflicted' | 'insufficient' | 'needs_inspection';
+
+export interface IndexedVideoAnswerSignal {
+  modality: string;
+  text: string;
+  confidenceScore: number;
+  startSecs: number;
+  endSecs: number;
+  conflict?: boolean;
+}
+
+// Concise inventory records average well under a paragraph. This ceiling is
+// high enough for long recordings while still bounding the answer model's
+// context and the browser payload.
+export const MAX_EXHAUSTIVE_EVIDENCE_ITEMS = 800;
+// Structured inventories are rendered by the browser's paginated data table,
+// never supplied to the answer model. They therefore deserve a much higher
+// transport limit than prose evidence while still protecting a chat response
+// from an unbounded malformed source.
+export const MAX_EXHAUSTIVE_TABLE_ROWS = 10_000;
+
+/** Follow a tool-owned chronological cursor without relying on model tool use. */
+export async function collectExhaustiveVideoEvidencePages(
+  execute: (input: Record<string, unknown>, options: { toolCallId: string }) => Promise<unknown>,
+  input: { mediaAssetId: string; query: string },
+  firstPage: unknown,
+  toolCallId: string,
+) {
+  if (!firstPage || typeof firstPage !== 'object' || Array.isArray(firstPage)) return firstPage;
+  const first = firstPage as Record<string, any>;
+  if (first.continuation?.exhaustive !== true || first.continuation?.hasMore !== true) {
+    return firstPage;
+  }
+
+  const evidence = Array.isArray(first.evidence) ? [...first.evidence] : [];
+  const seen = new Set(evidence.map((item: any) => item?.id ?? item?.evidenceId).filter(Boolean));
+  const rows = Array.isArray(first.rows) ? [...first.rows] : [];
+  const seenRows = new Set(
+    rows.map((row: any) => row?.id ?? row?._rowId).filter((id: unknown) => id != null),
+  );
+  const hasTable = Array.isArray(first.rows);
+  let continuation = first.continuation;
+  let lastPage = first;
+
+  while (
+    continuation?.hasMore === true &&
+    Number.isFinite(continuation.nextCursor) &&
+    (hasTable
+      ? rows.length < MAX_EXHAUSTIVE_TABLE_ROWS
+      : evidence.length < MAX_EXHAUSTIVE_EVIDENCE_ITEMS)
+  ) {
+    const cursor = Number(continuation.nextCursor);
+    const next = await execute({ ...input, exhaustive: true, cursor, limit: 48 }, { toolCallId });
+    if (!next || typeof next !== 'object' || Array.isArray(next)) break;
+    const page = next as Record<string, any>;
+    for (const item of Array.isArray(page.evidence) ? page.evidence : []) {
+      const id = item?.id ?? item?.evidenceId;
+      if (id && seen.has(id)) continue;
+      if (id) seen.add(id);
+      evidence.push(item);
+      if (evidence.length >= MAX_EXHAUSTIVE_EVIDENCE_ITEMS) break;
+    }
+    if (hasTable) {
+      for (const row of Array.isArray(page.rows) ? page.rows : []) {
+        const id = row?.id ?? row?._rowId;
+        if (id != null && seenRows.has(id)) continue;
+        if (id != null) seenRows.add(id);
+        rows.push(row);
+        if (rows.length >= MAX_EXHAUSTIVE_TABLE_ROWS) break;
+      }
+    }
+    lastPage = page;
+    continuation = page.continuation;
+    if (!continuation || Number(continuation.nextCursor) <= cursor) break;
+  }
+
+  return {
+    ...first,
+    evidence,
+    ...(hasTable
+      ? {
+          columns: Array.isArray(first.columns) ? first.columns : [],
+          rows,
+          totalRows: rows.length,
+        }
+      : {}),
+    continuation: {
+      ...(lastPage.continuation ?? continuation ?? first.continuation),
+      aggregatedItems: hasTable ? rows.length : evidence.length,
+      contextLimitReached:
+        (hasTable
+          ? rows.length >= MAX_EXHAUSTIVE_TABLE_ROWS
+          : evidence.length >= MAX_EXHAUSTIVE_EVIDENCE_ITEMS) && continuation?.hasMore === true,
+    },
+  };
+}
+
+/**
+ * Decide whether chat can answer from the immutable index before paying for a
+ * live source read. This gate is intentionally content-agnostic: it evaluates
+ * provenance, confidence, temporal coverage, and retrieval agreement rather
+ * than recognizing any particular kind of recording.
+ */
+export function indexedVideoEvidenceIsSufficient(input: {
+  verificationStatus: VideoEvidenceVerificationStatus;
+  questionKinds: string[];
+  evidence: IndexedVideoAnswerSignal[];
+  focusSources?: string[];
+  durationSecs?: number;
+  hierarchyRanges?: number;
+}) {
+  if (input.verificationStatus !== 'supported') return false;
+  const usable = input.evidence.filter(
+    (item) => !item.conflict && item.confidenceScore >= 0.5 && item.text.trim().length > 0,
+  );
+  if (usable.some((item) => /Claim verdict:\s*direct/i.test(item.text))) return true;
+
+  const descriptive = usable.filter((item) => ['visual', 'computed'].includes(item.modality));
+  const nonLexicalAgreement = (input.focusSources ?? []).some((source) => source !== 'lexical');
+
+  // A reconciled computed account is created from timestamped source evidence,
+  // not a remembered answer. It is the index's strongest reusable result.
+  if (descriptive.some((item) => item.modality === 'computed')) return true;
+
+  if (input.questionKinds.includes('state-change') || input.questionKinds.includes('outcome')) {
+    const duration = input.durationSecs;
+    const moments = new Set(descriptive.map((item) => Math.floor(item.startSecs / 15))).size;
+    const buckets = duration && duration > 0 ? 5 : 1;
+    const coveredBuckets = new Set(
+      descriptive.map((item) =>
+        duration && duration > 0
+          ? Math.min(buckets - 1, Math.floor((item.startSecs / duration) * buckets))
+          : 0,
+      ),
+    ).size;
+
+    // For outcome questions, if we have broad temporal coverage of the video,
+    // the index is sufficient without needing to rewatch the final 10 seconds blindly.
+    if (moments >= 3 && coveredBuckets / buckets >= 0.6 && (input.hierarchyRanges ?? 0) > 0) {
+      return true;
+    }
+  }
+
+  // A semantically located visual account can answer a focused appearance,
+  // comparison, count, or outcome question immediately. OCR/transcript-only
+  // neighbours do not pass this gate and therefore still trigger a close read.
+  return descriptive.length > 0 && nonLexicalAgreement;
+}

package/src/lib/chat/visual-routing.ts

@@ -0,0 +1,225 @@
+import { searchTerms } from './retrieval-ranking';
+
+type ImageDocument = {
+  metadata?: { images?: any[]; imageUrl?: unknown; index?: unknown; pageNumber?: unknown };
+};
+
+export type RetrievedPdfSource = {
+  documentId: string;
+  pageNumber?: number;
+  title?: string;
+};
+
+// A numbered object reference is stronger than a page number attached to a
+// vector hit. That hit may be an incidental mention, whereas this is an
+// explicit request for an object that the local PDF index can locate by its
+// own page text and caption.
+const NUMBERED_DOCUMENT_REFERENCE =
+  /\b(?:fig(?:ure)?|table|eq(?:uation)?|section|chapter|appendix)\.?\s*(?:no\.?\s*)?\(?\d+(?:\.\d+)+\b/iu;
+
+/** Resolve both current parent-document images and legacy standalone image records. */
+export function findIndexedImageSource<T extends ImageDocument>(documents: T[], imageUrl: string) {
+  const source = documents.find(
+    (document) =>
+      (Array.isArray(document.metadata?.images) &&
+        document.metadata.images.some((image: any) => image?.imageUrl === imageUrl)) ||
+      document.metadata?.imageUrl === imageUrl,
+  );
+  const image =
+    source?.metadata?.images?.find((candidate: any) => candidate?.imageUrl === imageUrl) ??
+    (source?.metadata?.imageUrl === imageUrl
+      ? {
+          imageUrl,
+          index: source.metadata.index,
+          pageNumber: source.metadata.pageNumber,
+        }
+      : undefined);
+  return image ? { source, image } : undefined;
+}
+
+function indexedImageDescriptions(value: unknown): string[] {
+  if (typeof value === 'string') {
+    try {
+      return indexedImageDescriptions(JSON.parse(value));
+    } catch {
+      return [];
+    }
+  }
+  if (Array.isArray(value)) return value.flatMap(indexedImageDescriptions);
+  if (!value || typeof value !== 'object') return [];
+  const record = value as Record<string, unknown>;
+  const here = Array.isArray(record.images)
+    ? record.images.flatMap((image) => {
+        if (!image || typeof image !== 'object') return [];
+        const description = (image as { description?: unknown }).description;
+        return typeof description === 'string' && description.trim() ? [description] : [];
+      })
+    : [];
+  return [...here, ...Object.values(record).flatMap(indexedImageDescriptions)];
+}
+
+/** Recognizes image-bearing retrieval results, including compact JSON envelopes. */
+export function hasRetrievedImageEvidence(value: unknown): boolean {
+  if (typeof value === 'string') {
+    try {
+      return hasRetrievedImageEvidence(JSON.parse(value));
+    } catch {
+      return false;
+    }
+  }
+  if (Array.isArray(value)) return value.some(hasRetrievedImageEvidence);
+  if (!value || typeof value !== 'object') return false;
+  const record = value as Record<string, unknown>;
+  if (Array.isArray(record.images) && record.images.length > 0) return true;
+  return Object.values(record).some(hasRetrievedImageEvidence);
+}
+
+/** Finds a PDF source returned by retrieval, even when it has no pre-indexed images. */
+export function hasRetrievedPdfEvidence(value: unknown): boolean {
+  if (findRetrievedPdfSource(value)) return true;
+  if (typeof value === 'string') {
+    try {
+      return hasRetrievedPdfEvidence(JSON.parse(value));
+    } catch {
+      return false;
+    }
+  }
+  if (Array.isArray(value)) return value.some(hasRetrievedPdfEvidence);
+  if (!value || typeof value !== 'object') return false;
+  const record = value as Record<string, unknown>;
+  const metadata =
+    record.metadata && typeof record.metadata === 'object'
+      ? (record.metadata as Record<string, unknown>)
+      : undefined;
+  return (
+    (typeof record.url === 'string' && /\.pdf(?:$|[?#])/i.test(record.url)) ||
+    (typeof record.title === 'string' && /\.pdf(?:$|[?#])/i.test(record.title)) ||
+    record.mimeType === 'application/pdf' ||
+    metadata?.mimeType === 'application/pdf' ||
+    metadata?.fileKind === 'pdf' ||
+    (typeof metadata?.originalFile === 'string' &&
+      /\.pdf(?:$|[?#])/i.test(metadata.originalFile)) ||
+    Object.values(record).some(hasRetrievedPdfEvidence)
+  );
+}
+
+function positivePageNumber(value: unknown): number | undefined {
+  return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined;
+}
+
+/** Whether the request names a stable, numbered object inside a document. */
+export function hasNumberedDocumentReference(text: string): boolean {
+  return NUMBERED_DOCUMENT_REFERENCE.test(text);
+}
+
+/** Return the nearest named document object from a chronological text history. */
+export function latestNumberedDocumentReferenceText(texts: readonly string[]): string | undefined {
+  for (let index = texts.length - 1; index >= 0; index -= 1) {
+    if (hasNumberedDocumentReference(texts[index])) return texts[index];
+  }
+  return undefined;
+}
+
+/**
+ * An explicit document reference must be ranked against the original PDF.
+ * Otherwise, a page attached to the retrieval hit is an efficient exact hint.
+ */
+export function preferredPdfPagesForInspection(
+  retrievedPageNumber: number | undefined,
+  question: string,
+): number[] | undefined {
+  if (hasNumberedDocumentReference(question)) return undefined;
+  return retrievedPageNumber ? [retrievedPageNumber] : undefined;
+}
+
+function pdfSourceFromRecord(value: unknown): RetrievedPdfSource | undefined {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
+  const record = value as Record<string, any>;
+  if (typeof record.documentId !== 'string' || !record.documentId) return undefined;
+  const metadata = record.metadata && typeof record.metadata === 'object' ? record.metadata : {};
+  const isPdf =
+    (typeof record.url === 'string' && /\.pdf(?:$|[?#])/i.test(record.url)) ||
+    (typeof record.title === 'string' && /\.pdf(?:$|[?#])/i.test(record.title)) ||
+    record.mimeType === 'application/pdf' ||
+    metadata.mimeType === 'application/pdf' ||
+    metadata.fileKind === 'pdf' ||
+    (typeof metadata.originalFile === 'string' && /\.pdf(?:$|[?#])/i.test(metadata.originalFile));
+  if (!isPdf) return undefined;
+  return {
+    documentId: record.documentId,
+    pageNumber: positivePageNumber(record.pageNumber) ?? positivePageNumber(metadata.pageNumber),
+    title: typeof record.title === 'string' ? record.title : undefined,
+  };
+}
+
+/**
+ * Select the leading PDF source returned by retrieval. A direct visual chunk
+ * carries its page number, so it is preferred over a document-level text
+ * chunk that would otherwise make a preview fall back to page one.
+ */
+export function findRetrievedPdfSource(value: unknown): RetrievedPdfSource | undefined {
+  if (typeof value === 'string') {
+    try {
+      return findRetrievedPdfSource(JSON.parse(value));
+    } catch {
+      return undefined;
+    }
+  }
+  if (!value || typeof value !== 'object') return undefined;
+  const record = value as Record<string, unknown>;
+  const hits = Array.isArray(record.hits) ? record.hits : [];
+  const sources = hits.map(pdfSourceFromRecord).filter(Boolean) as RetrievedPdfSource[];
+  return sources.find((source) => source.pageNumber !== undefined) ?? sources[0];
+}
+
+/**
+ * Reuse relevant descriptions produced during image indexing, and reserve a
+ * fresh vision pass for details those descriptions do not cover.
+ */
+export function shouldInspectRetrievedImage(text: string, evidence: unknown): boolean {
+  if (!text.trim() || !hasRetrievedImageEvidence(evidence) || requestsImagePresentation(text)) {
+    return false;
+  }
+
+  // Captions are useful for orientation, but they cannot prove an exhaustive
+  // visual request. A diagram question that asks for every label, a count, or
+  // an exact name must receive one bounded image read; otherwise a partial
+  // caption can make the assistant confidently omit the information the user
+  // actually asked it to enumerate.
+  if (
+    /\b(?:every|all|each|list|enumerate|count|how many|number of)\b|\b(?:view|routine|table|column|field|label|name)s?\b/i.test(
+      text,
+    )
+  ) {
+    return true;
+  }
+
+  const descriptions = indexedImageDescriptions(evidence).join(' ').toLocaleLowerCase();
+  if (!descriptions) return true;
+  const terms = searchTerms(text);
+  const matched = terms.filter((term) => descriptions.includes(term)).length;
+
+  // Image indexing already performed the expensive visual read. Reuse it when
+  // the stored description substantially covers this question; only reopen a
+  // vision call when the retrieval note does not address the requested detail.
+  return terms.length > 0 && matched / terms.length < 0.35;
+}
+
+/** Generic presentation intent, shared by PDFs, screenshots, scans, and media frames. */
+export function requestsImagePresentation(text: string): boolean {
+  return /\b(?:show|preview|display|open|view|see|need|give|provide|render)\b[\s\S]{0,40}\b(?:image|picture|diagram|page|visual|figure|it)\b|\b(?:image|picture|diagram|page|visual|figure)\s+(?:preview|view)\b/i.test(
+    text,
+  );
+}
+
+/**
+ * Page text is enough for ordinary prose. These requests explicitly depend on
+ * page layout, notation, or a visual object, so a rendered-page read is the
+ * higher-fidelity source of truth. The language is document-generic rather
+ * than tied to any particular paper or diagram format.
+ */
+export function requiresPdfVisualAnalysis(text: string): boolean {
+  return /\b(?:figure|fig\.?|diagram|chart|graph|image|picture|visual|table|equation|formula|notation|layout|caption|legend|label|matrix)\b|\\(?:frac|sqrt|sum|alpha|beta|gamma|delta|theta|[()[\]{}])|[∂∑√ᵀ]/i.test(
+    text,
+  );
+}

package/src/lib/enterprise-client.ts

@@ -0,0 +1,96 @@
+type EnterpriseProfile = {
+  dashboardUrl: string;
+  clientKey: string;
+  managedToolIds: string[];
+};
+
+export type EnterpriseToolDescriptor = {
+  id: string;
+  name: string;
+  description: string;
+  ui: 'notice' | 'table' | 'card';
+};
+
+function profile(config: any): EnterpriseProfile | null {
+  const enterprise = config?.enterprise as EnterpriseProfile | undefined;
+  if (!enterprise?.dashboardUrl || !enterprise.clientKey) return null;
+  return enterprise;
+}
+
+async function enterpriseFetch(config: any, path: string, init?: RequestInit) {
+  const enterprise = profile(config);
+  if (!enterprise) return null;
+  const url = new URL(path, enterprise.dashboardUrl);
+  const response = await fetch(url, {
+    ...init,
+    headers: {
+      Authorization: `Bearer ${enterprise.clientKey}`,
+      'Content-Type': 'application/json',
+      ...(init?.headers || {}),
+    },
+    signal: AbortSignal.timeout(12_000),
+  });
+  if (!response.ok) throw new Error('Enterprise private tool is unavailable.');
+  return response.json();
+}
+
+export async function getEnterpriseTools(config: any): Promise<EnterpriseToolDescriptor[]> {
+  const enterprise = profile(config);
+  if (!enterprise?.managedToolIds.length) return [];
+  try {
+    const data = (await enterpriseFetch(config, '/api/client/tools')) as {
+      tools?: EnterpriseToolDescriptor[];
+    } | null;
+    return (data?.tools || []).filter((tool) => enterprise.managedToolIds.includes(tool.id));
+  } catch (error) {
+    console.error('[enterprise] private tools unavailable:', error);
+    return [];
+  }
+}
+
+export async function executeEnterpriseTool(config: any, id: string, input: { query?: string }) {
+  return enterpriseFetch(config, `/api/client/tools/${encodeURIComponent(id)}/execute`, {
+    method: 'POST',
+    body: JSON.stringify(input),
+  });
+}
+
+/** Do this before model invocation so a paused or exhausted client cannot spend more. */
+export async function authorizeEnterpriseAiRequest(config: any) {
+  if (!profile(config)) return;
+  await enterpriseFetch(config, '/api/client/authorize', { method: 'POST', body: '{}' });
+}
+
+export function trackEnterpriseAiUsage(
+  config: any,
+  event: { modelId: string; inputTokens: number; outputTokens: number; costUsd: number },
+) {
+  void enterpriseFetch(config, '/api/client/usage', {
+    method: 'POST',
+    body: JSON.stringify({ service: 'ai', ...event }),
+  }).catch(() => undefined);
+}
+
+/**
+ * Reports one meter's usage for an installed Marketplace catalog tool. This
+ * is the `trackUsage` sink wired into every dynamically-loaded ToolExtension
+ * client for an EE-enrolled installation — generic across tools, since the
+ * meter id and quantity always come from the tool itself, never hardcoded
+ * here. A no-op when the workspace isn't enrolled with Enterprise.
+ */
+export function trackEnterpriseToolUsage(
+  config: any,
+  toolId: string,
+  event: { meter: string; quantity: number; unit: string },
+) {
+  void enterpriseFetch(config, '/api/client/usage', {
+    method: 'POST',
+    body: JSON.stringify({
+      service: 'tool',
+      toolId,
+      meterId: event.meter,
+      quantity: event.quantity,
+      unit: event.unit,
+    }),
+  }).catch(() => undefined);
+}

package/src/lib/marketplace/tool-runtime-config.ts

@@ -0,0 +1,155 @@
+import type { RagConfig } from '@larkup/core/types';
+
+/**
+ * Supplies a marketplace runtime with the user's selected AI Models settings
+ * without making the installed package import the host application's source.
+ */
+export function withGlobalVisionGatewayConfig(
+  toolConfig: Record<string, unknown>,
+  config: Pick<
+    RagConfig,
+    | 'visionProvider'
+    | 'visionApiKey'
+    | 'visionModelId'
+    | 'chatProvider'
+    | 'chatApiKey'
+    | 'chatModelId'
+    | 'embeddingProvider'
+    | 'embeddingApiKey'
+  >,
+): Record<string, unknown> {
+  const visionCapableProviders = new Set(['vercel_ai_gateway', 'openai', 'google']);
+  const toolConfigWithCurrentDefaults = migrateLegacyVideoIntelligenceDefaults(toolConfig, config);
+  const configuredGatewayKey =
+    (config.visionProvider === 'vercel_ai_gateway' ? config.visionApiKey : undefined) ||
+    (config.chatProvider === 'vercel_ai_gateway' ? config.chatApiKey : undefined) ||
+    (config.embeddingProvider === 'vercel_ai_gateway' ? config.embeddingApiKey : undefined);
+  // Model credentials must come from the user's saved project configuration.
+  // A server/developer environment key is intentionally never inherited.
+  const explicitVisionProvider =
+    config.visionProvider &&
+    config.visionApiKey &&
+    visionCapableProviders.has(config.visionProvider)
+      ? config.visionProvider
+      : undefined;
+  const inheritedChatProvider =
+    config.chatProvider && config.chatApiKey && visionCapableProviders.has(config.chatProvider)
+      ? config.chatProvider
+      : undefined;
+  const inheritedEmbeddingProvider =
+    config.embeddingProvider &&
+    config.embeddingApiKey &&
+    visionCapableProviders.has(config.embeddingProvider)
+      ? config.embeddingProvider
+      : undefined;
+  const visionProvider =
+    explicitVisionProvider ||
+    inheritedChatProvider ||
+    inheritedEmbeddingProvider ||
+    (configuredGatewayKey ? 'vercel_ai_gateway' : undefined);
+  const visionApiKey =
+    (explicitVisionProvider ? config.visionApiKey : undefined) ||
+    (visionProvider === config.chatProvider ? config.chatApiKey : undefined) ||
+    (visionProvider === config.embeddingProvider ? config.embeddingApiKey : undefined) ||
+    (visionProvider === 'vercel_ai_gateway' ? configuredGatewayKey : undefined);
+  const visionModel =
+    (explicitVisionProvider ? config.visionModelId : undefined) ||
+    (inheritedChatProvider && inheritedChatProvider === visionProvider
+      ? config.chatModelId
+      : undefined) ||
+    (visionProvider === 'vercel_ai_gateway' && config.chatProvider === 'vercel_ai_gateway'
+      ? config.chatModelId
+      : undefined);
+
+  // Older projects commonly have one OpenAI key saved with the embedding
+  // configuration before a separate Chat Model was introduced. Reuse it when
+  // it belongs to a provider that can run the video tool's planning pass.
+  const agentProvider =
+    (config.chatProvider && config.chatApiKey ? config.chatProvider : undefined) ||
+    (config.embeddingProvider &&
+    config.embeddingApiKey &&
+    visionCapableProviders.has(config.embeddingProvider)
+      ? config.embeddingProvider
+      : config.chatProvider);
+  const agentApiKey =
+    (agentProvider === config.chatProvider ? config.chatApiKey : undefined) ||
+    (agentProvider === config.embeddingProvider ? config.embeddingApiKey : undefined) ||
+    (agentProvider === 'vercel_ai_gateway' ? configuredGatewayKey : undefined);
+  return {
+    ...toolConfigWithCurrentDefaults,
+    larkupVisionProvider: visionProvider,
+    larkupVisionApiKey: visionApiKey,
+    larkupVisionModel: visionModel,
+    larkupAgentProvider: agentProvider,
+    larkupAgentApiKey: agentApiKey,
+    larkupAgentModel:
+      config.chatModelId ||
+      (agentProvider === 'google'
+        ? 'google/gemini-3.5-flash-lite'
+        : agentProvider === 'openai'
+          ? 'openai/gpt-5-mini'
+          : undefined),
+  };
+}
+
+/**
+ * Versions through 0.2.4 persisted provider-specific values that were meant
+ * to be defaults. Those values shadow AI Models forever, so an existing
+ * OpenAI setup can look unconfigured after installing Video Intelligence.
+ * Treat only the exact old, keyless defaults as automatic. Real overrides and
+ * Gateway-based AI Models selections remain untouched.
+ */
+function migrateLegacyVideoIntelligenceDefaults(
+  toolConfig: Record<string, unknown>,
+  config: Pick<
+    RagConfig,
+    | 'visionProvider'
+    | 'visionApiKey'
+    | 'chatProvider'
+    | 'chatApiKey'
+    | 'embeddingProvider'
+    | 'embeddingApiKey'
+  >,
+): Record<string, unknown> {
+  if (!('videoVisionProvider' in toolConfig) && !('videoAgentProvider' in toolConfig)) {
+    return toolConfig;
+  }
+
+  const value = (key: string) =>
+    typeof toolConfig[key] === 'string' ? toolConfig[key].trim() : undefined;
+  const hasSavedVisionDefault = Boolean(
+    (config.visionProvider && config.visionApiKey) ||
+    (config.chatProvider && config.chatApiKey) ||
+    (config.embeddingProvider && config.embeddingApiKey),
+  );
+  const hasSavedAgentDefault = Boolean(
+    (config.chatProvider && config.chatApiKey) ||
+    (config.embeddingProvider && config.embeddingApiKey),
+  );
+  const usesSavedGateway =
+    config.visionProvider === 'vercel_ai_gateway' || config.chatProvider === 'vercel_ai_gateway';
+  const migrated = { ...toolConfig };
+
+  if (
+    hasSavedVisionDefault &&
+    !usesSavedGateway &&
+    value('videoVisionProvider') === 'vercel_ai_gateway' &&
+    !value('videoVisionApiKey') &&
+    value('semanticVisionModel') === 'google/gemini-3.6-flash'
+  ) {
+    migrated.videoVisionProvider = 'auto';
+    migrated.semanticVisionModel = 'auto';
+  }
+  if (
+    hasSavedAgentDefault &&
+    config.chatProvider !== 'vercel_ai_gateway' &&
+    config.embeddingProvider !== 'vercel_ai_gateway' &&
+    value('videoAgentProvider') === 'vercel_ai_gateway' &&
+    !value('videoAgentApiKey') &&
+    value('agentModel') === 'openai/gpt-5-mini'
+  ) {
+    migrated.videoAgentProvider = 'auto';
+    migrated.agentModel = 'auto';
+  }
+  return migrated;
+}