larkup 0.4.12 → 0.4.14

package/.next/standalone/apps/web/lib/marketplace/tool-runtime-config.test.ts

@@ -0,0 +1,217 @@
+import { describe, expect, it } from 'vitest';
+import { withGlobalVisionGatewayConfig } from './tool-runtime-config';
+
+describe('withGlobalVisionGatewayConfig', () => {
+  it('does not inherit an operator AI Gateway environment secret', () => {
+    process.env.AI_GATEWAY_API_KEY = 'gateway-from-environment';
+
+    expect(
+      withGlobalVisionGatewayConfig(
+        {},
+        {
+          visionProvider: 'vercel_ai_gateway',
+          visionApiKey: '',
+          visionModelId: 'google/gemini-3.6-flash',
+          chatProvider: 'vercel_ai_gateway',
+          chatApiKey: '',
+          chatModelId: '',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({ larkupVisionApiKey: undefined, larkupAgentApiKey: undefined });
+    delete process.env.AI_GATEWAY_API_KEY;
+  });
+
+  it('keeps the selected tool-brain provider and model independent from vision', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {
+          videoVisionProvider: 'google',
+          videoVisionApiKey: 'vision-key',
+          semanticVisionModel: 'google/gemini-3.6-flash',
+          videoAgentProvider: 'openai',
+          videoAgentApiKey: 'agent-key',
+          agentModel: 'openai/gpt-5-mini',
+        },
+        {
+          visionProvider: 'google',
+          visionApiKey: 'global-vision-key',
+          visionModelId: 'google/gemini-3.6-flash',
+          chatProvider: 'openai',
+          chatApiKey: 'global-agent-key',
+          chatModelId: 'openai/gpt-5.2',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({
+      larkupVisionProvider: 'google',
+      larkupVisionApiKey: 'global-vision-key',
+      larkupVisionModel: 'google/gemini-3.6-flash',
+      videoVisionProvider: 'google',
+      videoVisionApiKey: 'vision-key',
+      semanticVisionModel: 'google/gemini-3.6-flash',
+      larkupAgentProvider: 'openai',
+      larkupAgentApiKey: 'global-agent-key',
+      larkupAgentModel: 'openai/gpt-5.2',
+      videoAgentProvider: 'openai',
+      videoAgentApiKey: 'agent-key',
+      agentModel: 'openai/gpt-5-mini',
+    });
+  });
+
+  it('leaves video model credentials unconfigured when the user saved none', () => {
+    process.env.AI_GATEWAY_API_KEY = 'gateway-from-environment';
+
+    expect(
+      withGlobalVisionGatewayConfig(
+        {},
+        {
+          visionProvider: '',
+          visionApiKey: '',
+          visionModelId: '',
+          chatProvider: '',
+          chatApiKey: '',
+          chatModelId: '',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({ larkupVisionProvider: undefined, larkupVisionApiKey: undefined });
+    delete process.env.AI_GATEWAY_API_KEY;
+  });
+
+  it('does not treat a text-only DeepSeek chat provider as the vision provider', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {},
+        {
+          visionProvider: '',
+          visionApiKey: '',
+          visionModelId: '',
+          chatProvider: 'deepseek',
+          chatApiKey: 'deepseek-key',
+          chatModelId: 'deepseek/deepseek-v4-pro',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({
+      larkupAgentProvider: 'deepseek',
+      larkupAgentApiKey: 'deepseek-key',
+      larkupAgentModel: 'deepseek/deepseek-v4-pro',
+      larkupVisionProvider: undefined,
+      larkupVisionApiKey: undefined,
+      larkupVisionModel: undefined,
+    });
+  });
+
+  it('ignores a saved DeepSeek vision selection because it is text-only', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {},
+        {
+          visionProvider: 'deepseek',
+          visionApiKey: 'deepseek-key',
+          visionModelId: 'deepseek/deepseek-v4-pro',
+          chatProvider: 'deepseek',
+          chatApiKey: 'deepseek-key',
+          chatModelId: 'deepseek/deepseek-v4-pro',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({
+      larkupAgentProvider: 'deepseek',
+      larkupAgentApiKey: 'deepseek-key',
+      larkupVisionProvider: undefined,
+      larkupVisionApiKey: undefined,
+      larkupVisionModel: undefined,
+    });
+  });
+
+  it('reuses one vision-capable chat provider and key automatically', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {},
+        {
+          visionProvider: '',
+          visionApiKey: '',
+          visionModelId: '',
+          chatProvider: 'google',
+          chatApiKey: 'google-key',
+          chatModelId: 'google/gemini-3.6-flash',
+          embeddingProvider: 'openai',
+          embeddingApiKey: '',
+        },
+      ),
+    ).toMatchObject({
+      larkupAgentProvider: 'google',
+      larkupAgentApiKey: 'google-key',
+      larkupVisionProvider: 'google',
+      larkupVisionApiKey: 'google-key',
+      larkupVisionModel: 'google/gemini-3.6-flash',
+    });
+  });
+
+  it('migrates the old keyless Video Intelligence defaults to saved OpenAI settings', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {
+          videoVisionProvider: 'vercel_ai_gateway',
+          semanticVisionModel: 'google/gemini-3.6-flash',
+          videoAgentProvider: 'vercel_ai_gateway',
+          agentModel: 'openai/gpt-5-mini',
+        },
+        {
+          visionProvider: '',
+          visionApiKey: '',
+          visionModelId: '',
+          chatProvider: 'openai',
+          chatApiKey: 'openai-key',
+          chatModelId: 'openai/gpt-5-mini',
+          embeddingProvider: 'openai',
+          embeddingApiKey: 'embedding-key',
+        },
+      ),
+    ).toMatchObject({
+      videoVisionProvider: 'auto',
+      semanticVisionModel: 'auto',
+      videoAgentProvider: 'auto',
+      agentModel: 'auto',
+      larkupVisionProvider: 'openai',
+      larkupVisionApiKey: 'openai-key',
+      larkupAgentProvider: 'openai',
+      larkupAgentApiKey: 'openai-key',
+    });
+  });
+
+  it('uses a compatible saved embedding key when no chat key has been configured yet', () => {
+    expect(
+      withGlobalVisionGatewayConfig(
+        {
+          videoVisionProvider: 'vercel_ai_gateway',
+          semanticVisionModel: 'google/gemini-3.6-flash',
+          videoAgentProvider: 'vercel_ai_gateway',
+          agentModel: 'openai/gpt-5-mini',
+        },
+        {
+          visionProvider: '',
+          visionApiKey: '',
+          visionModelId: '',
+          chatProvider: '',
+          chatApiKey: '',
+          chatModelId: '',
+          embeddingProvider: 'openai',
+          embeddingApiKey: 'openai-key',
+        },
+      ),
+    ).toMatchObject({
+      larkupVisionProvider: 'openai',
+      larkupVisionApiKey: 'openai-key',
+      larkupAgentProvider: 'openai',
+      larkupAgentApiKey: 'openai-key',
+    });
+  });
+});

package/.next/standalone/apps/web/lib/media/knowledge.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from 'vitest';
+import { buildMediaDocumentInputs } from './knowledge';
+
+describe('buildMediaDocumentInputs', () => {
+  it('clamps timestamp evidence and chapters to the source duration', () => {
+    const documents = buildMediaDocumentInputs({
+      assetId: 'asset-1',
+      title: 'Demo',
+      mediaType: 'video',
+      localUrl: '/api/media/asset-1',
+      durationSecs: 60,
+      summary: 'Summary',
+      segments: [
+        {
+          text: 'Opening evidence',
+          startSecs: -30,
+          endSecs: 90,
+          sequence: 0,
+        },
+      ],
+      fileName: 'demo.mp4',
+      mimeType: 'video/mp4',
+      transcriptSource: 'test',
+    });
+
+    const chapter = documents.find(
+      (document) => document.metadata?.contentKind === 'media-chapter',
+    );
+    const segment = documents.find(
+      (document) => document.metadata?.contentKind === 'multimodal-segment',
+    );
+    expect(chapter?.metadata).toMatchObject({ chapterIndex: 0, startSecs: 0, endSecs: 60 });
+    expect(segment?.metadata).toMatchObject({ startSecs: 0, endSecs: 60 });
+    expect(segment?.url).toBe('/api/media/asset-1#t=0');
+  });
+});

package/.next/standalone/apps/web/lib/media/pdf-inspection.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from 'vitest';
+import { selectInspectedPdfPages, selectRelevantPdfPages } from './pdf-inspection';
+
+describe('selectRelevantPdfPages', () => {
+  it('selects the pages nearest to the requested evidence without document-specific rules', () => {
+    expect(
+      selectRelevantPdfPages(
+        [
+          { num: 1, text: 'Overview and introduction.' },
+          { num: 2, text: 'The dependency diagram has service and storage relationships.' },
+          { num: 3, text: 'Appendix.' },
+        ],
+        'Explain the dependency relationships in the diagram',
+      ),
+    ).toEqual([2, 1, 3]);
+  });
+
+  it('uses a bounded fallback when the question has no searchable terms', () => {
+    expect(
+      selectRelevantPdfPages(
+        [
+          { num: 4, text: 'A' },
+          { num: 5, text: 'B' },
+        ],
+        '?',
+      ),
+    ).toEqual([4, 5]);
+  });
+
+  it('prefers an exact numbered document reference over incidental number matches', () => {
+    expect(
+      selectRelevantPdfPages(
+        [
+          { num: 1, text: 'Section 5 contains an overview and many references to attention.' },
+          { num: 2, text: 'Figure 5.1: Residual connections in the decoder architecture.' },
+          { num: 3, text: 'Table 5.1 lists implementation values.' },
+        ],
+        'What do the orange dashed lines in Figure 5.1 mean?',
+      )[0],
+    ).toBe(2);
+  });
+
+  it('prefers the caption page over prose that only refers to a numbered visual', () => {
+    expect(
+      selectRelevantPdfPages(
+        [
+          { num: 15, text: 'The computational flow is illustrated in Figure 2.1.' },
+          { num: 16, text: 'Figure 2.1: Computational flow of vectorized attention.' },
+        ],
+        'Render diagram 2.1',
+      )[0],
+    ).toBe(16);
+  });
+
+  it('returns the ranked full-text pages instead of the first pages in the PDF', () => {
+    const pages = [
+      { num: 1, text: 'Cover page.' },
+      { num: 2, text: 'Introduction.' },
+      { num: 8, text: 'Figure 2.1: Vectorized attention flow.' },
+    ];
+    expect(selectInspectedPdfPages(pages, [8, 2, 1]).map((page) => page.num)).toEqual([8, 2, 1]);
+  });
+});

package/.next/standalone/apps/web/lib/media/pdf.ts

@@ -0,0 +1,57 @@
+import * as pdfjsLib from 'pdfjs-dist';
+
+export async function extractImagesFromPDF(
+  file: File,
+): Promise<{ base64: string; pageNumber: number; index: number }[]> {
+  pdfjsLib.GlobalWorkerOptions.workerSrc = new URL(
+    'pdfjs-dist/build/pdf.worker.min.mjs',
+    import.meta.url,
+  ).toString();
+
+  const arrayBuffer = await file.arrayBuffer();
+  const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(arrayBuffer) }).promise;
+  const numPages = pdf.numPages;
+  const images: { base64: string; pageNumber: number; index: number }[] = [];
+
+  let imageIndex = 0;
+  const toDataUrl = (source: HTMLCanvasElement) => {
+    const maxDimension = 1_600;
+    const scale = Math.min(1, maxDimension / Math.max(source.width, source.height));
+    if (scale === 1) return source.toDataURL('image/jpeg', 0.86);
+
+    const resized = document.createElement('canvas');
+    resized.width = Math.max(1, Math.round(source.width * scale));
+    resized.height = Math.max(1, Math.round(source.height * scale));
+    resized.getContext('2d')?.drawImage(source, 0, 0, resized.width, resized.height);
+    return resized.toDataURL('image/jpeg', 0.86);
+  };
+
+  for (let i = 1; i <= numPages; i++) {
+    try {
+      const page = await pdf.getPage(i);
+      // A PDF page is a composite visual: diagrams are commonly vector
+      // drawings, while their caption, legend, and table may be text or
+      // separate XObjects. Indexing XObjects individually loses that context
+      // and makes image zero (often a cover) look like the best preview.
+      // One crisp rendered page preserves the complete source relationship.
+      const baseViewport = page.getViewport({ scale: 1 });
+      const desiredLongestEdge = 1_600;
+      const scale = Math.min(
+        2.5,
+        Math.max(1, desiredLongestEdge / Math.max(baseViewport.width, baseViewport.height)),
+      );
+      const viewport = page.getViewport({ scale });
+      const canvas = document.createElement('canvas');
+      canvas.width = Math.max(1, Math.round(viewport.width));
+      canvas.height = Math.max(1, Math.round(viewport.height));
+      const ctx = canvas.getContext('2d');
+      if (!ctx) continue;
+      await page.render({ canvas, viewport }).promise;
+      images.push({ base64: toDataUrl(canvas), pageNumber: i, index: imageIndex++ });
+    } catch (err) {
+      console.error(`Error processing page ${i}`, err);
+    }
+  }
+
+  return images;
+}

package/.next/standalone/apps/web/lib/media/progress.test.ts

@@ -0,0 +1,208 @@
+import { describe, expect, it } from 'vitest';
+import { MEDIA_PIPELINE_STAGES, weightedProcessingProgress } from '@larkup/core/media-store';
+import type { MediaProcessingStep } from '@larkup/core/types';
+import {
+  describeActiveMediaStep,
+  completedMediaProcessingSeconds,
+  estimateMediaStepRemainingSeconds,
+  formatMediaProgressPercent,
+  isMediaStepTelemetryStale,
+  primaryRunningMediaStep,
+  isFinalizingMediaStep,
+  advanceMediaProcessingProgress,
+  smoothMediaProcessingProgress,
+} from './progress';
+
+describe('smoothMediaProcessingProgress', () => {
+  it('stops projecting when the latest worker update is stale', () => {
+    const startedAt = '2026-08-31T10:00:00.000Z';
+    const asset = {
+      processingStatus: 'processing',
+      processingProgress: 42,
+      processingSteps: [
+        { stage: 'vision', status: 'running' as const, percent: 2, startedAt },
+        { stage: 'synthesize', status: 'waiting' as const },
+        { stage: 'index', status: 'waiting' as const },
+      ],
+    };
+
+    const later = smoothMediaProcessingProgress(
+      asset,
+      new Date('2026-08-31T10:01:00.000Z').getTime(),
+    );
+    expect(later).toBe(42);
+  });
+
+  it('keeps the latest worker measurement as the confirmed source value', () => {
+    const updatedAt = '2026-08-31T10:00:00.000Z';
+    const asset = {
+      processingStatus: 'processing',
+      processingProgress: 0,
+      processingSteps: [{ stage: 'download', status: 'running' as const, percent: 0, updatedAt }],
+    };
+
+    expect(
+      smoothMediaProcessingProgress(asset, new Date('2026-08-31T10:00:19.000Z').getTime()),
+    ).toBe(0);
+  });
+
+  it('eases a sparse worker jump over multiple browser ticks', () => {
+    expect(advanceMediaProcessingProgress(0, 8, 250)).toBe(0.5);
+    expect(advanceMediaProcessingProgress(0.5, 8, 250)).toBe(1);
+  });
+
+  it('keeps moving slowly only while fresh worker activity is present', () => {
+    expect(advanceMediaProcessingProgress(0, 0, 1_000, true)).toBeCloseTo(0.099, 3);
+    expect(advanceMediaProcessingProgress(0.099, 0, 1_000, false)).toBe(0.099);
+  });
+
+  it('does not animate a paused attempt or fabricate completion', () => {
+    const asset = {
+      processingStatus: 'processing',
+      processingPaused: true,
+      processingProgress: 27,
+      processingSteps: [
+        {
+          stage: 'vision',
+          status: 'running' as const,
+          percent: 2,
+          startedAt: '2026-08-31T10:00:00.000Z',
+        },
+      ],
+    };
+    expect(smoothMediaProcessingProgress(asset, Date.now())).toBe(27);
+  });
+});
+
+describe('live progress precision', () => {
+  it('preserves measured tenths in the weighted overall pipeline', () => {
+    const steps: MediaProcessingStep[] = MEDIA_PIPELINE_STAGES.map((stage) => ({
+      stage,
+      status: 'waiting' as const,
+      updatedAt: '2026-09-02T10:00:00.000Z',
+    }));
+    steps[0] = {
+      ...steps[0],
+      status: 'running',
+      percent: 21,
+    };
+
+    expect(weightedProcessingProgress(steps)).toBe(1.8);
+    expect(formatMediaProgressPercent(1.8)).toBe('1.8');
+    expect(formatMediaProgressPercent(47)).toBe('47');
+  });
+});
+
+describe('primaryRunningMediaStep', () => {
+  it('shows the freshest running stage instead of the last stage in pipeline order', () => {
+    const steps = [
+      {
+        stage: 'prepare',
+        status: 'running' as const,
+        updatedAt: '2026-09-01T10:00:10.000Z',
+      },
+      {
+        stage: 'extract',
+        status: 'running' as const,
+        updatedAt: '2026-09-01T10:00:00.000Z',
+      },
+    ];
+    expect(primaryRunningMediaStep(steps)?.stage).toBe('prepare');
+  });
+});
+
+describe('isFinalizingMediaStep', () => {
+  it('names Cloud source preparation as a determinate stage, not a fake finalization state', () => {
+    const preparation = { stage: 'prepare', status: 'running' as const, percent: 46 };
+
+    expect(isFinalizingMediaStep(preparation)).toBe(false);
+    expect(describeActiveMediaStep(preparation)).toBe('Cloud video preparation · 46% this step');
+  });
+
+  it('does not promise imminent completion while visual indexing is still running', () => {
+    const visualIndexing = { stage: 'vision', status: 'running' as const, percent: 99 };
+
+    expect(isFinalizingMediaStep(visualIndexing)).toBe(false);
+    expect(describeActiveMediaStep(visualIndexing)).toBe('Visual analysis · 99% this step');
+  });
+
+  it('uses finalizing only for a nearly complete final search-indexing step', () => {
+    expect(isFinalizingMediaStep({ stage: 'index', status: 'running', percent: 98 })).toBe(true);
+  });
+});
+
+describe('estimateMediaStepRemainingSeconds', () => {
+  it('hides an expired single-digit remote estimate', () => {
+    const now = Date.now();
+    expect(
+      estimateMediaStepRemainingSeconds(
+        {
+          stage: 'synthesize',
+          status: 'running',
+          updatedAt: new Date(now).toISOString(),
+          estimatedRemainingSeconds: 1,
+        },
+        now,
+      ),
+    ).toBeNull();
+  });
+  it('estimates from completed count rather than an optimistic stage percent', () => {
+    const startedAt = '2026-08-28T10:00:00.000Z';
+    expect(
+      estimateMediaStepRemainingSeconds(
+        { stage: 'vision', status: 'running', percent: 99, current: 20, total: 100, startedAt },
+        new Date('2026-08-28T10:02:00.000Z').getTime(),
+      ),
+    ).toBe(480);
+  });
+
+  it('does not invent an ETA before enough countable work has completed', () => {
+    expect(
+      estimateMediaStepRemainingSeconds({
+        stage: 'vision',
+        status: 'running',
+        current: 1,
+        total: 100,
+        startedAt: '2026-08-28T10:00:00.000Z',
+      }),
+    ).toBeNull();
+  });
+
+  it('hides an expired or stale worker ETA instead of parking at one second', () => {
+    const step = {
+      stage: 'vision',
+      status: 'running' as const,
+      estimatedRemainingSeconds: 10,
+      updatedAt: '2026-08-28T10:00:00.000Z',
+    };
+    const now = new Date('2026-08-28T10:00:25.000Z').getTime();
+    expect(isMediaStepTelemetryStale(step, now)).toBe(true);
+    expect(estimateMediaStepRemainingSeconds(step, now)).toBeNull();
+  });
+});
+
+describe('completedMediaProcessingSeconds', () => {
+  it('uses the persisted indexing attempt start and final step completion', () => {
+    expect(
+      completedMediaProcessingSeconds({
+        processingStatus: 'completed',
+        processingStartedAt: '2026-08-29T10:00:00.000Z',
+        processingSteps: [
+          {
+            stage: 'index',
+            status: 'completed',
+            finishedAt: '2026-08-29T10:07:32.000Z',
+          },
+        ],
+      }),
+    ).toBe(452);
+  });
+
+  it('does not show a duration when a completed legacy asset lacks an attempt start', () => {
+    expect(
+      completedMediaProcessingSeconds({
+        processingStatus: 'completed',
+      }),
+    ).toBeNull();
+  });
+});