larkup 0.1.14 → 0.1.21

package/app/api/chat/tools.ts

@@ -0,0 +1,995 @@
+import { tool } from 'ai';
+import { z } from 'zod';
+import { readConfig } from '@larkup/core/config-store';
+import { readRun } from '@larkup/core/index-store';
+import { refreshServerStatus } from '@larkup/core/generator/server-runtime';
+import { createAdapter } from '@larkup/vector-stores/factory';
+import { embedQuery } from '@larkup/core/indexing/embedder';
+import { runWithServer } from '@larkup/core/workspace';
+import { getTabularDataset, queryTabular } from '@larkup/core/tabular-store';
+import {
+  getCorpusDocuments,
+  exportCorpusAsCSV,
+  exportCorpusAsJSONL,
+  type CorpusFilter,
+} from '@larkup/core/corpus-retriever';
+import { SandboxManager } from '@larkup/sandbox';
+import { applyFieldEdits, applyContentEdits } from '@larkup/tool-doc-editor';
+import { loadTool } from '@larkup/marketplace/loader';
+import { getInstalledTools } from '@larkup/marketplace/installer';
+import { readDocuments } from '@larkup/core/documents-store';
+import { readMediaAssets } from '@larkup/core/media-store';
+import {
+  normalizeMediaCitationRange,
+  queryAwareExcerpt,
+  timestampMediaUrl,
+} from '@/lib/media-knowledge';
+
+function documentEditModelOutput({ output }: { output: any }) {
+  return {
+    type: 'json' as const,
+    value: output?.success
+      ? {
+          success: true,
+          action: output.action,
+          sessionId: output.sessionId,
+          updatedFields: output.updatedFields ?? [],
+          fileName: output.fileName,
+          totalPages: output.totalPages,
+          message:
+            'The document was updated successfully. Do not repeat the edit unless requested.',
+        }
+      : {
+          success: false,
+          error: output?.error ?? 'Document update failed',
+        },
+  };
+}
+
+/**
+ * Retrieves documents from the knowledge base
+ */
+export async function queryKnowledgeBase(query: string, topK: number, serverId: string | null) {
+  const doRetrieve = async () => {
+    const config = await readConfig();
+    const candidateCount = Math.max(topK * 4, 20);
+
+    // 1) Try running generated server first
+    const server = await refreshServerStatus();
+    if (server.running) {
+      try {
+        const start = Date.now();
+        const res = await fetch(`${server.endpoint}/query`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ query, topK: candidateCount }),
+          signal: AbortSignal.timeout(15_000),
+        });
+        const data = await res.json();
+
+        const { trackUsageEvent } = await import('@larkup/core/analytics-store');
+        void trackUsageEvent({
+          type: 'server_request',
+          endpoint: '/query',
+          method: 'POST',
+          statusCode: res.status,
+          latencyMs: Date.now() - start,
+          serverId: serverId ?? undefined,
+          timestamp: new Date().toISOString(),
+        });
+
+        if (res.ok && data.hits) {
+          return formatKnowledgeHits(query, data.hits as any[], topK);
+        }
+      } catch {
+        // Fall through to direct retrieval
+      }
+    }
+
+    // 2) Direct retrieval from local vector store
+    const run = await readRun();
+    if (!run || run.status !== 'completed' || (run.totalChunks ?? 0) === 0) {
+      return { query, hits: [] };
+    }
+
+    const vector = await embedQuery(config, query);
+    const adapter = await createAdapter(config);
+    const hits = await adapter.query(vector, candidateCount, query);
+    return formatKnowledgeHits(query, hits, topK);
+  };
+
+  return serverId ? runWithServer(serverId, doRetrieve) : doRetrieve();
+}
+
+async function formatKnowledgeHits(query: string, rawHits: any[], topK: number) {
+  const [documents, mediaAssets] = await Promise.all([readDocuments(), readMediaAssets()]);
+  const documentsById = new Map(documents.map((document) => [document.id, document]));
+  const activeMediaDocumentIds = new Set(mediaAssets.flatMap((asset) => asset.documentIds));
+  const hydrated = rawHits.map((hit) => {
+    const document = documentsById.get(hit.documentId);
+    return {
+      ...hit,
+      title: hit.title || document?.title || 'Untitled',
+      url: hit.url || document?.url || '',
+      text: hit.text || document?.content || '',
+      metadata: { ...document?.metadata, ...hit.metadata },
+    };
+  });
+
+  const selected: any[] = [];
+  const hitsPerDocument = new Map<string, number>();
+  for (const hit of hydrated) {
+    if (
+      hit.metadata?.mediaAssetId &&
+      (!documentsById.has(hit.documentId) || !activeMediaDocumentIds.has(hit.documentId))
+    ) {
+      continue;
+    }
+    const seen = hitsPerDocument.get(hit.documentId) ?? 0;
+    const isTimestampedMedia =
+      hit.metadata?.isMediaSummary === true ||
+      hit.metadata?.contentKind === 'multimodal-segment' ||
+      hit.metadata?.contentKind === 'audio-transcript-segment';
+    // One strong hit per timestamped evidence document leaves room for other
+    // moments instead of returning eight chunks from the same summary.
+    if (seen >= (isTimestampedMedia ? 1 : 3)) continue;
+    selected.push(hit);
+    hitsPerDocument.set(hit.documentId, seen + 1);
+    if (selected.length >= topK) break;
+  }
+  const primaryMediaAssetId = selected.find((hit) => hit.metadata?.mediaAssetId)?.metadata
+    .mediaAssetId;
+  const mediaDocuments = primaryMediaAssetId
+    ? documents.filter(
+        (document) =>
+          document.status === 'indexed' &&
+          document.metadata?.mediaAssetId === primaryMediaAssetId &&
+          activeMediaDocumentIds.has(document.id),
+      )
+    : [];
+  const timestampedDocuments = mediaDocuments
+    .filter((document) => Number.isFinite(Number(document.metadata?.endSecs)))
+    .sort(
+      (left, right) => Number(left.metadata?.endSecs ?? 0) - Number(right.metadata?.endSecs ?? 0),
+    );
+  const mediaDuration = Math.max(
+    0,
+    ...timestampedDocuments.map((document) =>
+      Number(document.metadata?.durationSecs ?? document.metadata?.endSecs ?? 0),
+    ),
+  );
+  const trailingHorizonSecs = Math.max(10 * 60, Math.min(30 * 60, mediaDuration * 0.2));
+  const trailingDocuments = timestampedDocuments.filter(
+    (document) => Number(document.metadata?.endSecs ?? 0) >= mediaDuration - trailingHorizonSecs,
+  );
+  const signaledEndingDocuments = isOutcomeQuery(query)
+    ? trailingDocuments.filter((document) => isOutcomeEvidence(document.content)).slice(-2)
+    : [];
+  const timestampedEnding = [
+    ...new Map(
+      [...signaledEndingDocuments, ...trailingDocuments.slice(-4)].map((document) => [
+        document.id,
+        document,
+      ]),
+    ).values(),
+  ].sort(
+    (left, right) => Number(left.metadata?.endSecs ?? 0) - Number(right.metadata?.endSecs ?? 0),
+  );
+  const mediaSummary = mediaDocuments.find(
+    (document) => document.metadata?.isMediaSummary === true,
+  );
+  const legacyWholeTimeline =
+    timestampedEnding.length === 0
+      ? mediaDocuments.find((document) => document.metadata?.contentKind === 'multimodal-timeline')
+      : undefined;
+  // A compact ending context is returned for every primary media match. This
+  // avoids depending on a brittle English/Arabic keyword gate for formulations
+  // such as "which team became champion?" or equivalent questions in any language.
+  const endingContext = primaryMediaAssetId
+    ? [
+        ...(mediaSummary
+          ? [
+              {
+                role: 'summary',
+                title: mediaSummary.title,
+                url: mediaSummary.url,
+                text: queryAwareExcerpt(mediaSummary.content, query, 6_000),
+                startSecs: undefined,
+                endSecs: undefined,
+              },
+            ]
+          : []),
+        ...timestampedEnding.map((document) => ({
+          role: 'ending',
+          title: document.title,
+          url: document.url,
+          text: queryAwareExcerpt(document.content, query, 3_200, true),
+          startSecs: document.metadata?.startSecs,
+          endSecs: document.metadata?.endSecs,
+        })),
+        ...(legacyWholeTimeline
+          ? [
+              {
+                role: 'ending',
+                title: legacyWholeTimeline.title,
+                url: legacyWholeTimeline.url,
+                // Legacy media was one giant document. Its ending is the tail,
+                // never the beginning.
+                text: queryAwareExcerpt(legacyWholeTimeline.content, query, 4_000, true),
+                startSecs: undefined,
+                endSecs: legacyWholeTimeline.metadata?.durationSecs,
+              },
+            ]
+          : []),
+      ]
+    : undefined;
+
+  return {
+    query,
+    hits: selected.map((hit, hitIndex) => {
+      const sequence = Number(hit.metadata?.sequence);
+      const startSecs = Number(hit.metadata?.startSecs);
+      const timelineContext = hit.metadata?.mediaAssetId
+        ? documents
+            .filter((document) => {
+              const candidateSequence = Number(document.metadata?.sequence);
+              const candidateStart = Number(document.metadata?.startSecs);
+              return (
+                document.status === 'indexed' &&
+                activeMediaDocumentIds.has(document.id) &&
+                document.metadata?.mediaAssetId === hit.metadata.mediaAssetId &&
+                Number.isFinite(sequence) &&
+                Number.isFinite(candidateSequence) &&
+                Math.abs(candidateSequence - sequence) <= 1 &&
+                // Sequence is the authoritative adjacency marker. Fixed
+                // 90-second distance broke wider adaptive windows used for
+                // multi-hour camera and screen recordings.
+                (!Number.isFinite(startSecs) ||
+                  !Number.isFinite(candidateStart) ||
+                  Math.abs(candidateStart - startSecs) <=
+                    Math.max(
+                      90,
+                      Number(hit.metadata?.endSecs ?? startSecs) - startSecs,
+                      Number(document.metadata?.endSecs ?? candidateStart) - candidateStart,
+                    ))
+              );
+            })
+            .sort(
+              (left, right) =>
+                Number(left.metadata?.startSecs ?? 0) - Number(right.metadata?.startSecs ?? 0),
+            )
+            .map((document) => ({
+              role:
+                Number(document.metadata?.sequence) < sequence
+                  ? 'before'
+                  : Number(document.metadata?.sequence) > sequence
+                  ? 'after'
+                  : 'matched',
+              title: document.title,
+              url: document.url,
+              text: queryAwareExcerpt(document.content, query, 3_200),
+              startSecs: document.metadata?.startSecs,
+              endSecs: document.metadata?.endSecs,
+            }))
+        : undefined;
+
+      return {
+        documentId: hit.documentId,
+        title: hit.title ?? 'Untitled',
+        url: hit.url ?? '',
+        score: Number((hit.score ?? 0).toFixed(3)),
+        text: queryAwareExcerpt(hit.text ?? '', query, 4_000),
+        metadata: hit.metadata,
+        timelineContext,
+        endingContext: hitIndex === 0 ? endingContext : undefined,
+      };
+    }),
+  };
+}
+
+function isOutcomeQuery(query: string): boolean {
+  return /\b(who won|winner|won|final score|scoreline|result|beat|defeated|victor|champion|took (?:the )?(?:game|match|round))\b|من فاز|مين فاز|الفائز|النتيجة|نتيجة|فاز|كسب|البطل|حسم|انتصر/i.test(
+    query,
+  );
+}
+
+function isOutcomeEvidence(text: string): boolean {
+  return /\b(winner|won|final score|total score|scoreline|champion|victor|defeated|beat)\b|الفائز|النتيجة|فاز|كسب|البطل|حسم|انتصر/i.test(
+    text,
+  );
+}
+
+export async function getChatTools(context: {
+  serverId?: string;
+  docSessionId?: string;
+  config?: any;
+}) {
+  const { serverId, docSessionId, config } = context;
+
+  const builtInTools: Record<string, any> = {
+    searchKnowledgeBase: tool({
+      description:
+        "Search the user's private RAG knowledge base when their question depends on indexed documents or media and the needed evidence is not already in the conversation. Do not use for general knowledge, casual conversation, writing tasks, or a follow-up that can be answered from an earlier search result. Search once with a focused query. For media, inspect the returned timeline and endingContext when relevant, and reuse exact mediaAssetId/startSecs/endSecs values with presentMedia when a preview is useful. Never guess timestamps.",
+      inputSchema: z.object({
+        query: z.string().describe('The search query for the knowledge base.'),
+      }),
+      execute: async ({ query }) => {
+        return queryKnowledgeBase(query, 5, serverId ?? null);
+      },
+    }),
+
+    presentMedia: tool({
+      description:
+        'Embed one indexed image, video segment, or audio segment as a compact citation in the chat. Use the exact mediaAssetId and timestamps from a current or earlier search result. Call this when the user asks to see, watch, play, or hear the media, or when one preview materially supports the answer. Prefer one best citation and do not call this for every search hit. Do not search again if the needed asset ID is already in the conversation.',
+      inputSchema: z.object({
+        assetId: z.string().describe('Exact mediaAssetId returned by searchKnowledgeBase.'),
+        startSecs: z
+          .number()
+          .nonnegative()
+          .optional()
+          .describe('Exact segment start time returned by searchKnowledgeBase.'),
+        endSecs: z
+          .number()
+          .nonnegative()
+          .optional()
+          .describe('Exact segment end time returned by searchKnowledgeBase.'),
+      }),
+      execute: async ({ assetId, startSecs, endSecs }) => {
+        const resolvePresentation = async () => {
+          const assets = await readMediaAssets();
+          const asset = assets.find((candidate) => candidate.id === assetId);
+          if (!asset || asset.processingStatus !== 'completed') {
+            return {
+              success: false,
+              error: 'That indexed media asset is no longer available.',
+            };
+          }
+
+          const range = normalizeMediaCitationRange(
+            asset.type,
+            asset.durationSecs,
+            startSecs,
+            endSecs,
+          );
+          const serverQuery = serverId ? `?serverId=${encodeURIComponent(serverId)}` : '';
+          const mediaUrl = `/api/media/${asset.id}${serverQuery}`;
+          const sourceUrl =
+            range.startSecs !== undefined
+              ? timestampMediaUrl(asset.originalUrl || mediaUrl, range.startSecs)
+              : asset.originalUrl || mediaUrl;
+
+          return {
+            success: true,
+            assetId: asset.id,
+            mediaType: asset.type,
+            fileName: asset.fileName,
+            mediaUrl,
+            sourceUrl,
+            startSecs: range.startSecs,
+            endSecs: range.endSecs,
+          };
+        };
+
+        return serverId ? runWithServer(serverId, resolvePresentation) : resolvePresentation();
+      },
+    }),
+
+    queryTabularData: tool({
+      description:
+        'Query a tabular dataset (CSV/Excel/JSON) for exact values, filtering, grouping, and aggregations. Use this when the user asks about specific data values, comparisons, averages, sums, counts, highest/lowest, or any numerical question about uploaded data.',
+      inputSchema: z.object({
+        datasetId: z
+          .string()
+          .describe(
+            'The ID of the dataset to query. Get this from the available datasets list in the system prompt.',
+          ),
+        filters: z
+          .array(
+            z.object({
+              column: z.string(),
+              op: z.enum(['eq', 'neq', 'gt', 'gte', 'lt', 'lte', 'contains', 'in']),
+              value: z.any(),
+            }),
+          )
+          .optional()
+          .describe('Filters to apply to the data.'),
+        groupBy: z.array(z.string()).optional().describe('Columns to group by for aggregations.'),
+        aggregations: z
+          .array(
+            z.object({
+              column: z.string(),
+              op: z.enum(['sum', 'avg', 'count', 'min', 'max', 'median']),
+            }),
+          )
+          .optional()
+          .describe('Aggregation operations to perform.'),
+        sortBy: z.string().optional().describe('Column to sort results by.'),
+        sortOrder: z.enum(['asc', 'desc']).optional().describe('Sort direction.'),
+        limit: z.number().optional().describe('Max number of rows to return.'),
+        columns: z.array(z.string()).optional().describe('Specific columns to include in results.'),
+      }),
+      execute: async (params) => {
+        try {
+          return await queryTabular(params);
+        } catch (err: any) {
+          return {
+            columns: [],
+            rows: [],
+            totalRows: 0,
+            error: err.message ?? 'Query failed',
+          };
+        }
+      },
+    }),
+
+    generateVisualization: tool({
+      description:
+        'Generate an interactive chart visualization. Use this when the user asks to see trends, comparisons, distributions, or any visual representation of data. The UI will render this as an interactive Recharts chart.',
+      inputSchema: z.object({
+        chartType: z
+          .enum(['bar', 'area', 'line', 'pie', 'scatter', 'radar'])
+          .describe('The type of chart to create.'),
+        title: z.string().describe('Chart title.'),
+        subtitle: z.string().optional().describe('Chart subtitle for additional context.'),
+        data: z
+          .array(z.record(z.string(), z.any()))
+          .describe(
+            "Array of data points to plot. You MUST provide the data here. Example: [{'name': 'North', 'value': 3206}, {'name': 'East', 'value': 2492}]",
+          ),
+        xAxisKey: z
+          .string()
+          .describe("The key in data objects to use for the X axis (usually 'name')."),
+        series: z
+          .array(
+            z.object({
+              dataKey: z.string().describe('The key in data objects for this series values.'),
+              label: z.string().optional().describe('Display label for this series in the legend.'),
+              color: z
+                .string()
+                .optional()
+                .describe('Color for this series (hex). If omitted, theme colors are used.'),
+            }),
+          )
+          .describe('Data series to plot.'),
+        stacked: z.boolean().optional().describe('Whether to stack bar/area charts.'),
+        showLegend: z.boolean().optional().default(true).describe('Whether to show the legend.'),
+        xAxisLabel: z.string().optional().describe('Label for the X axis.'),
+        yAxisLabel: z.string().optional().describe('Label for the Y axis.'),
+      }),
+      execute: async (config) => {
+        // Pass through — the UI renders this directly
+        return config;
+      },
+    }),
+
+    executeAnalysis: tool({
+      description:
+        'Execute Python code in a secure sandbox for deep data analysis. Use this for complex statistical computations (correlations, regressions, clustering), data transformations, or when creating custom matplotlib visualizations. The code has access to pandas, numpy, matplotlib, scipy, scikit-learn, and seaborn. Always print results to stdout and use plt.show() for charts.',
+      inputSchema: z.object({
+        code: z
+          .string()
+          .describe(
+            'Python code to execute. Has pandas (2.0+), numpy, matplotlib, scipy, sklearn, seaborn available. ALWAYS use "ME" instead of "M" for resample frequency. Print results and use plt.show() for charts.',
+          ),
+        datasetId: z
+          .string()
+          .optional()
+          .describe(
+            "If provided, the dataset CSV will be available as 'data.csv' in the working directory.",
+          ),
+      }),
+      execute: async ({ code, datasetId }) => {
+        try {
+          const sandboxManager = new SandboxManager({ backend: 'docker' });
+          const files: { name: string; content: string }[] = [];
+
+          if (datasetId) {
+            const dataset = await getTabularDataset(datasetId);
+            if (dataset && dataset.rows.length > 0) {
+              const cols = dataset.columns.map((c) => c.name);
+              const csvLines = [cols.join(',')];
+              for (const row of dataset.rows) {
+                csvLines.push(
+                  cols
+                    .map((c) => {
+                      const v = row[c];
+                      const s = String(v ?? '');
+                      return s.includes(',') || s.includes('"') ? `"${s.replace(/"/g, '""')}"` : s;
+                    })
+                    .join(','),
+                );
+              }
+              files.push({
+                name: 'data.csv',
+                content: csvLines.join('\n'),
+              });
+            }
+          }
+
+          let finalCode = code;
+          let result = await sandboxManager.execute({
+            code: finalCode,
+            language: 'python',
+            files,
+            timeout: 30_000,
+          });
+
+          // Auto-retry once if code crashes
+          if (result.exitCode !== 0) {
+            // Apply regex patches for common LLM mistakes
+            finalCode = finalCode.replace(/resample\((['"])M(['"])\)/g, 'resample($1ME$2)');
+            finalCode = finalCode.replace(/resample\((['"])Q(['"])\)/g, 'resample($1QE$2)');
+            finalCode = finalCode.replace(/resample\((['"])Y(['"])\)/g, 'resample($1YE$2)');
+            finalCode = finalCode.replace(/freq=(['"])M(['"])/g, 'freq=$1ME$2');
+
+            result = await sandboxManager.execute({
+              code: finalCode,
+              language: 'python',
+              files,
+              timeout: 30_000,
+            });
+          }
+
+          return {
+            stdout: result.stdout.slice(0, 5000),
+            stderr: result.stderr.slice(0, 2000),
+            exitCode: result.exitCode,
+            artifacts: result.artifacts.map((a) => ({
+              name: a.name,
+              mimeType: a.mimeType,
+              data: a.data.slice(0, 500_000), // cap at around 375KB base64
+            })),
+            executionTimeMs: result.executionTimeMs,
+          };
+        } catch (err: any) {
+          return {
+            stdout: '',
+            stderr: err.message ?? 'Sandbox execution failed',
+            exitCode: 1,
+            artifacts: [],
+            executionTimeMs: 0,
+          };
+        }
+      },
+    }),
+
+    analyzeImageDeeply: tool({
+      description:
+        'Use this tool when you need to answer a detailed question about an image (like a diagram, chart, or schema) found in the knowledge base. Pass the exact imageUrl found in the document metadata, along with specific instructions on what you need to extract from it.',
+      inputSchema: z.object({
+        imageUrl: z
+          .string()
+          .describe('The URL of the image to analyze (from metadata.images.imageUrl)'),
+        prompt: z
+          .string()
+          .describe(
+            'Specific instructions on what to extract or analyze from this image. Be as detailed as possible to get the best result.',
+          ),
+      }),
+      execute: async ({ imageUrl, prompt }) => {
+        try {
+          // Fetch the image to base64
+          // Handle both absolute URLs and relative local uploads
+          const fetchUrl = imageUrl.startsWith('/')
+            ? `http://127.0.0.1:${process.env.PORT || 3000}${imageUrl}`
+            : imageUrl;
+          const res = await fetch(fetchUrl);
+          if (!res.ok) throw new Error(`Failed to fetch image: ${res.statusText}`);
+          const buffer = await res.arrayBuffer();
+          const base64 = Buffer.from(buffer).toString('base64');
+          const mimeType = res.headers.get('content-type') || 'image/png';
+          const dataUrl = `data:${mimeType};base64,${base64}`;
+
+          // Post to our describe-image endpoint which handles model setup
+          const descRes = await fetch(
+            `http://127.0.0.1:${process.env.PORT || 3000}/api/describe-image`,
+            {
+              method: 'POST',
+              headers: { 'Content-Type': 'application/json' },
+              body: JSON.stringify({ base64, prompt }),
+            },
+          );
+
+          if (!descRes.ok) throw new Error(`Vision API error: ${descRes.statusText}`);
+          const data = await descRes.json();
+          return { analysis: data.description };
+        } catch (err: any) {
+          return { error: `Failed to analyze image: ${err.message}` };
+        }
+      },
+    }),
+
+    getIndexedData: tool({
+      description:
+        'Get structured access to ALL indexed source documents. Use this for counting, listing, filtering, and overview questions about the knowledge base. Returns document metadata (title, source type, status, custom metadata fields, dates). Use this when the user asks "how many documents?", "list all X", "show progress", "what sources?", or any structural question about the corpus.',
+      inputSchema: z.object({
+        filter: z
+          .object({
+            source: z
+              .enum(['paste', 'upload', 'scrape', 'tabular', 'media'])
+              .optional()
+              .describe('Filter by document source type.'),
+            status: z
+              .enum(['indexed', 'unindexed'])
+              .optional()
+              .describe('Filter by indexing status.'),
+            metadataKey: z.string().optional().describe('Filter by metadata key existence.'),
+            metadataValue: z
+              .string()
+              .optional()
+              .describe('Filter by metadata key=value (requires metadataKey).'),
+            titleContains: z.string().optional().describe('Case-insensitive text search in title.'),
+            createdAfter: z
+              .string()
+              .optional()
+              .describe('Only docs created on or after this ISO date.'),
+            createdBefore: z
+              .string()
+              .optional()
+              .describe('Only docs created on or before this ISO date.'),
+          })
+          .optional()
+          .describe('Optional filters to narrow down documents.'),
+        limit: z
+          .number()
+          .optional()
+          .default(200)
+          .describe('Max number of document summaries to return (default 200).'),
+        offset: z.number().optional().default(0).describe('Pagination offset.'),
+        includeContent: z
+          .boolean()
+          .optional()
+          .default(false)
+          .describe(
+            'Whether to include truncated document content. Default false for performance. Set true only when you need the actual text.',
+          ),
+      }),
+      execute: async ({ filter, limit, offset, includeContent }) => {
+        try {
+          const maxLimit = includeContent ? 20 : 100;
+          const actualLimit = Math.min(limit ?? (includeContent ? 20 : 100), maxLimit);
+
+          return await getCorpusDocuments(
+            filter as CorpusFilter | undefined,
+            actualLimit,
+            offset ?? 0,
+            includeContent ?? false,
+          );
+        } catch (err: any) {
+          return {
+            documents: [],
+            total: 0,
+            summary: {
+              totalDocuments: 0,
+              bySource: {},
+              byStatus: {},
+              topMetadataKeys: [],
+              totalCharacters: 0,
+              dateRange: null,
+            },
+            error: err.message ?? 'Failed to retrieve corpus data',
+          };
+        }
+      },
+    }),
+
+    analyzeCorpusWithCode: tool({
+      description:
+        'Run Python code in a sandbox with the FULL indexed corpus available as a file. Use this for complex analysis of documents: parsing fields from content, computing progress percentages, grouping by patterns, detecting duplicates, generating charts from corpus data, or any question that requires programmatic processing of hundreds/thousands of documents. The corpus is available as "corpus.csv" (or "corpus.jsonl" if format is jsonl). CSV columns: id, title, source, url, charCount, status, createdAt, content, plus any metadata_* columns.',
+      inputSchema: z.object({
+        code: z
+          .string()
+          .describe(
+            'Python code to execute. The corpus file is in the working directory. Use pandas to load it: df = pd.read_csv("corpus.csv") or for JSONL: df = pd.read_json("corpus.jsonl", lines=True). Has pandas (2.0+), numpy, matplotlib, scipy, sklearn, seaborn available. ALWAYS use "ME" instead of "M" for resample frequency. Print results and use plt.show() for charts.',
+          ),
+        format: z
+          .enum(['csv', 'jsonl'])
+          .optional()
+          .default('csv')
+          .describe('Format of the corpus file. Default is CSV.'),
+      }),
+      execute: async ({ code, format }) => {
+        try {
+          const sandboxManager = new SandboxManager({ backend: 'docker' });
+
+          // Export corpus in the requested format
+          const corpusData =
+            format === 'jsonl' ? await exportCorpusAsJSONL() : await exportCorpusAsCSV();
+
+          if (!corpusData) {
+            return {
+              stdout: 'No documents found in the corpus.',
+              stderr: '',
+              exitCode: 0,
+              artifacts: [],
+              executionTimeMs: 0,
+            };
+          }
+
+          const fileName = format === 'jsonl' ? 'corpus.jsonl' : 'corpus.csv';
+          const files = [{ name: fileName, content: corpusData }];
+
+          let finalCode = code;
+          let result = await sandboxManager.execute({
+            code: finalCode,
+            language: 'python',
+            files,
+            timeout: 30_000,
+          });
+
+          // Auto-retry once if code crashes
+          if (result.exitCode !== 0) {
+            // Apply regex patches for common LLM mistakes
+            finalCode = finalCode.replace(/resample\((['"])M(['"])\)/g, 'resample($1ME$2)');
+            finalCode = finalCode.replace(/resample\((['"])Q(['"])\)/g, 'resample($1QE$2)');
+            finalCode = finalCode.replace(/resample\((['"])Y(['"])\)/g, 'resample($1YE$2)');
+            finalCode = finalCode.replace(/freq=(['"])M(['"])/g, 'freq=$1ME$2');
+
+            result = await sandboxManager.execute({
+              code: finalCode,
+              language: 'python',
+              files,
+              timeout: 30_000,
+            });
+          }
+
+          return {
+            stdout: result.stdout.slice(0, 5000),
+            stderr: result.stderr.slice(0, 2000),
+            exitCode: result.exitCode,
+            artifacts: result.artifacts.map((a) => ({
+              name: a.name,
+              mimeType: a.mimeType,
+              data: a.data.slice(0, 500_000),
+            })),
+            executionTimeMs: result.executionTimeMs,
+          };
+        } catch (err: any) {
+          return {
+            stdout: '',
+            stderr: err.message ?? 'Corpus analysis failed',
+            exitCode: 1,
+            artifacts: [],
+            executionTimeMs: 0,
+          };
+        }
+      },
+    }),
+
+    fillDocumentForm: tool({
+      description:
+        'Fill form fields, text inputs, or checkboxes in the currently active document. Use this when the user asks to "fill the form", "add data to fields", or "fill with dummy data". You should use the searchKnowledgeBase tool to find answers to form questions if you do not know them. The available fields are provided in your system prompt.',
+      inputSchema: z.object({
+        edits: z
+          .array(
+            z.object({
+              fieldId: z
+                .string()
+                .describe('The ID of the field to fill (from the active document context)'),
+              value: z.string().describe('The value to set for the field'),
+            }),
+          )
+          .describe('List of fields to fill and their new values'),
+      }),
+      execute: async ({ edits }) => {
+        if (!docSessionId)
+          return {
+            success: false,
+            error:
+              'Document not loaded yet. Please wait a moment or ask the user to click the document to open it in the canvas before filling it.',
+          };
+
+        try {
+          const fieldEdits = edits.map((edit) => ({
+            ...edit,
+            type: 'fill' as const,
+          }));
+          const result = await applyFieldEdits(docSessionId, fieldEdits);
+          if (!result.success) {
+            return { success: false, error: result.error };
+          }
+
+          return {
+            success: true,
+            action: 'fill_document',
+            sessionId: docSessionId,
+            edits,
+            updatedFields: result.updatedFields,
+            fileName: result.parsed.fileName,
+            mimeType: result.parsed.mimeType,
+            fields: result.parsed.fields,
+            pages: result.parsed.pages.map((p) => ({
+              index: p.index,
+              text: p.text.slice(0, 2000),
+              html: p.html,
+              fields: p.fields,
+              dimensions: p.dimensions,
+            })),
+            totalPages: result.parsed.totalPages,
+            fileBase64: result.fileBase64,
+          };
+        } catch (err: any) {
+          return { success: false, error: err.message };
+        }
+      },
+      toModelOutput: documentEditModelOutput,
+    }),
+
+    editDocument: tool({
+      description:
+        'Edit the content (text) of the currently active document. Use this when the user asks to modify paragraphs, rewrite text, or change slides (not form fields).',
+      inputSchema: z.object({
+        edits: z
+          .array(
+            z.object({
+              pageIndex: z.number().default(0).describe('Page or slide index (0-based)'),
+              type: z.enum(['replace_text', 'insert_text']).describe('Type of edit'),
+              search: z.string().optional().describe('Text to search for (if replacing)'),
+              text: z.string().describe('Replacement text or text to insert'),
+            }),
+          )
+          .describe('List of content edits to apply'),
+      }),
+      execute: async ({ edits }) => {
+        if (!docSessionId)
+          return {
+            success: false,
+            error:
+              'Document not loaded yet. Please wait a moment or ask the user to click the document to open it in the canvas before editing it.',
+          };
+
+        try {
+          const result = await applyContentEdits(docSessionId, edits);
+          if (!result.success) {
+            return { success: false, error: result.error };
+          }
+
+          return {
+            success: true,
+            action: 'edit_document',
+            sessionId: docSessionId,
+            edits,
+            updatedFields: result.updatedFields,
+            fileName: result.parsed.fileName,
+            mimeType: result.parsed.mimeType,
+            pages: result.parsed.pages.map((p) => ({
+              index: p.index,
+              text: p.text.slice(0, 2000),
+              html: p.html,
+              fields: p.fields,
+              dimensions: p.dimensions,
+            })),
+            totalPages: result.parsed.totalPages,
+            fileBase64: result.fileBase64,
+          };
+        } catch (err: any) {
+          return { success: false, error: err.message };
+        }
+      },
+      toModelOutput: documentEditModelOutput,
+    }),
+
+    requestDocumentSignature: tool({
+      description:
+        'STRICTLY ONLY use this when the user explicitly asks to "sign" the currently active document or asks for a "signature". CRITICAL: Do NOT use this tool for filling forms, answering questions, or adding dummy data. CRITICAL: NEVER call this tool more than once per user request. Yields a UI widget for the user to confirm the placement and provide their signature.',
+      inputSchema: z.object({
+        detectedLocations: z
+          .array(
+            z.object({
+              pageIndex: z.number(),
+              context: z.string(),
+            }),
+          )
+          .optional()
+          .describe('Optional. The detected signature locations in the document.'),
+      }),
+    }),
+
+    webSearch: tool({
+      description:
+        'Search the public internet for current information, news, or facts that are not available in the local knowledge base.',
+      inputSchema: z.object({
+        query: z.string().describe('The search query.'),
+      }),
+      execute: async ({ query }) => {
+        const provider = config?.webSearchProvider || 'tavily';
+        const req = new Request('http://localhost/api/search', {
+          method: 'POST',
+          body: JSON.stringify({ query }),
+        });
+
+        let res: any;
+        try {
+          switch (provider) {
+            case 'google':
+            case 'serper':
+              const { POST: searchGoogle } = await import('../search/google/route');
+              res = await searchGoogle(req);
+              break;
+            case 'tavily':
+              const { POST: searchTavily } = await import('../search/tavily/route');
+              res = await searchTavily(req);
+              break;
+            case 'brave':
+              const { POST: searchBrave } = await import('../search/brave/route');
+              res = await searchBrave(req);
+              break;
+            case 'bing':
+              const { POST: searchBing } = await import('../search/bing/route');
+              res = await searchBing(req);
+              break;
+            case 'exa':
+              const { POST: searchExa } = await import('../search/exa/route');
+              res = await searchExa(req);
+              break;
+            case 'local':
+            default:
+              const { POST: searchLocal } = await import('../search/route');
+              res = await searchLocal(req);
+              break;
+          }
+
+          if (res.status >= 400) {
+            const err = await res.json();
+            return { error: err.error || 'Search failed', results: [] };
+          }
+
+          const data = await res.json();
+          const items = data.items || data.results || [];
+          return {
+            results: items.slice(0, 10).map((i: any) => ({
+              title: i.title,
+              url: i.url,
+              snippet: i.description || i.snippet || '',
+            })),
+          };
+        } catch (e: any) {
+          return { error: e.message || 'Search failed', results: [] };
+        }
+      },
+    }),
+  };
+
+  const enabledTools = config?.enabledTools || [];
+  const finalTools: Record<string, any> = {};
+
+  // If empty, default to all built-in tools (backwards compatibility)
+  const isEnabled = (id: string) => enabledTools.length === 0 || enabledTools.includes(id);
+
+  // 1. Add enabled built-in tools
+  for (const [id, toolDef] of Object.entries(builtInTools)) {
+    if (id === 'webSearch') continue; // Handled separately
+
+    if (
+      isEnabled(id) ||
+      ['presentMedia', 'fillDocumentForm', 'editDocument', 'requestDocumentSignature'].includes(id)
+    ) {
+      finalTools[id] = toolDef;
+    }
+  }
+
+  // Add webSearch tool if explicitly enabled globally
+  if (config?.webSearchEnabled === true) {
+    finalTools['webSearch'] = builtInTools['webSearch'];
+  }
+
+  // 2. Add enabled marketplace tools
+  // We fetch installed tools to see which ones are available
+  try {
+    const installed = await getInstalledTools();
+    for (const t of installed) {
+      if (isEnabled(t.id)) {
+        const mod = await loadTool(t.id);
+        if (mod && mod.default) {
+          finalTools[t.id] = mod.default(context);
+        } else if (mod && mod.tool) {
+          finalTools[t.id] = mod.tool(context);
+        }
+      }
+    }
+  } catch (err) {
+    console.error('[marketplace] Failed to load marketplace tools for chat:', err);
+  }
+
+  return finalTools;
+}