@promptev/context-engine 0.0.3 → 0.0.5

package/dist/index.cjs

@@ -5790,15 +5790,16 @@ function parseDocumentId(documentId) {
   }
   return raw;
 }
-function scopeSql(sourceIds, principals, params) {
+function scopeSql(sourceIds, principals, params, alias = "") {
+  const col = alias ? `${alias}.` : "";
   const where = [];
   if (sourceIds != null) {
     params.push(sourceIds);
-    where.push(`source_id = ANY($${params.length}::text[])`);
+    where.push(`${col}source_id = ANY($${params.length}::text[])`);
   }
   if (principals != null) {
     params.push(principals);
-    where.push(`(acl IS NULL OR acl && $${params.length}::text[])`);
+    where.push(`(${col}acl IS NULL OR ${col}acl && $${params.length}::text[])`);
   }
   return where.length ? `WHERE ${where.join(" AND ")}` : "";
 }
@@ -6057,6 +6058,19 @@ async function listDocuments(opts) {
   const principals = opts.principals ?? null;
   const params = [];
   let where = scopeSql(sourceIds, principals, params);
+  if (opts.documentIds != null) {
+    const parsedIds = [];
+    for (const did of opts.documentIds) {
+      try {
+        parsedIds.push(parseDocumentId(did));
+      } catch {
+        console.warn("listDocuments: skipping invalid document id %s", did);
+      }
+    }
+    params.push(parsedIds);
+    const clause = `id = ANY($${params.length}::uuid[])`;
+    where = where ? `${where} AND ${clause}` : `WHERE ${clause}`;
+  }
   const cursorTime = parsedCursor?.time ?? parsedCursor?.created_at;
   const cursorId = parsedCursor?.id;
   if (cursorTime && cursorId && UUID_RE2.test(String(cursorId))) {
@@ -6116,6 +6130,350 @@ async function listDocuments(opts) {
   }
   return result;
 }
+async function spreadsheetSchema(opts) {
+  const ids = [];
+  for (const did of opts.documentIds ?? []) {
+    try {
+      ids.push(parseDocumentId(did));
+    } catch {
+      console.warn("spreadsheetSchema: skipping invalid document id %s", did);
+    }
+  }
+  if (!ids.length) return [];
+  const principals = opts.principals ?? null;
+  const budget = opts.budget ?? MAX_SCHEMA_TEXT_CHARS;
+  const params = [];
+  let where = scopeSql(opts.sourceIds ?? null, principals, params);
+  const mimeClause = TABULAR_MIME_PATTERNS.map((pattern) => {
+    params.push(pattern);
+    return `mime_type ILIKE $${params.length}`;
+  }).join(" OR ");
+  where = where ? `${where} AND (${mimeClause})` : `WHERE (${mimeClause})`;
+  params.push(ids);
+  where += ` AND id = ANY($${params.length}::uuid[])`;
+  const sized = await opts.pool.query(
+    `SELECT id, name, source_id, mime_type, COALESCE(LENGTH(text), 0) AS text_len
+     FROM context_engine_documents
+     ${where}`,
+    params
+  );
+  const byId = /* @__PURE__ */ new Map();
+  for (const row of sized.rows) {
+    if (isTabularMime(row.mime_type)) byId.set(String(row.id), row);
+  }
+  const rows = ids.map((id) => byId.get(id)).filter((r) => r != null);
+  const chosen = [];
+  let spent = 0;
+  for (const row of rows) {
+    const len = Number(row.text_len ?? 0);
+    if (chosen.length && spent + len > budget) break;
+    chosen.push(String(row.id));
+    spent += len;
+  }
+  const texts = /* @__PURE__ */ new Map();
+  if (chosen.length) {
+    const bodyParams = [];
+    const bodyWhere = scopeSql(opts.sourceIds ?? null, principals, bodyParams);
+    bodyParams.push(chosen);
+    const clause = `id = ANY($${bodyParams.length}::uuid[])`;
+    const { rows: bodies } = await opts.pool.query(
+      `SELECT id, text FROM context_engine_documents
+       ${bodyWhere ? `${bodyWhere} AND ${clause}` : `WHERE ${clause}`}`,
+      bodyParams
+    );
+    for (const row of bodies) texts.set(String(row.id), row.text ?? null);
+  }
+  return rows.map((row) => {
+    const id = String(row.id);
+    if (!texts.has(id)) {
+      return {
+        document_id: id,
+        name: row.name,
+        source_id: row.source_id,
+        sheets: null,
+        schema_unavailable: "not read: this page of spreadsheets is past the text budget \u2014 narrow with sourceIds, or ask for a smaller limit"
+      };
+    }
+    return {
+      document_id: id,
+      name: row.name,
+      source_id: row.source_id,
+      sheets: redactValueRecursive(spreadsheetSchemaFromText(texts.get(id)), opts.redaction, {
+        principals,
+        secretKey: opts.secretKey ?? null,
+        hooks: opts.hooks
+      })
+    };
+  });
+}
+function remainder(structure) {
+  let total = 0;
+  for (const [key, value] of Object.entries(structure)) {
+    if (key.startsWith("more_") && typeof value === "number") total += value;
+  }
+  const sheets = structure.sheets;
+  if (Array.isArray(sheets)) {
+    for (const sheet of sheets) {
+      if (sheet && typeof sheet === "object" && typeof sheet.more_columns === "number") {
+        total += sheet.more_columns;
+      }
+    }
+  }
+  return total;
+}
+function boundedSheets(sheets) {
+  if (!sheets?.length) return sheets;
+  return sheets.slice(0, MAX_STRUCTURE_ITEMS).map((sheet) => {
+    const columns = sheet.columns ?? [];
+    const trimmed = {
+      ...sheet,
+      columns: columns.slice(0, MAX_STRUCTURE_ITEMS)
+    };
+    if (columns.length > MAX_STRUCTURE_ITEMS) {
+      trimmed.more_columns = columns.length - MAX_STRUCTURE_ITEMS;
+    }
+    return trimmed;
+  });
+}
+async function documentStructure(opts) {
+  const ids = [];
+  for (const did of opts.documentIds ?? []) {
+    try {
+      ids.push(parseDocumentId(did));
+    } catch {
+      console.warn("documentStructure: skipping invalid document id %s", did);
+    }
+  }
+  if (!ids.length) return {};
+  const principals = opts.principals ?? null;
+  const bounded = opts.bounded ?? true;
+  const described2 = await spreadsheetSchema({
+    pool: opts.pool,
+    documentIds: ids,
+    sourceIds: opts.sourceIds,
+    principals,
+    budget: opts.budget,
+    redaction: opts.redaction,
+    secretKey: opts.secretKey,
+    hooks: opts.hooks
+  });
+  const tabular = new Map(described2.map((entry) => [entry.document_id, entry]));
+  const params = [];
+  let where = scopeSql(opts.sourceIds ?? null, principals, params, "d");
+  params.push(ids);
+  const clause = `c.document_id = ANY($${params.length}::uuid[])`;
+  where = where ? `${where} AND ${clause}` : `WHERE ${clause}`;
+  const { rows } = await opts.pool.query(
+    `SELECT c.document_id AS document_id,
+            c.meta_data->>'section_title' AS section,
+            c.meta_data->>'top_level_key' AS key,
+            MIN(c.idx) AS first_idx,
+            COUNT(*) AS chunks,
+            -- Only the page-marker pass writes the page number, and it writes an int
+            -- \u2014 but a cast that meets anything else raises for the whole
+            -- query, so the type is checked in SQL rather than assumed.
+            MAX(CASE WHEN jsonb_typeof(c.meta_data->'page') = 'number'
+                     THEN (c.meta_data->>'page')::int END) AS page
+     FROM context_engine_chunks c
+     JOIN context_engine_documents d ON d.id = c.document_id
+     ${where}
+     GROUP BY c.document_id, section, key`,
+    params
+  );
+  const outline = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const id = String(row.document_id);
+    let found = outline.get(id);
+    if (!found) {
+      found = { chunks: 0, page: null, parts: [] };
+      outline.set(id, found);
+    }
+    found.chunks += Number(row.chunks ?? 0);
+    if (row.page != null) found.page = Math.max(found.page ?? 0, Number(row.page));
+    found.parts.push([Number(row.first_idx ?? 0), row.section ?? null, row.key ?? null]);
+  }
+  const ordered = (parts, pick, cap) => {
+    const seen = [];
+    for (const part of [...parts].sort((a, b) => a[0] - b[0])) {
+      const value = pick(part);
+      if (value && !seen.includes(value)) seen.push(value);
+    }
+    return [cap === null ? seen : seen.slice(0, cap), seen.length];
+  };
+  const out = {};
+  for (const id of ids) {
+    const found = outline.get(id);
+    const structure = { chunks: found ? found.chunks : 0 };
+    const sheetDoc = tabular.get(id);
+    if (sheetDoc) {
+      const sheets = sheetDoc.sheets ?? null;
+      structure.sheets = bounded ? boundedSheets(sheets) : sheets;
+      if (bounded && sheets && sheets.length > MAX_STRUCTURE_ITEMS) {
+        structure.more_sheets = sheets.length - MAX_STRUCTURE_ITEMS;
+      }
+      if (sheetDoc.schema_unavailable) structure.schema_unavailable = sheetDoc.schema_unavailable;
+      out[id] = structure;
+      continue;
+    }
+    if (found) {
+      if (found.page != null) {
+        structure.last_page = found.page;
+      }
+      const cap = bounded ? MAX_STRUCTURE_ITEMS : null;
+      const [sections, totalSections] = ordered(found.parts, (p) => p[1], cap);
+      if (sections.length) {
+        structure.sections = sections;
+        if (totalSections > sections.length) structure.more_sections = totalSections - sections.length;
+      }
+      const [keys, totalKeys] = ordered(found.parts, (p) => p[2], cap);
+      if (keys.length) {
+        structure.keys = keys;
+        if (totalKeys > keys.length) structure.more_keys = totalKeys - keys.length;
+      }
+    }
+    out[id] = structure;
+  }
+  for (const [id, structure] of Object.entries(out)) {
+    const left = remainder(structure);
+    if (!left) continue;
+    if (left <= MAX_INVITED_ITEMS) {
+      structure.next_action = { action: "discover", document_ids: [id] };
+    } else if ("sheets" in structure) {
+      structure.instead = "too many to list: a sheet this wide is one to COMPUTE over, not to read. Use action 'compute' and name the columns above, or search for a value to find the one row you want.";
+    } else {
+      structure.instead = "too many to list: read the document with get_chunks, or search it for the part you need.";
+    }
+  }
+  const redactOpts = { principals, secretKey: opts.secretKey ?? null, hooks: opts.hooks };
+  return Object.fromEntries(
+    Object.entries(out).map(([id, structure]) => [
+      id,
+      redactValueRecursive(structure, opts.redaction, redactOpts)
+    ])
+  );
+}
+async function documentTypes(opts) {
+  const principals = opts.principals ?? null;
+  const params = [];
+  let where = scopeSql(opts.sourceIds ?? null, principals, params);
+  if (opts.documentIds != null) {
+    const parsedIds = [];
+    for (const did of opts.documentIds) {
+      try {
+        parsedIds.push(parseDocumentId(did));
+      } catch {
+        console.warn("documentTypes: skipping invalid document id %s", did);
+      }
+    }
+    params.push(parsedIds);
+    const clause = `id = ANY($${params.length}::uuid[])`;
+    where = where ? `${where} AND ${clause}` : `WHERE ${clause}`;
+  }
+  params.push(MAX_CENSUS_ROWS);
+  const { rows } = await opts.pool.query(
+    `SELECT document_type AS doc_type,
+            mime_type,
+            COUNT(*) AS documents,
+            COUNT(*) FILTER (
+              WHERE structured_data IS NOT NULL AND structured_data::text <> '{}'
+            ) AS with_fields
+     FROM context_engine_documents
+     ${where}
+     GROUP BY document_type, mime_type
+     ORDER BY COUNT(*) DESC
+     LIMIT $${params.length}`,
+    params
+  );
+  const buckets = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const kind = isTabularMime(row.mime_type) ? "spreadsheet" : "text";
+    const type = row.doc_type ?? null;
+    const bucketKey = `${kind}\0${type ?? ""}`;
+    let bucket = buckets.get(bucketKey);
+    if (!bucket) {
+      bucket = { kind, type, documents: 0, with_fields: 0 };
+      buckets.set(bucketKey, bucket);
+    }
+    bucket.documents += Number(row.documents ?? 0);
+    bucket.with_fields += Number(row.with_fields ?? 0);
+  }
+  const census = [...buckets.values()].sort(
+    (a, b) => b.documents - a.documents || a.kind.localeCompare(b.kind) || (a.type ?? "").localeCompare(b.type ?? "")
+  ).slice(0, opts.limit ?? MAX_DOCUMENT_TYPES);
+  return redactValueRecursive(census, opts.redaction, {
+    principals,
+    secretKey: opts.secretKey ?? null,
+    hooks: opts.hooks
+  });
+}
+async function fieldSummary(opts) {
+  const principals = opts.principals ?? null;
+  const maxFieldsPerType = opts.maxFieldsPerType ?? MAX_FIELDS_PER_TYPE;
+  const params = [];
+  let where = scopeSql(opts.sourceIds ?? null, principals, params);
+  if (opts.documentIds != null) {
+    const parsedIds = [];
+    for (const did of opts.documentIds) {
+      try {
+        parsedIds.push(parseDocumentId(did));
+      } catch {
+        console.warn("fieldSummary: skipping invalid document id %s", did);
+      }
+    }
+    params.push(parsedIds);
+    const clause = `id = ANY($${params.length}::uuid[])`;
+    where = where ? `${where} AND ${clause}` : `WHERE ${clause}`;
+  }
+  const typed = "jsonb_typeof(structured_data) = 'object'";
+  where = where ? `${where} AND ${typed}` : `WHERE ${typed}`;
+  params.push(MAX_FIELD_ROWS);
+  const { rows } = await opts.pool.query(
+    `SELECT d.document_type AS doc_type,
+            d.mime_type AS mime_type,
+            kv.key AS key,
+            COUNT(*) AS documents,
+            MAX(kv.value::text) AS sample
+     FROM context_engine_documents d
+     JOIN LATERAL jsonb_each(d.structured_data) AS kv(key, value) ON TRUE
+     ${where}
+     GROUP BY d.document_type, d.mime_type, kv.key
+     ORDER BY d.document_type, COUNT(*) DESC, kv.key
+     LIMIT $${params.length}`,
+    params
+  );
+  const groups = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const kind = isTabularMime(row.mime_type) ? "spreadsheet" : "text";
+    const type = row.doc_type ?? null;
+    const groupKey = `${kind}\0${type ?? ""}`;
+    let group = groups.get(groupKey);
+    if (!group) {
+      group = { kind, type, fields: [] };
+      groups.set(groupKey, group);
+    }
+    let sample = null;
+    try {
+      sample = row.sample == null ? null : JSON.parse(String(row.sample));
+    } catch {
+      sample = row.sample;
+    }
+    if (group.fields.length < maxFieldsPerType) {
+      group.fields.push({
+        field: String(row.key),
+        type: inferDataType(String(row.key), sample),
+        documents: Number(row.documents)
+      });
+    } else {
+      group.more_fields ??= [];
+      group.more_fields.push(String(row.key));
+    }
+  }
+  return redactValueRecursive([...groups.values()], opts.redaction, {
+    principals,
+    secretKey: opts.secretKey ?? null,
+    hooks: opts.hooks
+  });
+}
 function candidateFieldTokens(question) {
   const candidates = [];
   const seen = /* @__PURE__ */ new Set();
@@ -6196,8 +6554,7 @@ async function requireDanfo() {
     throw new exports.ExtraMissingError("compute", "danfojs-node", "compute() dataframes");
   }
 }
-function parseSpreadsheetText(dfd, text) {
-  if (!text) return {};
+function splitSheets(text) {
   const sheets = {};
   let current = "sheet1";
   for (const line of text.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n")) {
@@ -6214,6 +6571,34 @@ function parseSpreadsheetText(dfd, text) {
     }
     bucket.push(line);
   }
+  return sheets;
+}
+function spreadsheetSchemaFromText(text) {
+  if (!text) return [];
+  const out = [];
+  for (const [name, lines] of Object.entries(splitSheets(text))) {
+    const body = lines.join("\n").trim();
+    if (!body) continue;
+    let rows;
+    try {
+      rows = parseCsv(body);
+    } catch (exc) {
+      console.warn("discover: sheet '%s' not parseable as CSV: %s", name, exc);
+      continue;
+    }
+    rows = rows.filter((row) => row.some((cell) => (cell ?? "").trim()));
+    if (!rows.length) continue;
+    out.push({
+      name,
+      columns: rows[0].map((cell) => String(cell).trim()),
+      row_count: rows.length - 1
+    });
+  }
+  return out;
+}
+function parseSpreadsheetText(dfd, text) {
+  if (!text) return {};
+  const sheets = splitSheets(text);
   const out = {};
   for (const [name, lines] of Object.entries(sheets)) {
     const body = lines.join("\n").trim();
@@ -6426,7 +6811,7 @@ ${schemaLines.join("\n")}`;
     hooks
   });
 }
-var MAX_LIST_LIMIT, MAX_COMPUTE_TEXT_CHARS, DEFAULT_COMPUTE_TIMEOUT, MAX_COMPUTE_DOCUMENTS, TABULAR_MIME_PATTERNS, UUID_RE2, WORD_RE, STOPWORDS, SHEET_MARKER_RE2, CODE_FENCE_RE, COMPUTE_SYSTEM_PROMPT, visible, MAX_CHUNKS_PER_READ, MAX_MAP_REDUCE_DOCS, DEFAULT_MAP_REDUCE_DOCS, MAX_MAP_REDUCE_CONCURRENCY;
+var MAX_LIST_LIMIT, MAX_COMPUTE_TEXT_CHARS, DEFAULT_COMPUTE_TIMEOUT, MAX_COMPUTE_DOCUMENTS, MAX_SCHEMA_TEXT_CHARS, MAX_FIELDS_PER_TYPE, MAX_FIELD_ROWS, MAX_STRUCTURE_ITEMS, MAX_INVITED_ITEMS, MAX_DOCUMENT_TYPES, MAX_CENSUS_ROWS, TABULAR_MIME_PATTERNS, UUID_RE2, WORD_RE, STOPWORDS, SHEET_MARKER_RE2, CODE_FENCE_RE, COMPUTE_SYSTEM_PROMPT, visible, MAX_CHUNKS_PER_READ, MAX_MAP_REDUCE_DOCS, DEFAULT_MAP_REDUCE_DOCS, MAX_MAP_REDUCE_CONCURRENCY;
 var init_actions = __esm({
   "src/actions.ts"() {
     init_chunkers();
@@ -6442,6 +6827,13 @@ var init_actions = __esm({
     MAX_COMPUTE_TEXT_CHARS = 2e6;
     DEFAULT_COMPUTE_TIMEOUT = 30;
     MAX_COMPUTE_DOCUMENTS = 50;
+    MAX_SCHEMA_TEXT_CHARS = MAX_COMPUTE_TEXT_CHARS;
+    MAX_FIELDS_PER_TYPE = 40;
+    MAX_FIELD_ROWS = 2e3;
+    MAX_STRUCTURE_ITEMS = 40;
+    MAX_INVITED_ITEMS = 400;
+    MAX_DOCUMENT_TYPES = 30;
+    MAX_CENSUS_ROWS = 500;
     TABULAR_MIME_PATTERNS = ["%csv%", "%sheet%", "%excel%", "%spreadsheetml%", "%tab-separated%"];
     UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
     WORD_RE = /[^\W\d_]+/gu;
@@ -8275,11 +8667,16 @@ function parseCursor2(cursor) {
   }
   return parsed;
 }
-async function listPage(engine, sourceIds, principals, action, limit, cursor, ceiling, redaction) {
+async function listPage(engine, sourceIds, principals, action, limit, cursor, documentIds, ceiling, redaction) {
   const page = await engine.listDocuments({
     // `!= null`, NOT truthiness: an EMPTY array means "nothing is in scope"
     // and collapsing it to null would list the whole corpus.
     sourceIds: sourceIds != null ? [...new Set(sourceIds)] : null,
+    // Already intersected with the host's ceiling by `narrowToCeiling`, and
+    // filtered in SQL rather than after the page is built — three named
+    // documents sitting on page four must come back as themselves, not as an
+    // empty page.
+    documentIds: documentIds != null ? [...new Set(documentIds)] : null,
     principals,
     cursor,
     limit: Math.max(1, Math.min(limit, MAX_LIST_LIMIT)),
@@ -8289,7 +8686,9 @@ async function listPage(engine, sourceIds, principals, action, limit, cursor, ce
     id: String(raw.id),
     name: raw.name,
     source_id: raw.sourceId ?? raw.source_id,
-    kind: documentKind(raw)
+    kind: documentKind(raw),
+    document_type: raw.documentType ?? raw.document_type ?? null,
+    mode: raw.mode ?? null
   }));
   if (ceiling.documentIds != null) {
     const allowed = new Set(ceiling.documentIds);
@@ -8342,30 +8741,62 @@ async function callKnowledgeTool(engine, args) {
       action,
       args.limit ?? 50,
       parseCursor2(args.cursor),
+      documentIds,
       ceiling,
       args.redaction
     );
     if (action === "list") return { success: true, ...page };
-    const spreadsheets = page.documents.filter((d) => d.kind === "spreadsheet").map((d) => d.name);
+    const structures = await engine.documentStructure({
+      documentIds: page.documents.map((d) => d.id),
+      sourceIds,
+      principals,
+      // The cap is for the call that did NOT name its documents. A caller
+      // that asked about specific documents asked for all of them, and the
+      // truncated payload tells it to make exactly this call — so answering
+      // it truncated again would be a loop.
+      bounded: !requestedDocumentIds?.length,
+      redaction: args.redaction
+    });
+    for (const doc of page.documents) doc.structure = structures[doc.id] ?? {};
+    const hasSpreadsheet = page.documents.some((d) => d.kind === "spreadsheet");
+    const fieldsByType = await engine.fieldSummary({
+      sourceIds,
+      documentIds,
+      principals,
+      redaction: args.redaction
+    });
+    const census = await engine.documentTypes({
+      sourceIds,
+      documentIds,
+      principals,
+      redaction: args.redaction
+    });
     const forAFact = { action: "search", query: "<bare identifier or key words>" };
     const nextAction = {};
-    if (spreadsheets.length && available.compute) {
+    if (hasSpreadsheet && available.compute) {
       nextAction["for a figure from a spreadsheet"] = {
         action: "compute",
-        query: "<what to compute, columns as named>"
+        query: "<what to compute, columns as named above>"
       };
     }
     nextAction["for a clause or a fact"] = forAFact;
+    if (fieldsByType.length && available.query_meta) {
+      nextAction["for documents by a field value"] = {
+        action: "query_meta",
+        query: "<a question naming a field from fields_by_type>"
+      };
+    }
     if (available.get_neighbors) {
       nextAction["for how things connect"] = {
         action: "get_neighbors",
         entity: "<a name that appears in the documents>"
       };
     }
-    return {
+    const discovered = {
       success: true,
       ...page,
-      spreadsheets,
+      document_types: census,
+      fields_by_type: fieldsByType,
       available_actions: Object.fromEntries(
         Object.keys(ACTION_HELP).map((name) => [
           name,
@@ -8374,6 +8805,7 @@ async function callKnowledgeTool(engine, args) {
       ),
       next_action: nextAction
     };
+    return discovered;
   }
   if (action === "search") {
     if (!text) return { success: false, error: "search needs a query" };
@@ -8603,7 +9035,7 @@ var init_knowledge_tool = __esm({
       "get_neighbors",
       "community_summary"
     ];
-    exports.KNOWLEDGE_TOOL_DESCRIPTION = "The knowledge base \u2014 the ingested documents and data files (PDF, Word, Excel, CSV and the rest) \u2014 as ONE tool with actions. Nothing is searched for you: use it before answering anything that should come from those documents, and do not use it for general knowledge. Call action 'discover' FIRST when you do not already know what is there: it lists the documents, says which are spreadsheets, and says which actions this deployment can run. Then match the action to the task. 'search' finds passages by meaning or keywords \u2014 for questions answered by reading text. Looking up an identifier (an ID, code, SKU or invoice number) is the exception: search the BARE identifier alone, e.g. '2525', never the whole question. Those are CONTENT identifiers \u2014 written inside a document \u2014 and they belong in a query; a document_id is a system id (a uuid) that only discover, list or a search hit can give you, so never search a uuid as text and never hand an invoice number to get_doc. 'get_doc' reads one whole document by id, 'get_docs' reads several at once, and 'get_chunks' walks one long document in order a piece at a time when you need more of it than an excerpt; 'list' browses the documents without searching. 'map_reduce' asks the SAME question of every document in scope and answers once per document \u2014 for 'which contracts mention X', where search would return a handful of passages and miss the rest. 'query_meta' filters and aggregates documents by their structured fields (dates, amounts, categories) \u2014 usually the right action for a question about spreadsheet data. 'compute' runs code over the spreadsheets for any figure DERIVED from them \u2014 a total, average, count, ranking, margin or comparison across rows \u2014 AND for finding the exact row matching one id or value. In a large table, search cannot reliably locate an individual row; compute can. When the question is about how things are CONNECTED rather than what a document says \u2014 who works with whom, what belongs to what, what a change touches \u2014 use the graph actions: 'get_neighbors' for what is one step from one thing, 'traverse' for everything within a few steps of it, 'find_related' for connections of a kind across the corpus, and 'community_summary' for the themes the corpus groups into. They are available only where a graph was built; discover says so. Searching with mode='graph' ranks passages by those same connections instead of by wording alone, which finds a passage that never repeats your words. Rules that decide answers: search returns EXCERPTS, and rows of a spreadsheet are not arithmetic \u2014 never add up, average or rank rows yourself from what search returned, and never say a figure is not available before computing over the sheet that holds it. A listing that says has_more has MORE: send its next_cursor back as 'cursor' for the next page, and never conclude a document is absent from a first page that was truncated. Search again with different words before saying a document is missing. A result may carry a next_action (or next_page): it is the call to make next, already filled in \u2014 follow it rather than guessing the next step. Cite document names.";
+    exports.KNOWLEDGE_TOOL_DESCRIPTION = "The knowledge base \u2014 the ingested documents and data files (PDF, Word, Excel, CSV and the rest) \u2014 as ONE tool with actions. Nothing is searched for you: use it before answering anything that should come from those documents, and do not use it for general knowledge. Call action 'discover' FIRST when you do not already know what is there: it lists the documents and says what is INSIDE each one \u2014 a spreadsheet's sheet names, column headers and row counts; a document's section titles and last page; a JSON file's top-level keys \u2014 plus the extracted field names grouped by document type with each field's data type and how many documents carry it, a census of the whole corpus, and which actions this deployment can run. Use those names verbatim: they are what compute, query_meta and get_chunks match on, so one discover is enough and you never have to go looking for a column, a section or a field name. A long list is shortened there, with the rest reported as a count and, when it is worth the trip, the exact call that returns it in full \u2014 discover again with document_ids set to that one document, which answers whole. When it is not worth the trip the payload says so and says what to do instead: a sheet with hundreds of columns is one to compute over, never one to read back. Then match the action to the task. 'search' finds passages by meaning or keywords \u2014 for questions answered by reading text. Looking up an identifier (an ID, code, SKU or invoice number) is the exception: search the BARE identifier alone, e.g. '2525', never the whole question. Those are CONTENT identifiers \u2014 written inside a document \u2014 and they belong in a query; a document_id is a system id (a uuid) that only discover, list or a search hit can give you, so never search a uuid as text and never hand an invoice number to get_doc. 'get_doc' reads one whole document by id, 'get_docs' reads several at once, and 'get_chunks' walks one long document in order a piece at a time when you need more of it than an excerpt; 'list' browses the documents without searching. 'map_reduce' asks the SAME question of every document in scope and answers once per document \u2014 for 'which contracts mention X', where search would return a handful of passages and miss the rest. 'query_meta' filters and aggregates documents by their structured fields (dates, amounts, categories) \u2014 usually the right action for a question about spreadsheet data. 'compute' runs code over the spreadsheets for any figure DERIVED from them \u2014 a total, average, count, ranking, margin or comparison across rows \u2014 AND for finding the exact row matching one id or value. In a large table, search cannot reliably locate an individual row; compute can. When the question is about how things are CONNECTED rather than what a document says \u2014 who works with whom, what belongs to what, what a change touches \u2014 use the graph actions: 'get_neighbors' for what is one step from one thing, 'traverse' for everything within a few steps of it, 'find_related' for connections of a kind across the corpus, and 'community_summary' for the themes the corpus groups into. They are available only where a graph was built; discover says so. Searching with mode='graph' ranks passages by those same connections instead of by wording alone, which finds a passage that never repeats your words. Rules that decide answers: search returns EXCERPTS, and rows of a spreadsheet are not arithmetic \u2014 never add up, average or rank rows yourself from what search returned, and never say a figure is not available before computing over the sheet that holds it. A listing that says has_more has MORE: send its next_cursor back as 'cursor' for the next page, and never conclude a document is absent from a first page that was truncated. Search again with different words before saying a document is missing. A result may carry a next_action (or next_page): it is the call to make next, already filled in \u2014 follow it rather than guessing the next step. Cite document names.";
     ACTION_PARAM_DESCRIPTION = "Match it to the task: reading questions -> search (a bare identifier for an ID or code); spreadsheet analysis, or the exact row for one id or value -> query_meta or compute; how things connect -> get_neighbors, traverse, find_related or community_summary; browse everything -> list; unsure what exists -> discover first.";
     ACTION_HELP = {
       search: "passages by meaning or keywords; an ID, code or number as the BARE identifier",
@@ -8626,7 +9058,7 @@ var init_knowledge_tool = __esm({
       graph: "the graph actions are off: this deployment has no graph configured, so nothing has been linked up. Use search, get_doc or list instead."
     };
     GRAPH_ACTIONS = ["traverse", "find_related", "get_neighbors", "community_summary"];
-    DOCUMENT_ID_ACTIONS = ["search", "compute", "get_docs", "map_reduce"];
+    DOCUMENT_ID_ACTIONS = ["search", "compute", "get_docs", "map_reduce", "discover", "list"];
     DEFAULT_GET_DOCS_CHARS = 2e5;
     INPUT_PROPERTIES = {
       action: { type: "string", enum: [...exports.KNOWLEDGE_ACTIONS], description: ACTION_PARAM_DESCRIPTION },
@@ -8646,7 +9078,7 @@ var init_knowledge_tool = __esm({
       document_ids: {
         type: "array",
         items: { type: "string" },
-        description: "Narrows to these documents, using ids from discover, list or a search hit. It INTERSECTS with source_ids, so a document outside the sources you named returns nothing. (for search, compute)"
+        description: "Narrows to these documents, using ids from discover, list or a search hit. It INTERSECTS with source_ids, so a document outside the sources you named returns nothing. On discover it also asks for those documents' structure IN FULL, past the shortening a whole-page discover applies. (for search, compute, get_docs, map_reduce, discover, list)"
       },
       entity: {
         type: "string",
@@ -11401,7 +11833,7 @@ var SCHEMAS = {
       url: { type: "string", description: "Endpoint URL, may contain {path} params" },
       method: {
         type: "string",
-        enum: ["GET", "POST", "PUT", "PATCH", "DELETE"],
+        enum: ["GET", "POST", "PUT", "PATCH", "DELETE", "QUERY"],
         default: "GET"
       },
       headers: {
@@ -11437,6 +11869,15 @@ var SCHEMAS = {
       llmQueryParameters: {
         type: "object",
         description: "LLM-filled parameters sent as the query string"
+      },
+      // How the response comes back to the caller, whatever the method. `tsv`
+      // turns every array of objects in it into a TSV string (`formatResult`
+      // in tools/response-mode.ts); a caller's explicit `responseMode` wins.
+      response_mode: {
+        type: "string",
+        enum: ["json", "tsv"],
+        default: "json",
+        description: "Return the response as JSON, or its arrays of objects as TSV"
       }
     },
     required: ["url", "method"]
@@ -12602,6 +13043,89 @@ function findTool(tools, callName) {
   return tools.find((t) => t.callName === callName);
 }
 
+// src/tools/response-mode.ts
+var RESPONSE_MODES = ["json", "tsv"];
+function validateResponseMode(value, name = "responseMode") {
+  if (!RESPONSE_MODES.includes(value)) {
+    throw new Error(`${name} must be one of json, tsv, got ${JSON.stringify(value)}`);
+  }
+  return value;
+}
+function tsvCell(value) {
+  if (value === null || value === void 0) return "\\N";
+  let text;
+  if (typeof value === "string") text = value;
+  else if (value instanceof Date) text = value.toISOString();
+  else if (typeof value === "bigint") text = value.toString();
+  else {
+    try {
+      text = JSON.stringify(value) ?? String(value);
+    } catch {
+      text = String(value);
+    }
+  }
+  return text.replaceAll("\\", "\\\\").replaceAll("	", "\\t").replaceAll("\n", "\\n").replaceAll("\r", "\\r");
+}
+function isPlainObject2(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+function isTable(value) {
+  return Array.isArray(value) && value.length > 0 && value.every(isPlainObject2);
+}
+function tsvLines(rows) {
+  const columns = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const row of rows) {
+    for (const key of Object.keys(row)) {
+      if (!seen.has(key)) {
+        seen.add(key);
+        columns.push(key);
+      }
+    }
+  }
+  return [
+    columns.map(tsvCell).join("	"),
+    ...rows.map((row) => columns.map((c) => tsvCell(row[c])).join("	"))
+  ];
+}
+function rowsToTsv(rows) {
+  return rows.length ? tsvLines(rows).join("\n") : "";
+}
+function collectTables(value, path, out) {
+  if (isTable(value)) {
+    out.set(JSON.stringify(path), { path, lines: tsvLines(value) });
+  } else if (isPlainObject2(value)) {
+    for (const [key, child] of Object.entries(value)) collectTables(child, [...path, key], out);
+  } else if (Array.isArray(value)) {
+    value.forEach((child, index) => {
+      collectTables(child, [...path, index], out);
+    });
+  }
+}
+function buildTsv(value, path, tables, kept) {
+  const id = JSON.stringify(path);
+  const table = tables.get(id);
+  if (table) return table.lines.slice(0, (kept.get(id) ?? 0) + 1).join("\n");
+  if (isPlainObject2(value)) {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) out[k] = buildTsv(v, [...path, k], tables, kept);
+    return out;
+  }
+  if (Array.isArray(value)) return value.map((v, i) => buildTsv(v, [...path, i], tables, kept));
+  return value;
+}
+function formatResult(result, responseMode = "json") {
+  if (validateResponseMode(responseMode) === "json") return result;
+  const root = { result };
+  const tables = /* @__PURE__ */ new Map();
+  collectTables(root, [], tables);
+  if (!tables.size) return result;
+  const kept = new Map([...tables].map(([id, t]) => [id, t.lines.length - 1]));
+  return buildTsv(root, [], tables, kept).result;
+}
+
 // src/tools/governance.ts
 var RESULT_MAX_CHARS = 8e3;
 var RESULT_MAX_ROWS = 100;
@@ -12633,12 +13157,90 @@ function stripUnderscoreArgs(args) {
   }
   return out;
 }
+function validateResultBudget(name, value, fallback) {
+  if (value === void 0) return fallback;
+  if (value === null) return null;
+  if (typeof value !== "number" || !Number.isInteger(value)) {
+    throw new TypeError(`${name} must be a positive integer or null, got ${typeof value}`);
+  }
+  if (value < 1) throw new Error(`${name} must be a positive integer or null, got ${value}`);
+  return value;
+}
+function serialize(value) {
+  try {
+    return JSON.stringify(value) ?? JSON.stringify(String(value));
+  } catch {
+    return JSON.stringify(String(value));
+  }
+}
+function clip(serialized, maxChars) {
+  return {
+    _truncated: serialized.slice(0, maxChars),
+    _original_size: serialized.length,
+    _note: "tool result exceeded the context budget and was truncated"
+  };
+}
+function shapeTsv(result, maxChars, maxRows) {
+  const wrapped = !isPlainObject2(result);
+  const root = wrapped ? { result } : result;
+  const tables = /* @__PURE__ */ new Map();
+  collectTables(root, [], tables);
+  if (!tables.size) return null;
+  const sizes = new Map([...tables].map(([id, t]) => [id, t.lines.length - 1]));
+  const kept = new Map([...sizes].map(([id, n]) => [id, maxRows === null ? n : Math.min(n, maxRows)]));
+  const render = () => {
+    const built = buildTsv(root, [], tables, kept);
+    const notes = {};
+    for (const [id, t] of tables) {
+      const k = kept.get(id);
+      const n = sizes.get(id);
+      if (k < n) notes[t.path.join(".")] = { rows_returned: k, rows_omitted: n - k };
+    }
+    if (Object.keys(notes).length) {
+      built._result_shaping = notes;
+      return built;
+    }
+    return wrapped ? built.result : built;
+  };
+  const fits = () => maxChars === null || serialize(render()).length <= maxChars;
+  const capped2 = [...tables.keys()].some((id) => kept.get(id) < sizes.get(id));
+  if (fits()) return [render(), capped2];
+  const initial = new Map(kept);
+  const tableSize = (id) => tables.get(id).lines.slice(0, kept.get(id) + 1).join("\n").length;
+  const bySize = [...tables.keys()].sort((a, b) => tableSize(b) - tableSize(a));
+  for (const id of bySize) {
+    let lo = 0;
+    let hi = kept.get(id) - 1;
+    let best = null;
+    while (lo <= hi) {
+      const mid = Math.floor((lo + hi) / 2);
+      kept.set(id, mid);
+      if (fits()) {
+        best = mid;
+        lo = mid + 1;
+      } else {
+        hi = mid - 1;
+      }
+    }
+    if (best !== null) {
+      kept.set(id, best);
+      return [render(), true];
+    }
+    kept.set(id, 0);
+  }
+  for (const [id, n] of initial) kept.set(id, n);
+  return [clip(serialize(render()), maxChars), true];
+}
 function shapeResult(result, opts = {}) {
-  const maxChars = opts.maxChars ?? RESULT_MAX_CHARS;
-  const maxRows = opts.maxRows ?? RESULT_MAX_ROWS;
+  const maxChars = opts.maxChars === void 0 ? RESULT_MAX_CHARS : opts.maxChars;
+  const maxRows = opts.maxRows === void 0 ? RESULT_MAX_ROWS : opts.maxRows;
+  if (opts.responseMode === "tsv") {
+    const tsv = shapeTsv(result, maxChars, maxRows);
+    if (tsv) return tsv;
+  }
   let truncated = false;
   let shaped = result;
-  if (result && typeof result === "object" && Array.isArray(result.rows) && result.rows.length > maxRows) {
+  if (maxRows !== null && result && typeof result === "object" && Array.isArray(result.rows) && result.rows.length > maxRows) {
     const rows = result.rows;
     const kept = rows.slice(0, maxRows);
     shaped = {
@@ -12648,22 +13250,9 @@ function shapeResult(result, opts = {}) {
     };
     truncated = true;
   }
-  let serialized;
-  try {
-    serialized = JSON.stringify(shaped);
-  } catch {
-    serialized = JSON.stringify(String(shaped));
-  }
-  if (serialized.length > maxChars) {
-    return [
-      {
-        _truncated: serialized.slice(0, maxChars),
-        _original_size: serialized.length,
-        _note: "tool result exceeded the context budget and was truncated"
-      },
-      true
-    ];
-  }
+  if (maxChars === null) return [shaped, truncated];
+  const serialized = serialize(shaped);
+  if (serialized.length > maxChars) return [clip(serialized, maxChars), true];
   return [shaped, truncated];
 }
 function redactToolResult(result, policy, opts) {
@@ -12774,9 +13363,29 @@ function canonicalToPublic(ct) {
     params_schema: ct.paramsSchema
   };
 }
+function configuredResponseMode(ct, config) {
+  if (ct.kind !== "http" || config.response_mode == null) return "json";
+  try {
+    return validateResponseMode(config.response_mode, "config.response_mode");
+  } catch {
+    console.warn(
+      `tool ${ct.callName} has an unknown config response_mode ${JSON.stringify(config.response_mode)}; returning json`
+    );
+    return "json";
+  }
+}
+function checkConfigResponseMode(kind, config) {
+  if (kind !== "http" || !config || config.response_mode == null) return;
+  try {
+    validateResponseMode(config.response_mode, "config.response_mode");
+  } catch (exc) {
+    throw new ConfigTemplateError(exc.message);
+  }
+}
 async function registerTool(engine, tc) {
   canonicalFromConfig(tc);
   const config = tc.config ?? {};
+  checkConfigResponseMode(tc.kind, config);
   if (containsSentinel(config)) {
     throw new ConfigTemplateError(
       `config contains the ${JSON.stringify(REDACTED_SENTINEL)} placeholder \u2014 a redacted template cannot be registered as a new tool; re-enter the secret values`
@@ -12812,6 +13421,12 @@ async function updateTool(engine, id, opts) {
   if (!row || !aclVisible(row.acl != null ? [...row.acl] : null, principals)) {
     throw new exports.EngineActionError(`tool not found: ${id}`);
   }
+  if ("config" in fields) {
+    checkConfigResponseMode(
+      "kind" in fields ? fields.kind : row.kind,
+      fields.config
+    );
+  }
   const sets = [];
   const params = [];
   let i = 1;
@@ -13121,6 +13736,9 @@ function warnUnscopedApproval(callName) {
 async function executeTool(engine, callName, args, opts = {}) {
   const runtimeArgs = args ?? {};
   const approvalScope = validateApprovalScope(opts.approvalScope);
+  const resultMaxChars = validateResultBudget("resultMaxChars", opts.resultMaxChars, RESULT_MAX_CHARS);
+  const resultMaxRows = validateResultBudget("resultMaxRows", opts.resultMaxRows, RESULT_MAX_ROWS);
+  let responseMode = opts.responseMode === void 0 || opts.responseMode === null ? null : validateResponseMode(opts.responseMode);
   const ct = findTool(await mergedTools(engine, opts.sourceId ?? null), callName);
   if (!ct) throw new exports.EngineActionError(`tool not found: ${callName}`);
   if (!toolVisible(ct, opts.principals ?? null)) {
@@ -13163,6 +13781,7 @@ async function executeTool(engine, callName, args, opts = {}) {
   if (ct.kind !== "function" && ct.id != null) {
     config = await decryptCtConfig(engine, ct.id);
   }
+  responseMode ??= configuredResponseMode(ct, config);
   const actorType = opts.actor?.type ?? null;
   const actorId = opts.actor?.id ?? null;
   let rawResult = null;
@@ -13186,7 +13805,16 @@ async function executeTool(engine, callName, args, opts = {}) {
         hooks: engine.hooks
       });
       rawResult = redacted;
-      [shaped, truncated] = shapeResult(rawResult);
+      let toShape = rawResult;
+      if (responseMode === "tsv" && ct.kind === "db" && isPlainObject2(rawResult)) {
+        const { text: _preview, ...rest } = rawResult;
+        toShape = rest;
+      }
+      [shaped, truncated] = shapeResult(toShape, {
+        maxChars: resultMaxChars,
+        maxRows: resultMaxRows,
+        responseMode
+      });
     } catch (e) {
       exc = e;
       success = false;
@@ -13679,6 +14307,7 @@ var ContextEngine = class _ContextEngine {
       pool,
       sourceId: opts.sourceId,
       sourceIds: opts.sourceIds,
+      documentIds: opts.documentIds,
       principals: resolvePrincipals(opts.principals, "listDocuments"),
       cursor: opts.cursor,
       limit: opts.limit,
@@ -13687,6 +14316,84 @@ var ContextEngine = class _ContextEngine {
       hooks: this.hooks
     });
   }
+  /**
+   * Sheet names, columns and row counts for the named spreadsheets.
+   *
+   * See `actions.spreadsheetSchema`. This is the half of `discover` that lets
+   * a model write ONE `compute` call: sheet names here are the keys it will
+   * index `dfs` by, and columns are the names it will use inside the code it
+   * writes.
+   */
+  async spreadsheetSchema(opts) {
+    const pool = await this.ensurePool();
+    return spreadsheetSchema({
+      pool,
+      documentIds: opts.documentIds,
+      sourceIds: opts.sourceIds,
+      principals: resolvePrincipals(opts.principals, "spreadsheetSchema"),
+      redaction: opts.redaction ?? this.config.redaction,
+      secretKey: this.config.secretKey,
+      hooks: this.hooks
+    });
+  }
+  /**
+   * What is inside each of the named documents, whatever its type.
+   *
+   * See `actions.documentStructure`. Sheets and columns for a workbook,
+   * sections and the last page for a document with headings, top-level keys
+   * for JSON, and a chunk count for everything — so `discover` describes the
+   * whole corpus rather than only the spreadsheets in it.
+   */
+  async documentStructure(opts) {
+    const pool = await this.ensurePool();
+    return documentStructure({
+      pool,
+      documentIds: opts.documentIds,
+      sourceIds: opts.sourceIds,
+      bounded: opts.bounded,
+      principals: resolvePrincipals(opts.principals, "documentStructure"),
+      redaction: opts.redaction ?? this.config.redaction,
+      secretKey: this.config.secretKey,
+      hooks: this.hooks
+    });
+  }
+  /**
+   * The corpus census — `[{kind, type, documents, with_fields}]`.
+   *
+   * See `actions.documentTypes`. `kind` comes from the mime and is always
+   * known; `type` is the LLM-written document type and exists only where
+   * structured extraction was opted into.
+   */
+  async documentTypes(opts = {}) {
+    const pool = await this.ensurePool();
+    return documentTypes({
+      pool,
+      sourceIds: opts.sourceIds,
+      documentIds: opts.documentIds,
+      principals: resolvePrincipals(opts.principals, "documentTypes"),
+      redaction: opts.redaction ?? this.config.redaction,
+      secretKey: this.config.secretKey,
+      hooks: this.hooks
+    });
+  }
+  /**
+   * Extracted structured field names grouped by document kind and type.
+   *
+   * See `actions.fieldSummary`. One row per group, keyed by the same
+   * `(kind, type)` pair `documentTypes` uses.
+   */
+  async fieldSummary(opts = {}) {
+    const pool = await this.ensurePool();
+    return fieldSummary({
+      pool,
+      sourceIds: opts.sourceIds,
+      documentIds: opts.documentIds,
+      principals: resolvePrincipals(opts.principals, "fieldSummary"),
+      redaction: opts.redaction ?? this.config.redaction,
+      secretKey: this.config.secretKey,
+      hooks: this.hooks
+    });
+  }
   async queryStructured(question, opts = {}) {
     const pool = await this.ensurePool();
     return queryStructured(question, {
@@ -13763,7 +14470,10 @@ var ContextEngine = class _ContextEngine {
       principals: resolvePrincipals(opts.principals, "executeTool"),
       actor: opts.actor,
       source: opts.source ?? "api",
-      approvalScope: opts.approvalScope
+      approvalScope: opts.approvalScope,
+      resultMaxChars: opts.resultMaxChars,
+      resultMaxRows: opts.resultMaxRows,
+      responseMode: opts.responseMode
     });
   }
 };
@@ -14071,6 +14781,8 @@ exports.computeOverFrames = computeOverFrames;
 exports.configSchema = configSchema;
 exports.createMcpApp = createMcpApp;
 exports.decryptDict = decryptDict;
+exports.documentStructure = documentStructure;
+exports.documentTypes = documentTypes;
 exports.emitError = emitError;
 exports.emitProgress = emitProgress;
 exports.emitToolCall = emitToolCall;
@@ -14078,6 +14790,8 @@ exports.emitUsage = emitUsage;
 exports.encryptDict = encryptDict;
 exports.extract = extract2;
 exports.extractStructuredData = extractStructuredData;
+exports.fieldSummary = fieldSummary;
+exports.formatResult = formatResult;
 exports.functionTool = functionTool;
 exports.getDocumentText = getDocumentText;
 exports.getSecretKey = getSecretKey;
@@ -14092,10 +14806,13 @@ exports.resolveApproval = resolveApproval;
 exports.resolveFields = resolveFields;
 exports.resolvePrincipals = resolvePrincipals;
 exports.resolveScope = resolveScope;
+exports.rowsToTsv = rowsToTsv;
 exports.rrfFuse = rrfFuse;
 exports.runMigrate = runMigrate;
 exports.runSearch = runSearch;
 exports.shouldRequireApproval = shouldRequireApproval;
+exports.spreadsheetSchema = spreadsheetSchema;
+exports.spreadsheetSchemaFromText = spreadsheetSchemaFromText;
 exports.unitsForFile = unitsForFile;
 exports.upsertRegistry = upsertRegistry;
 //# sourceMappingURL=index.cjs.map