bunnyquery 1.5.6 → 1.5.7

package/bunnyquery.js

@@ -149,6 +149,13 @@
     if (isTextMime(m)) return true;
     return m.includes("officedocument") || m.includes("opendocument") || m.includes("hwp") || m.includes("epub") || m === "application/msword" || m === "application/vnd.ms-excel" || m === "application/vnd.ms-powerpoint";
   }
+  var PAGED_READ_EXTENSIONS = /* @__PURE__ */ new Set(["xls", "xlsx", "xlsm", "ods", "pdf"]);
+  function isPagedReadFile(name, mime) {
+    const ext = (name || "").split(".").pop()?.toLowerCase() || "";
+    if (PAGED_READ_EXTENSIONS.has(ext)) return true;
+    const m = (mime || "").toLowerCase();
+    return m === "application/pdf" || m === "application/vnd.ms-excel" || m.includes("spreadsheetml") || m.includes("opendocument.spreadsheet");
+  }
   var _extractPlaceholderSeq = 0;
   function makeExtractPlaceholder(seed) {
     _extractPlaceholderSeq += 1;
@@ -251,12 +258,13 @@ Project description: """${serviceDescription}"""`;
     const { service, serviceName, serviceDescription } = params;
     let systemPrompt = `You are a background indexing agent for project ${service}.
 - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
-- Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly and do NOT call web_fetch for those files. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
-- For any file given to you as a temporary URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
+- Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
+- BIG or SCANNED files: the inline content may be only the FIRST part of a large file (it can end with a truncation or "more remains" note), and scanned PDFs / files with embedded photos are not fully captured inline. In those cases READ THE FILE WITH THE readFileContent TOOL: it returns the file ONE WINDOW at a time (spreadsheets as coordinate-tagged grid rows, scanned/large PDFs as rendered PAGE IMAGES, text as a range of characters). Pass the file's storage path. After each window: datafy it into records and SAVE them, THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed - never stop after the first window.
+- VISION: when a readFileContent window (or an inline attachment) includes IMAGES - scanned PDF pages, or photos embedded in a spreadsheet next to a row/block - LOOK at them and capture what they show as record data (the reading/values in a scanned table, the part/defect/condition visible in a photo). The image IS part of the data; correlate each photo with its labelled block ("PHOTO A3" markers tie a photo to that grid row).
 - Whatever the file type, use the file's storage path (the "storage path" metadata line) as the "src::" unique_id - never the inline content or a temporary URL.
-- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet. If a sheet has many rows, make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
+- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.xls/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet, paging through it with readFileContent when it is large. Make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
 - EPUB / e-books / long-form books (.epub or any book-length prose, provided inline in reading order with chapter headings preserved): you MUST save ONE record per CHAPTER (or, when chapters are unclear, per major section/topic) in a dedicated table (e.g. "book_chapters") - never collapse the whole book into a single record. Each chapter record's "data" must capture the chapter title plus its order/number AND a substantive summary of that chapter's content (key events, arguments, characters, places, concepts, terms, notable quotes). Apply AS MANY relevant tags as possible to EVERY chapter record (characters, locations, themes, topics, key concepts, key terms, dates, named entities) so the book is easy to SEARCH and cross-reference later - this is the whole point. ALSO save one book-level record (title, author, language, overall summary, chapter list / table of contents, genre/subjects) and link each chapter record to it via reference. This per-chapter completeness OVERRIDES brevity; human-readable summaries only, never raw/binary bytes.
-- This is a ONE-SHOT background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions or invite back-and-forth. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
+- This is a background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). SAVE AS YOU GO: persist each window's records before reading the next, so progress is never lost. If the file is so large you cannot finish in one turn, still save everything you have read so far and note the last cursor/page you reached; a follow-up will continue from there. Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
 - Only AFTER every save is done, send exactly ONE final message summarizing what you indexed - never just "Indexing complete", and never a raw/base64/binary value or a large pasted dump. Keep it to a few factual sentences or a short markdown bullet list covering: the file name, its content type, each table you wrote to with its record/row count and the key columns/fields or topics captured, and anything that could not be extracted. Follow this shape - Indexed <file name> (<content type>): saved <N> records to <table(s)> capturing <key columns/fields or topics>; could not extract: <gaps, or none>.`;
     if (serviceDescription) {
       systemPrompt += `
@@ -291,6 +299,11 @@ The file's text content was extracted on the server and is provided inline below
 ----- BEGIN FILE CONTENT -----
 ${options.inlineContentPlaceholder}
 ----- END FILE CONTENT -----`;
+    }
+    if (options?.pagedRead) {
+      return head + `
+Read this file with the readFileContent tool, using the storage path above - do NOT fetch a URL and do NOT rely on a single sample. readFileContent returns the file ONE WINDOW at a time: spreadsheets as coordinate-tagged grid rows (e.g. 'R4 A:E&I NUMBER | B:E1007'), scanned/large PDFs as rendered PAGE IMAGES, and windows may include embedded photos - LOOK at any images and datafy what they show. Page through EVERY window: for each window SAVE records for its rows/items/pages (postRecords, one record per row/item), THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed. Do NOT stop after the first window and do NOT just write a summary. Use the storage path above for the "src::" unique_id.` + (attachment.url ? `
+(A temporary URL is provided ONLY as a fallback if readFileContent fails: ${attachment.url})` : "");
     }
     return head + `- temporary URL (fetch this to read the file contents): ${attachment.url}`;
   }
@@ -797,13 +810,14 @@ ${options.inlineContentPlaceholder}
   }
   async function notifyAgentSaveAttachment(info) {
     const { platform, service, owner, attachment, parsedContent } = info;
-    const serverExtract = !parsedContent && isServerExtractable(attachment.name, attachment.mime);
+    const pagedRead = !parsedContent && isPagedReadFile(attachment.name, attachment.mime);
+    const serverExtract = !parsedContent && !pagedRead && isServerExtractable(attachment.name, attachment.mime);
     const placeholder = serverExtract ? makeExtractPlaceholder(attachment.storagePath) : void 0;
     const extractContent = serverExtract && placeholder ? [{ path: attachment.storagePath, placeholder, name: attachment.name, mime: attachment.mime }] : void 0;
     const skapiExtract = extractContent && extractContent.length ? { _skapi_extract: extractContent } : {};
     const userMessage = buildIndexingUserMessage(
       attachment,
-      parsedContent ? { inlineContent: parsedContent } : placeholder ? { inlineContentPlaceholder: placeholder } : void 0
+      parsedContent ? { inlineContent: parsedContent } : placeholder ? { inlineContentPlaceholder: placeholder } : pagedRead ? { pagedRead: true } : void 0
     );
     const systemPrompt = buildIndexingSystemPrompt({
       service,
@@ -2214,7 +2228,7 @@ ${options.inlineContentPlaceholder}
   (function() {
     var MCP_PROD = "https://mcp.broadwayinc.computer";
     var MCP_DEV = "https://mcp-dev.broadwayinc.computer";
-    var BQ_VERSION = "1.5.5" ;
+    var BQ_VERSION = "1.5.6" ;
     var ATTACHMENT_URL_EXPIRES_SECONDS = 600;
     var GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
     var GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token";

package/dist/engine.cjs

@@ -155,6 +155,13 @@ function isServerExtractable(name, mime) {
   return m.includes("officedocument") || m.includes("opendocument") || m.includes("hwp") || m.includes("epub") || m === "application/msword" || m === "application/vnd.ms-excel" || m === "application/vnd.ms-powerpoint";
 }
 var isOfficeFile = isServerExtractable;
+var PAGED_READ_EXTENSIONS = /* @__PURE__ */ new Set(["xls", "xlsx", "xlsm", "ods", "pdf"]);
+function isPagedReadFile(name, mime) {
+  const ext = (name || "").split(".").pop()?.toLowerCase() || "";
+  if (PAGED_READ_EXTENSIONS.has(ext)) return true;
+  const m = (mime || "").toLowerCase();
+  return m === "application/pdf" || m === "application/vnd.ms-excel" || m.includes("spreadsheetml") || m.includes("opendocument.spreadsheet");
+}
 var _extractPlaceholderSeq = 0;
 function makeExtractPlaceholder(seed) {
   _extractPlaceholderSeq += 1;
@@ -257,12 +264,13 @@ function buildIndexingSystemPrompt(params) {
   const { service, serviceName, serviceDescription } = params;
   let systemPrompt = `You are a background indexing agent for project ${service}.
 - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
-- Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly and do NOT call web_fetch for those files. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
-- For any file given to you as a temporary URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
+- Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
+- BIG or SCANNED files: the inline content may be only the FIRST part of a large file (it can end with a truncation or "more remains" note), and scanned PDFs / files with embedded photos are not fully captured inline. In those cases READ THE FILE WITH THE readFileContent TOOL: it returns the file ONE WINDOW at a time (spreadsheets as coordinate-tagged grid rows, scanned/large PDFs as rendered PAGE IMAGES, text as a range of characters). Pass the file's storage path. After each window: datafy it into records and SAVE them, THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed - never stop after the first window.
+- VISION: when a readFileContent window (or an inline attachment) includes IMAGES - scanned PDF pages, or photos embedded in a spreadsheet next to a row/block - LOOK at them and capture what they show as record data (the reading/values in a scanned table, the part/defect/condition visible in a photo). The image IS part of the data; correlate each photo with its labelled block ("PHOTO A3" markers tie a photo to that grid row).
 - Whatever the file type, use the file's storage path (the "storage path" metadata line) as the "src::" unique_id - never the inline content or a temporary URL.
-- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet. If a sheet has many rows, make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
+- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.xls/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet, paging through it with readFileContent when it is large. Make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
 - EPUB / e-books / long-form books (.epub or any book-length prose, provided inline in reading order with chapter headings preserved): you MUST save ONE record per CHAPTER (or, when chapters are unclear, per major section/topic) in a dedicated table (e.g. "book_chapters") - never collapse the whole book into a single record. Each chapter record's "data" must capture the chapter title plus its order/number AND a substantive summary of that chapter's content (key events, arguments, characters, places, concepts, terms, notable quotes). Apply AS MANY relevant tags as possible to EVERY chapter record (characters, locations, themes, topics, key concepts, key terms, dates, named entities) so the book is easy to SEARCH and cross-reference later - this is the whole point. ALSO save one book-level record (title, author, language, overall summary, chapter list / table of contents, genre/subjects) and link each chapter record to it via reference. This per-chapter completeness OVERRIDES brevity; human-readable summaries only, never raw/binary bytes.
-- This is a ONE-SHOT background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions or invite back-and-forth. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
+- This is a background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). SAVE AS YOU GO: persist each window's records before reading the next, so progress is never lost. If the file is so large you cannot finish in one turn, still save everything you have read so far and note the last cursor/page you reached; a follow-up will continue from there. Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
 - Only AFTER every save is done, send exactly ONE final message summarizing what you indexed - never just "Indexing complete", and never a raw/base64/binary value or a large pasted dump. Keep it to a few factual sentences or a short markdown bullet list covering: the file name, its content type, each table you wrote to with its record/row count and the key columns/fields or topics captured, and anything that could not be extracted. Follow this shape - Indexed <file name> (<content type>): saved <N> records to <table(s)> capturing <key columns/fields or topics>; could not extract: <gaps, or none>.`;
   if (serviceDescription) {
     systemPrompt += `
@@ -297,6 +305,11 @@ The file's text content was extracted on the server and is provided inline below
 ----- BEGIN FILE CONTENT -----
 ${options.inlineContentPlaceholder}
 ----- END FILE CONTENT -----`;
+  }
+  if (options?.pagedRead) {
+    return head + `
+Read this file with the readFileContent tool, using the storage path above - do NOT fetch a URL and do NOT rely on a single sample. readFileContent returns the file ONE WINDOW at a time: spreadsheets as coordinate-tagged grid rows (e.g. 'R4 A:E&I NUMBER | B:E1007'), scanned/large PDFs as rendered PAGE IMAGES, and windows may include embedded photos - LOOK at any images and datafy what they show. Page through EVERY window: for each window SAVE records for its rows/items/pages (postRecords, one record per row/item), THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed. Do NOT stop after the first window and do NOT just write a summary. Use the storage path above for the "src::" unique_id.` + (attachment.url ? `
+(A temporary URL is provided ONLY as a fallback if readFileContent fails: ${attachment.url})` : "");
   }
   return head + `- temporary URL (fetch this to read the file contents): ${attachment.url}`;
 }
@@ -806,13 +819,14 @@ async function callOpenAIWithPublicMcp(prompt, service, owner, messages, system,
 }
 async function notifyAgentSaveAttachment(info) {
   const { platform, service, owner, attachment, parsedContent } = info;
-  const serverExtract = !parsedContent && isServerExtractable(attachment.name, attachment.mime);
+  const pagedRead = !parsedContent && isPagedReadFile(attachment.name, attachment.mime);
+  const serverExtract = !parsedContent && !pagedRead && isServerExtractable(attachment.name, attachment.mime);
   const placeholder = serverExtract ? makeExtractPlaceholder(attachment.storagePath) : void 0;
   const extractContent = serverExtract && placeholder ? [{ path: attachment.storagePath, placeholder, name: attachment.name, mime: attachment.mime }] : void 0;
   const skapiExtract = extractContent && extractContent.length ? { _skapi_extract: extractContent } : {};
   const userMessage = buildIndexingUserMessage(
     attachment,
-    parsedContent ? { inlineContent: parsedContent } : placeholder ? { inlineContentPlaceholder: placeholder } : void 0
+    parsedContent ? { inlineContent: parsedContent } : placeholder ? { inlineContentPlaceholder: placeholder } : pagedRead ? { pagedRead: true } : void 0
   );
   const systemPrompt = buildIndexingSystemPrompt({
     service,