bunnyquery 1.6.0 → 1.7.0

package/src/engine/links.ts

@@ -10,8 +10,45 @@ export var EXPIRED_ATTACHMENT_URL_HOST = '_expired_.url';
 export var EXPIRED_ATTACHMENT_URL_ORIGIN = 'https://' + EXPIRED_ATTACHMENT_URL_HOST;
 export var LINK_LABEL_MAX_DISPLAY_CHARS = 32;
 
+/**
+ * Lifetime of the url minted when a user clicks an expired attachment chip.
+ *
+ * Mint it as a PLAIN get-db presign, never with generate_temporary_cdn_url: the
+ * cdn branch ignores `expires` entirely and hands back a url good for the rest of
+ * the current UTC day plus the next one, so a "20 minute" link would in fact live
+ * 24 to 48 hours. The dashboard has always done this correctly and the widget did
+ * not, which is precisely the kind of divergence a shared constant exists to stop.
+ */
+export var EXPIRED_LINK_REFRESH_EXPIRES_SECONDS = 20 * 60;
+
+/**
+ * How long a client may keep serving an href it already minted before dropping
+ * back to the placeholder and re-minting.
+ *
+ * DERIVED from the TTL above, with five minutes of headroom, because the
+ * invariant "the cache must expire before the url does" used to be a comment
+ * next to two independent literals. If it is ever violated a client serves a
+ * dead url with no way to notice; deriving it makes that unrepresentable.
+ */
+export var LINK_REFRESH_WINDOW_MS = (EXPIRED_LINK_REFRESH_EXPIRES_SECONDS - 5 * 60) * 1000;
+
+// The two "balanced parens" groups match ONE CHARACTER per step, never a `+`
+// run, so each position has exactly one way to be matched: `[^()\n]` cannot
+// start with `(`, and the nested-paren alternative always does. That disjointness
+// is load-bearing, not style.
+//
+// The obvious spelling — `(?:[^()\n]+|\([^()\n]*\))+` — is a nested quantifier:
+// a run of N plain characters can be split across the outer `+` in 2^N ways, and
+// the engine tries EVERY one before it can conclude the branch failed. A branch
+// fails on ordinary input: a link whose url contains a space (the url branch
+// forbids spaces), a link broken across a newline, or a reply truncated
+// mid-link. Measured on the unfixed pattern: `[label](` plus 30 characters with
+// no closing paren took 62 SECONDS, 45 characters never finished. Since this
+// regex is scanned over the whole reply by the typewriter (session.ts) and by
+// every message render (parseMsgParts), that is a permanently frozen tab the
+// moment such a message arrives. Same matches either way, linear time.
 export function createInlineLinkRegex(): RegExp {
-	return /src::(\S+)|\[([^\]\n]+)\]\((https?:\/\/(?:[^\s()]+|\([^\s()]*\))+)\)|\[([^\]\n]+)\]\(((?:[^()\n]+|\([^()\n]*\))+)\)|(https?:\/\/[^\s<>"']+)/g;
+	return /src::(\S+)|\[([^\]\n]+)\]\((https?:\/\/(?:[^\s()]|\([^\s()]*\))+)\)|\[([^\]\n]+)\]\(((?:[^()\n]|\([^()\n]*\))+)\)|(https?:\/\/[^\s<>"']+)/g;
 }
 
 export function safeDecodeURIComponent(v: string): string {
@@ -70,25 +107,288 @@ export function isServiceDbAttachmentHref(href: string, serviceId: string): bool
 	} catch (e) { return false; }
 }
 
+/**
+ * Read the storage path back out of an `_expired_.url` placeholder.
+ *
+ * The placeholder is not a display detail: sanitizeAttachmentLinksForHistory
+ * writes it into PERSISTED history, and buildBoundedChatMessages replays it into
+ * the model's context. So it round-trips constantly and MUST be recognised on the
+ * way back in. Returns null for anything that is not the carrier.
+ */
+export function readExpiredAttachmentHref(href: string): string | null {
+	if (!href) return null;
+	try {
+		var parsed = new URL(href);
+		if (parsed.hostname !== EXPIRED_ATTACHMENT_URL_HOST) return null;
+		return normalizeAttachmentPathCandidate(parsed.pathname || '') || null;
+	} catch (e) { return null; }
+}
+
 // Replace volatile attachment URLs with their durable `_expired_.url/<path>`
 // placeholder so a stored/replayed copy re-mints on demand instead of going stale.
-// USER turns: the "Attached files:" block gates the (broad) rewrite — every url
-// there is a db link. ASSISTANT turns (forAssistant=true): no marker exists and the
-// text may contain external citation urls, so restrict the rewrite to THIS
-// service's db attachment urls and leave every other link untouched.
+//
+// Only THIS service's db urls are rewritten, whichever role wrote them. The user
+// branch used to rewrite every url in any message that carried an "Attached
+// files:" block, which quietly destroyed a third-party link the user happened to
+// paste in the same message: it became a placeholder for a storage path that
+// never existed. We can only re-mint what we host, so we only rewrite what we
+// host.
 export function sanitizeAttachmentLinksForHistory(content: string, serviceId: string, forAssistant?: boolean): string {
 	if (!content) return content;
 	if (!forAssistant && content.indexOf('Attached files:') === -1) return content;
 	return content.replace(/\[([^\]\n]+)\]\((https?:\/\/[^\s)]+)\)/g, function (_m: string, label: string, href: string) {
-		if (forAssistant && !isServiceDbAttachmentHref(href, serviceId)) return _m;
+		if (!isServiceDbAttachmentHref(href, serviceId)) return _m;
 		var remotePath = extractRemotePathFromAttachmentHref(href, serviceId);
-		var labelPath = normalizeAttachmentPathCandidate(label);
-		var fullPath = remotePath || labelPath;
-		if (!fullPath) return forAssistant ? _m : '[' + label + '](' + EXPIRED_ATTACHMENT_URL_ORIGIN + '/file)';
+		var fullPath = remotePath || normalizeAttachmentPathCandidate(label);
+		if (!fullPath) return _m;
 		return '[' + label + '](' + buildDisplayExpiredAttachmentHref(fullPath, label) + ')';
 	});
 }
 
+/**
+ * Is this markdown link target a URL rather than a db storage path?
+ *
+ * The inline-link regex decides that by whether the target contains whitespace:
+ * its url branch forbids it, its bare-path branch allows it (a db path really can
+ * contain spaces). So a url that picked up a stray space anywhere in transit
+ * falls out of the url branch and is claimed by the path branch, and the view
+ * renders it as an `_expired_.url/https%3A/…` attachment chip that resolves to
+ * nothing. The view asks this FIRST, so what a link IS never depends on damage.
+ */
+export function isHttpUrlLike(target: string): boolean {
+	return /^https?:\/\//i.test((target || '').trim());
+}
+
+/**
+ * Repair whitespace inside a url. RFC 3986 has no legal whitespace anywhere in a
+ * URI, so a space in an href is always damage, never content.
+ *
+ * Two repairs, because the right one differs:
+ *   - Our own `/download/<id>` capability links (skapi-mcp file-download.js) are
+ *     base64url, optionally with a single `.` separating the payload and hmac of
+ *     the older self-describing token. That alphabet cannot contain whitespace,
+ *     so the spaces are purely damage and REMOVING them restores the exact link,
+ *     which is what makes an already-sent message clickable again. A model
+ *     reproducing one of these into its reply is exactly where the spaces come
+ *     from, which is also why the id is now short.
+ *   - Anything else keeps every character and only has the whitespace encoded,
+ *     the same thing a browser does with a space in an href. Stripping would be
+ *     wrong there: `…/exports/my report.csv` is a real file whose name has a
+ *     space in it, and deleting it points at a file that does not exist.
+ */
+export function repairUrlWhitespace(href: string): string {
+	if (!href || !/\s/.test(href)) return href;
+	var stripped = href.replace(/\s+/g, '');
+	if (/^https?:\/\/[^/\s]+\/download\/[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)?$/i.test(stripped)) return stripped;
+	return href.trim().replace(/\s/g, '%20');
+}
+
+/**
+ * Trim punctuation and unmatched wrappers that cling to a token in prose.
+ * `src::a/b.pdf).` -> `src::a/b.pdf`, while a balanced `file (v2).pdf` is kept.
+ */
+export function normalizeTrailingInlineToken(value: string): string {
+	if (!value) return value;
+	var out = value.replace(/[.,;:!?]+$/, '');
+	var trimUnmatched = function (openCh: string, closeCh: string) {
+		while (out.charAt(out.length - 1) === closeCh) {
+			var openCount = (out.match(new RegExp('\\' + openCh, 'g')) || []).length;
+			var closeCount = (out.match(new RegExp('\\' + closeCh, 'g')) || []).length;
+			if (closeCount > openCount) out = out.slice(0, -1); else break;
+		}
+	};
+	trimUnmatched('(', ')');
+	trimUnmatched('[', ']');
+	trimUnmatched('{', '}');
+	out = out.replace(/[`'"*>]+$/, '');
+	return out;
+}
+
+/** A link the view renders. `expired` means the href is the `_expired_.url`
+ *  placeholder and a click must mint a fresh one from `remotePath`. */
+export interface InlineLinkPart {
+	type: 'link';
+	label: string;
+	fullLabel: string;
+	href: string;
+	expired: boolean;
+	expiredHref?: string;
+	remotePath?: string;
+}
+
+export interface InlineLinkContext {
+	/** Current project id: the leading segment to strip off a db url. */
+	serviceId: string;
+	/** `https://db.<hostDomain>` for this deployment. */
+	dbHostPrefix: string;
+	/** A fresh url already minted for this placeholder, if the view cached one. */
+	resolveFreshHref?: (expiredHref: string) => string | undefined;
+}
+
+/**
+ * Decide what ONE inline-link regex match actually is, and how to render it.
+ *
+ * This is the single place that answers "is this an external url, this project's
+ * db file, or a bare storage path", for every consumer. It used to live twice,
+ * once in agent.vue and once in the widget, and both copies had to be found and
+ * corrected for each of the link bugs this file's history records. A view now
+ * supplies its own context (project id, db host, cached-href lookup) and does
+ * nothing but turn the returned part into markup.
+ *
+ * `groups` is [g1..g6] from createInlineLinkRegex, in that order:
+ *   g1 src::<token>   g2/g3 [label](url)   g4/g5 [label](path)   g6 bare url
+ */
+export function classifyInlineLink(
+	full: string,
+	groups: Array<string | undefined>,
+	ctx: InlineLinkContext,
+): { part: InlineLinkPart; tail?: string } | null {
+	var g1 = groups[0], g2 = groups[1], g3 = groups[2], g4 = groups[3], g5 = groups[4], g6 = groups[5];
+	var dbHostPrefix = (ctx.dbHostPrefix || '').toLowerCase();
+	var fresh = function (expiredHref: string): string | undefined {
+		return ctx.resolveFreshHref ? ctx.resolveFreshHref(expiredHref) : undefined;
+	};
+	var isDbHost = function (url: string): boolean {
+		return !!dbHostPrefix && url.toLowerCase().indexOf(dbHostPrefix) === 0;
+	};
+	// A db path rendered as the placeholder the click handler resolves.
+	var asStoredFile = function (remotePath: string, label: string): { part: InlineLinkPart } | null {
+		if (!remotePath) return null;
+		var expiredHref = buildDisplayExpiredAttachmentHref(remotePath, label);
+		var cached = fresh(expiredHref);
+		return {
+			part: {
+				type: 'link',
+				label: truncateLabelForDisplay(label),
+				fullLabel: label,
+				href: cached || expiredHref,
+				expired: !cached,
+				expiredHref: expiredHref,
+				remotePath: remotePath,
+			},
+		};
+	};
+
+	// src::<token> — a path, or a url the model copied out of a record.
+	if (g1) {
+		var rawPath = normalizeTrailingInlineToken(g1);
+		var tail = full.slice(('src::' + rawPath).length);
+		var srcIsUrl = isHttpUrlLike(rawPath);
+		// `src::` values come straight out of a record's unique_id, and the prompt
+		// says that may be "the file's storage path or original URL". http:// is as
+		// much a url as https://; testing only for https sent every plain-http
+		// source into the storage-path branch, where it became a chip pointing at
+		// this project for someone else's file.
+		if (srcIsUrl && !isDbHost(rawPath) && !readExpiredAttachmentHref(rawPath)) {
+			return {
+				part: { type: 'link', label: truncateLabelForDisplay(rawPath), fullLabel: rawPath, href: rawPath, expired: false },
+				tail: tail,
+			};
+		}
+		var srcPath = readExpiredAttachmentHref(rawPath)
+			|| (srcIsUrl
+				? (extractRemotePathFromAttachmentHref(rawPath, ctx.serviceId) || normalizeAttachmentPathCandidate(rawPath))
+				: normalizeAttachmentPathCandidate(rawPath));
+		var srcBuilt = asStoredFile(srcPath, srcPath);
+		return srcBuilt ? { part: srcBuilt.part, tail: tail } : null;
+	}
+
+	// [label](target) where target is NOT a url by the regex's reckoning.
+	if (g4 && g5) {
+		// An EXPLICIT db target, `[label](db:folder/file.csv)`, says what it is
+		// instead of leaving us to infer it from the absence of "http". That is the
+		// only form here that cannot be confused with anything else, and it matches
+		// the scheme the backend already uses internally (db:<service>/<key>).
+		// Accepted now so the clients tolerate it everywhere before anything starts
+		// EMITTING it; until then this branch simply never fires.
+		var dbTarget = /^db:(.+)$/i.exec(g5.trim());
+		if (dbTarget) {
+			var declared = asStoredFile(normalizeAttachmentPathCandidate(dbTarget[1]), g4);
+			if (!declared) return null;
+			declared.part.label = truncateLabelForDisplay(g4);
+			declared.part.fullLabel = g4;
+			return declared;
+		}
+		// ...except when it is one. It only lands here because it contains
+		// whitespace, which the url branch forbids and this one allows, and
+		// reading a damaged url as a storage path is how a download link became a
+		// dead chip. Repair it and classify it as what it is.
+		if (isHttpUrlLike(g5)) {
+			return classifyInlineLink(full, [undefined, g4, repairUrlWhitespace(g5), undefined, undefined, undefined], ctx);
+		}
+		// Any OTHER scheme, or a fragment, is a link the user wrote and not a file
+		// we host. `mailto:`, `tel:` and `#section` were all being turned into
+		// download chips for storage paths of that literal text, so clicking one
+		// asked this project for a file called "mailto:a@b.com".
+		var trimmedTarget = g5.trim();
+		if (/^[a-z][a-z0-9+.-]*:/i.test(trimmedTarget) || trimmedTarget.charAt(0) === '#') {
+			return {
+				part: { type: 'link', label: truncateLabelForDisplay(g4), fullLabel: g4, href: trimmedTarget, expired: false },
+			};
+		}
+		var built = asStoredFile(normalizeAttachmentPathCandidate(g5), g4);
+		if (!built) return null;
+		// The label is the model's, not the path: keep it verbatim.
+		built.part.label = truncateLabelForDisplay(g4);
+		built.part.fullLabel = g4;
+		return built;
+	}
+
+	// [label](url) and bare urls.
+	var originalHref = g3 || g6 || '';
+	if (!originalHref) return null;
+	// A bare url swallows the punctuation that ends the sentence it sits in, so
+	// `see https://host/a.pdf.` linked to `a.pdf.` and 404'd. Trim it and hand the
+	// trimmed text back as `tail`, exactly as the src:: branch does.
+	var urlTail: string | undefined;
+	if (!g3 && g6) {
+		var trimmedUrl = normalizeTrailingInlineToken(originalHref);
+		if (trimmedUrl !== originalHref) urlTail = originalHref.slice(trimmedUrl.length);
+		originalHref = trimmedUrl;
+	}
+	var withTail = function (r: { part: InlineLinkPart }): { part: InlineLinkPart; tail?: string } {
+		return urlTail ? { part: r.part, tail: urlTail } : r;
+	};
+	var urlLabel = g2 || originalHref;
+
+	// THE PLACEHOLDER, read back. sanitizeAttachmentLinksForHistory writes this
+	// form into stored history and buildBoundedChatMessages replays it to the
+	// model, so it arrives here constantly: as a rebuilt bubble on every reload,
+	// and as text the model copied out of its own context. It has to be checked
+	// BEFORE the generic https branch, because it IS https and it is NOT the db
+	// host, so that branch claimed it and rendered `expired: false` — a link to a
+	// hostname that does not resolve, with no way to ever refresh it. Every stored
+	// attachment link went dead on reload for exactly that reason.
+	var carried = readExpiredAttachmentHref(originalHref);
+	if (carried) {
+		var carriedBuilt = asStoredFile(carried, g2 || carried);
+		if (carriedBuilt) {
+			if (g2) { carriedBuilt.part.label = truncateLabelForDisplay(g2); carriedBuilt.part.fullLabel = g2; }
+			return withTail(carriedBuilt);
+		}
+	}
+
+	// This project's own db url: volatile, so render it re-mintable. A db url for
+	// a DIFFERENT project is not ours to mint, so it stays an ordinary link rather
+	// than a chip that would query this project for someone else's key.
+	if (isServiceDbAttachmentHref(originalHref, ctx.serviceId)) {
+		var remotePath = extractRemotePathFromAttachmentHref(originalHref, ctx.serviceId);
+		if (remotePath) {
+			var dbBuilt = asStoredFile(remotePath, getExpiredAttachmentVisiblePath(remotePath, urlLabel));
+			if (dbBuilt) return withTail(dbBuilt);
+		}
+	}
+
+	// Everything else is a link, not a path. The old rule tested for `https://`
+	// specifically and treated ANY other target as db storage, so a plain
+	// `http://` citation, a `mailto:`, a `#anchor` and a `/relative` link all
+	// rendered as download chips for storage paths that never existed, and
+	// clicking one raised "failed to refresh" on a file the user never had.
+	return withTail({
+		part: { type: 'link', label: truncateLabelForDisplay(urlLabel), fullLabel: urlLabel, href: originalHref, expired: false },
+	});
+}
+
 export function truncateLabelForDisplay(label: string): string {
 	if (!label) return label;
 	if (label.length <= LINK_LABEL_MAX_DISPLAY_CHARS) return label;

package/src/engine/office.ts

@@ -101,11 +101,32 @@ export function isServerExtractable(name?: string, mime?: string): boolean {
 /** @deprecated renamed to {@link isServerExtractable} (now also covers text files). */
 export const isOfficeFile = isServerExtractable;
 
-// Extensions that are best read WINDOW-BY-WINDOW via the readFileContent tool instead of
-// inlined once: spreadsheets (grid rows + embedded photos, and can be huge) and PDFs
-// (scanned page images). For these the indexing agent is told to page through the whole
-// file with readFileContent rather than reading a capped inline dump or web_fetching a URL.
-const PAGED_READ_EXTENSIONS = new Set(['xls', 'xlsx', 'xlsm', 'ods', 'pdf']);
+// Extensions read WINDOW-BY-WINDOW via the readFileContent tool instead of inlined once.
+//
+// Anything NOT in this set falls back to one-shot server extraction, which is capped at
+// MAX_EXTRACTED_CHARS (200k). Measured against real files, that cap was discarding the
+// overwhelming majority of every large non-spreadsheet upload:
+//   5MB .txt   -> 4.0% indexed (4.8M characters silently dropped)
+//   4.8MB .json-> 4.2%
+//   1.9M-char Korean .txt -> 10.5%
+//   .docx      -> 70.6%
+// The truncation was invisible: the agent received a plausible-looking document and had
+// no way to know most of it was missing. Windowing these types is what makes "index the
+// whole file" true rather than aspirational.
+//
+// CSV/TSV specifically must be here rather than in the inline path: the layer now gives
+// them ROW-bounded windows with absolute row numbers, where the character windower used
+// to split rows across boundaries and emit no row numbers at all.
+const PAGED_READ_EXTENSIONS = new Set([
+	// grids
+	'xls', 'xlsx', 'xlsm', 'ods',
+	// delimited text (row-windowed by the layer)
+	'csv', 'tsv', 'tab',
+	// documents
+	'pdf', 'docx', 'pptx',
+	// plain text / data / markup
+	'txt', 'md', 'markdown', 'log', 'json', 'jsonl', 'ndjson', 'xml', 'yaml', 'yml',
+]);
 
 /**
  * True when a file should be indexed by PAGING through readFileContent (spreadsheets and
@@ -160,6 +181,33 @@ export function makeRenderPlaceholder(seed: string): string {
 // resume window (from = pass * PAGES) lines up with what the worker renders.
 export const RENDER_PAGES_PER_WINDOW = 5;
 
+// Token the WORKER substitutes with a human description of the next window's position.
+// Shared with the render loop so one substitution path serves both.
+export const WINDOW_CURSOR_TOKEN = '{{RENDER_FROM}}';
+
+let _windowPlaceholderSeq = 0;
+
+/**
+ * Placeholder the worker replaces with ONE window of a file's text/grid content.
+ * Distinct token from the render (page-image) and extract (whole-file) placeholders so
+ * a stale token from either can never be mistaken for this one.
+ */
+export function makeWindowPlaceholder(seed: string): string {
+	_windowPlaceholderSeq += 1;
+	const slug = (seed || 'file').replace(/[^a-zA-Z0-9]+/g, '_').slice(-48);
+	return `{{SKAPI_WINDOW::${slug}-${_windowPlaceholderSeq}}}`;
+}
+
+/**
+ * True when a file should be read server-side, one window at a time, by the worker.
+ * PDFs are excluded: they go through the VISION path, where pages are rendered to
+ * images because their text layer is often absent or unreliable.
+ */
+export function isWindowedReadFile(name?: string, mime?: string): boolean {
+	if (isImageVisionFile(name, mime)) return false;
+	return isPagedReadFile(name, mime);
+}
+
 export interface ComposedUserMessage {
 	/** Clean display/history copy (attachment links, NO extraction placeholders). */
 	composed: string;

package/src/engine/prompts/chat_system_prompt.ts

@@ -30,7 +30,7 @@ File attachments: When a user message contains an "Attached files:" section with
 - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
 - Most attached files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY had their text extracted on the server and inlined in the same message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read it directly there and do NOT call web_fetch for those files. A "[skapi: ...]" note in that block means the file could not be extracted.
 - For any file given to you as a URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL before answering. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
-File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix — do NOT show it. Format: [filename](path/to/file) for storage paths, or [filename](https://...) for external URLs. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand — so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
+File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix — do NOT show it. Format: [filename](db:path/to/file) for storage paths, or [filename](https://...) for external URLs. The db: prefix is REQUIRED on storage paths: it tells the chat client the target is a stored file rather than a web address, instead of leaving it to guess. Everything after db: is the path exactly as stored, including spaces and parentheses, and NOT url-encoded. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand — so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
 File lookup: When the user asks to see, list, or show files (e.g. "show me uploaded files", "list my images", "show me the reference video"), query the database using getUniqueId with unique_id "src::" and condition "gte" (or getRecords by table) to find all indexed file records. Present each result as a markdown link as described above. Never say you cannot access file storage — the file paths are indexed in the database and are always reachable through it.
 File generation: When the user asks you to generate a file — or to produce specifically-formatted text such as HTML, CSV, JSON, or Markdown — put the file's full contents inside a fenced code block whose info string is the intended filename WITH its extension (e.g. report.csv), NOT a language name like "csv". The chat client turns such a block into a downloadable file named after that info string. Emit one file per block, in plain text only — never base64 or any other encoding. Example for CSV:
 \`\`\`filename.csv

package/src/engine/prompts/index.ts

@@ -11,6 +11,9 @@ export {
 	buildIndexingUserMessage,
 	buildIndexingContinueMessage,
 	buildIndexingRenderMessage,
+	buildIndexingRenderContinueTemplate,
+	buildIndexingWindowMessage,
+	RENDER_FROM_TOKEN,
 	type IndexingAttachmentInfo,
 	type BuildIndexingUserMessageOptions,
 } from './indexing_user_message';

package/src/engine/prompts/indexing_user_message.ts

@@ -101,16 +101,29 @@ export function buildIndexingUserMessage(
 	return head + `- temporary URL (fetch this to read the file contents): ${attachment.url}`;
 }
 
+/**
+ * Token the WORKER substitutes with the 1-based first page of the window it is about to
+ * render, when it builds the next pass of a document from `RENDER_CONTINUE_TEMPLATE`.
+ * Must match the worker's RENDER_FROM_TOKEN.
+ */
+export const RENDER_FROM_TOKEN = '{{RENDER_FROM}}';
+const WINDOW_CURSOR_TOKEN = RENDER_FROM_TOKEN;
+
 /**
  * User message for a VISION file (PDF): its pages are delivered as RENDERED PAGE IMAGES that
  * the proxy worker injects into THIS message at the `placeholder` token (tool-result images
  * render on neither provider, so the pages must be image blocks in the message itself). Each
- * pass shows one WINDOW of pages starting at `renderFrom` (0-based); the resume loop advances
- * the window a pass at a time until the injected note says the last window was reached.
+ * pass shows one WINDOW of pages starting at `renderFrom` (0-based).
+ *
+ * The WORKER advances the window: when its renderer reports pages remaining it enqueues the
+ * next pass itself, off the true page count, so a document indexes end-to-end with no browser
+ * involved. This message therefore only ever describes ONE window, and the model is never
+ * asked to decide whether the document is finished.
  *
  * renderFrom === 0 is the FIRST pass (leads with "A new file has just been uploaded." so the
- * client builds the "Indexing: <name>" bubble); renderFrom > 0 is a RESUME pass (leads with
- * "CONTINUE indexing" like the paged continue message, so it is not a duplicate primary bubble).
+ * client builds the "Indexing: <name>" bubble); a continue pass (built by the worker from
+ * buildIndexingRenderContinueTemplate) leads with "CONTINUE indexing" so it is not a duplicate
+ * primary bubble.
  */
 export function buildIndexingRenderMessage(
 	attachment: IndexingAttachmentInfo,
@@ -118,44 +131,112 @@ export function buildIndexingRenderMessage(
 	renderFrom: number,
 ): string {
 	const from = Math.max(0, renderFrom || 0);
+	if (from > 0) return buildIndexingRenderContinueTemplate(attachment, placeholder, String(from + 1));
+
+	return (
+		`A new file has just been uploaded. Index it now.\n\n` +
+		buildRenderMeta(attachment) +
+		`\nThis is a PDF. Its pages are delivered to you as RENDERED PAGE IMAGES embedded directly in this ` +
+		`message (you do NOT need any tool, URL, or web_fetch to see them). You are shown a WINDOW of pages ` +
+		`at a time, starting at page ${from + 1}.\n` +
+		buildRenderDatafy(placeholder)
+	);
+}
+
+/**
+ * The CONTINUE pass, as a template the worker fills in. `pageLabel` defaults to the
+ * RENDER_FROM_TOKEN placeholder, which the worker replaces with the real 1-based start page
+ * of the window it is rendering; passing an explicit label produces a ready-to-send message.
+ */
+export function buildIndexingRenderContinueTemplate(
+	attachment: IndexingAttachmentInfo,
+	placeholder: string,
+	pageLabel: string = RENDER_FROM_TOKEN,
+): string {
 	const src = `src::${attachment.storagePath}`;
-	const meta =
+	return (
+		`CONTINUE indexing a PDF whose previous pass did not finish.\n\n` +
+		buildRenderMeta(attachment) +
+		`\nRecords for the earlier pages are ALREADY saved (they reference "${src}"). The NEXT window of ` +
+		`rendered page images (starting at page ${pageLabel}) is embedded in this message. Datafy each page as ` +
+		`before and do NOT re-save pages that are already saved.\n` +
+		buildRenderDatafy(placeholder)
+	);
+}
+
+function buildRenderMeta(attachment: IndexingAttachmentInfo): string {
+	return (
 		`File metadata:\n` +
 		`- name: ${attachment.name}\n` +
 		`- storage path: ${attachment.storagePath}\n` +
-		(attachment.mime ? `- mime type: ${attachment.mime}\n` : '');
+		(attachment.mime ? `- mime type: ${attachment.mime}\n` : '')
+	);
+}
 
-	// Shared datafy + completion guidance. The placeholder is where the worker splices the
-	// note + rendered page images; instructions reference "the page images in this message"
-	// so they read correctly whether the images land before or after this text.
-	const datafy =
+// Shared datafy guidance. The placeholder is where the worker splices the note + rendered
+// page images; instructions reference "the page images in this message" so they read
+// correctly whether the images land before or after this text.
+//
+// Deliberately says nothing about INDEXING_COMPLETE or about whether the document is
+// finished: the worker decides that from the renderer's page count. Asking the model was
+// what used to end an 88-page file at page 15.
+function buildRenderDatafy(placeholder: string): string {
+	return (
 		`\n${placeholder}\n\n` +
 		`LOOK at each rendered page image in this message and DATAFY what it shows: for EVERY page ` +
 		`call postRecords and save records - one record per row / table entry / line item visible on the page ` +
 		`(or one record for the page if it is prose), capturing every value you can read (OCR the text, read tables ` +
 		`cell by cell, describe any photos/diagrams). Use the storage path above for the "src::" unique_id.\n\n` +
-		`The note next to the images tells you whether MORE pages remain after this window. ` +
-		`If MORE remain: save this window's records and STOP - do NOT write INDEXING_COMPLETE; another pass shows the next window automatically. ` +
-		`Only when the note says this is the LAST window (you have seen the whole file) AND everything is saved, end your message with the token INDEXING_COMPLETE.`;
+		`Save records for THIS window of pages only, then stop and report what you saved. Do NOT try to read ` +
+		`the rest of the file and do NOT worry about the pages after this window: if any remain, the next window ` +
+		`is rendered and sent to you automatically. Report only the pages you were actually shown - never imply ` +
+		`you have seen the whole document.`
+	);
+}
 
-	if (from === 0) {
-		return (
-			`A new file has just been uploaded. Index it now.\n\n` +
-			meta +
-			`\nThis is a PDF. Its pages are delivered to you as RENDERED PAGE IMAGES embedded directly in this ` +
-			`message (you do NOT need any tool, URL, or web_fetch to see them). You are shown a WINDOW of pages ` +
-			`at a time, starting at page ${from + 1}.\n` +
-			datafy
-		);
-	}
+/**
+ * User message for a WINDOWED file: the worker splices ONE window of the file's rows or
+ * text into this message at `placeholder`, then continues from the reader's own cursor
+ * until the file is exhausted.
+ *
+ * The agent is deliberately NOT asked to page the file itself, and is NOT asked to judge
+ * whether it is finished. Both used to be its job, and both failed the same way: the
+ * traversal lived inside a single turn's budget, so a large file simply stopped partway
+ * with a confident summary of the part it had seen.
+ */
+export function buildIndexingWindowMessage(
+	attachment: IndexingAttachmentInfo,
+	placeholder: string,
+	isContinuation: boolean,
+	positionLabel?: string,
+): string {
+	const src = `src::${attachment.storagePath}`;
+	const head = isContinuation
+		? `CONTINUE indexing a file whose previous pass did not finish.\n\n`
+		: `A new file has just been uploaded. Index it now.\n\n`;
+	const where = isContinuation
+		? `\nRecords for the earlier windows are ALREADY saved (they reference "${src}"). The NEXT window ` +
+		  `(starting at ${positionLabel || WINDOW_CURSOR_TOKEN}) is embedded below. Do NOT re-save windows that are already saved.\n`
+		: `\nThis file is delivered to you ONE WINDOW at a time, embedded directly in this message. ` +
+		  `You do NOT need any tool, URL, or web_fetch to read it.\n`;
 
 	return (
-		`CONTINUE indexing a PDF whose previous pass did not finish.\n\n` +
-		meta +
-		`\nRecords for the earlier pages are ALREADY saved (they reference "${src}"). The NEXT window of ` +
-		`rendered page images (starting at page ${from + 1}) is embedded in this message. Datafy each page as ` +
-		`before and do NOT re-save pages that are already saved.\n` +
-		datafy
+		head +
+		buildRenderMeta(attachment) +
+		where +
+		`\n${placeholder}\n\n` +
+		`DATAFY this window: call postRecords and save records for everything in it - ONE RECORD PER ROW ` +
+		`for tabular data (keyed by the column headers), or one record per section for prose. Capture every ` +
+		`value you can read. Use the storage path above for the "src::" unique_id on the file-level record, ` +
+		`and link every row/section record to it by reference.\n\n` +
+		`If this window has PHOTOS attached as images, LOOK at each one and datafy what it actually shows ` +
+		`into the record for the row it is anchored to (a «PHOTO A88» marker in the grid text only says WHERE ` +
+		`a picture sits - the picture itself is attached to this message). Never report that photo contents ` +
+		`could not be extracted when images are attached here.\n\n` +
+		`Save records for THIS window only, then stop and report what you saved. Do NOT try to read the rest ` +
+		`of the file, and do NOT call readFileContent - if more remains, the next window is read and sent to ` +
+		`you automatically. Report only what you were actually shown, and never imply you have seen the whole ` +
+		`file when the note beside the window says more remains.`
 	);
 }