bunnyquery 1.8.2 → 1.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  /**
2
2
  * Pure link/path helpers (no DOM, no marked). Moved verbatim from the chatbox.
3
- * `serviceId` is passed as a PARAMETER (the original read it from a global) so
3
+ * `projectId` is passed as a PARAMETER (the original read it from a global) so
4
4
  * the engine stays consumer-agnostic. The HTML-emitting helpers
5
5
  * (buildLinkPartFromGroups, linkToAnchorHtml, fileToAnchorHtml, parseMsgParts*)
6
6
  * stay in each VIEW — only these pure pieces move here.
@@ -21,6 +21,34 @@ export var LINK_LABEL_MAX_DISPLAY_CHARS = 32;
21
21
  */
22
22
  export var EXPIRED_LINK_REFRESH_EXPIRES_SECONDS = 20 * 60;
23
23
 
24
+ /**
25
+ * Seconds the browser may reuse a minted preview url (`browser_cache`).
26
+ *
27
+ * A presigned url is a fresh SigV4 query string on every mint, so it can never
28
+ * be a browser cache key on its own and every reload re-downloads every image.
29
+ * Asking for the MINT with a cacheable GET fixes it from the other end: the same
30
+ * url comes back out of the browser cache, so the body already on disk stays
31
+ * addressable.
32
+ *
33
+ * Deliberately far longer than EXPIRED_LINK_REFRESH_EXPIRES_SECONDS above, and
34
+ * that is the whole trick: the url is short-lived while the file stays available
35
+ * locally for a WEEK. What keeps an image painting is the cached BODY, not a live
36
+ * url. Once the browser evicts that body it refetches with a url that has since
37
+ * expired, gets a 403, and the error path re-mints with `refresh`. That path is
38
+ * therefore load-bearing, not a rare fallback.
39
+ *
40
+ * A week is the platform default for reading a private file, not a number chosen
41
+ * here: skapi-js reads every private record file with
42
+ * PRIVATE_FILE_BROWSER_CACHE_SECONDS = 7 days against the same 20-minute url, and
43
+ * get_signed_url caps the header at BROWSER_CACHE_MAX_SECONDS = 7 days. A chat
44
+ * that asked for a day was re-downloading images the rest of the product would
45
+ * have served from disk.
46
+ *
47
+ * Applies to previews only. A CLICK must open a live url, so the chip refresh
48
+ * stays on an uncached POST mint.
49
+ */
50
+ export var PREVIEW_BROWSER_CACHE_SECONDS = 7 * 24 * 60 * 60;
51
+
24
52
  /**
25
53
  * How long a client may keep serving an href it already minted before dropping
26
54
  * back to the placeholder and re-minting.
@@ -63,7 +91,7 @@ export function normalizeAttachmentPathCandidate(value: string): string {
63
91
  return safeDecodeURIComponent((value || '').trim()).replace(/\\/g, '/').replace(/^\/+/, '').replace(/\/+/g, '/');
64
92
  }
65
93
 
66
- export function extractRemotePathFromAttachmentHref(href: string, serviceId: string): string | null {
94
+ export function extractRemotePathFromAttachmentHref(href: string, projectId: string): string | null {
67
95
  try {
68
96
  var parsed = new URL(href);
69
97
  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
@@ -71,7 +99,7 @@ export function extractRemotePathFromAttachmentHref(href: string, serviceId: str
71
99
  var segs = path.split('/').filter(Boolean);
72
100
  if (!segs.length) return null;
73
101
  var HEX = /^[a-f0-9]{32,}$/i;
74
- var sid = serviceId || '';
102
+ var sid = projectId || '';
75
103
  var start = 0;
76
104
  while (start < segs.length) {
77
105
  var seg = segs[start];
@@ -94,16 +122,16 @@ export function buildDisplayExpiredAttachmentHref(remotePath: string, fallback?:
94
122
  }
95
123
 
96
124
  // Does `href` point at THIS service's db attachment storage? A db attachment URL's
97
- // path always begins with the serviceId segment (…/<serviceId>/<hash>/<path>). Used
125
+ // path always begins with the projectId segment (…/<projectId>/<hash>/<path>). Used
98
126
  // to SAFELY sanitize assistant messages — where an arbitrary external citation URL
99
127
  // must never be rewritten, only the service's own volatile db links.
100
- export function isServiceDbAttachmentHref(href: string, serviceId: string): boolean {
101
- if (!serviceId) return false;
128
+ export function isServiceDbAttachmentHref(href: string, projectId: string): boolean {
129
+ if (!projectId) return false;
102
130
  try {
103
131
  var parsed = new URL(href);
104
132
  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false;
105
133
  var segs = normalizeAttachmentPathCandidate(parsed.pathname || '').split('/').filter(Boolean);
106
- return segs.length > 0 && segs[0] === serviceId;
134
+ return segs.length > 0 && segs[0] === projectId;
107
135
  } catch (e) { return false; }
108
136
  }
109
137
 
@@ -133,12 +161,12 @@ export function readExpiredAttachmentHref(href: string): string | null {
133
161
  // paste in the same message: it became a placeholder for a storage path that
134
162
  // never existed. We can only re-mint what we host, so we only rewrite what we
135
163
  // host.
136
- export function sanitizeAttachmentLinksForHistory(content: string, serviceId: string, forAssistant?: boolean): string {
164
+ export function sanitizeAttachmentLinksForHistory(content: string, projectId: string, forAssistant?: boolean): string {
137
165
  if (!content) return content;
138
166
  if (!forAssistant && content.indexOf('Attached files:') === -1) return content;
139
167
  return content.replace(/\[([^\]\n]+)\]\((https?:\/\/[^\s)]+)\)/g, function (_m: string, label: string, href: string) {
140
- if (!isServiceDbAttachmentHref(href, serviceId)) return _m;
141
- var remotePath = extractRemotePathFromAttachmentHref(href, serviceId);
168
+ if (!isServiceDbAttachmentHref(href, projectId)) return _m;
169
+ var remotePath = extractRemotePathFromAttachmentHref(href, projectId);
142
170
  var fullPath = remotePath || normalizeAttachmentPathCandidate(label);
143
171
  if (!fullPath) return _m;
144
172
  return '[' + label + '](' + buildDisplayExpiredAttachmentHref(fullPath, label) + ')';
@@ -231,6 +259,59 @@ export function normalizeTrailingInlineToken(value: string): string {
231
259
  return out;
232
260
  }
233
261
 
262
+ /**
263
+ * Extensions a BROWSER can paint in an <img>, mapped to the content type the
264
+ * presign must declare.
265
+ *
266
+ * The content type is not optional here. get_signed_url only sets
267
+ * ResponseContentType when the caller passes `contentType`, and otherwise falls
268
+ * back to application/octet-stream, which a new tab DOWNLOADS instead of
269
+ * displaying. Since the whole point of the preview is that clicking it shows the
270
+ * picture, the mint has to name the real type.
271
+ *
272
+ * Deliberately narrower than the extraction/vision lists elsewhere in the repo:
273
+ * heic/heif out: Safari paints them, Chrome and Firefox show a broken image,
274
+ * and it is the format every iPhone photo arrives in, so the
275
+ * failure would be common and would read as a bug.
276
+ * tif/wmf/emf out: no mainstream browser paints them.
277
+ * svg out: inside an <img> an SVG is script-disabled and safe, but this
278
+ * feature's click target is a TOP-LEVEL navigation, where an
279
+ * SVG executes its own <script> in the serving origin with that
280
+ * origin's cookies, from user-uploaded content. A preview is an
281
+ * invitation to click exactly that.
282
+ */
283
+ export var PREVIEWABLE_IMAGE_CONTENT_TYPES: Record<string, string> = {
284
+ png: 'image/png',
285
+ jpg: 'image/jpeg',
286
+ jpeg: 'image/jpeg',
287
+ gif: 'image/gif',
288
+ webp: 'image/webp',
289
+ avif: 'image/avif',
290
+ bmp: 'image/bmp',
291
+ };
292
+
293
+ /** Extension of a path or url, query and fragment stripped, '' when none. */
294
+ export function previewableExtOf(nameOrPath: string | null | undefined): string {
295
+ var v = String(nameOrPath || '');
296
+ // A storage path may legally contain '?', so this cannot reuse extOf().
297
+ var cut = v.search(/[?#]/);
298
+ if (cut !== -1) v = v.slice(0, cut);
299
+ v = v.replace(/[\\/]+$/, '');
300
+ var dot = v.lastIndexOf('.');
301
+ if (dot <= 0) return '';
302
+ var ext = v.slice(dot + 1).trim().toLowerCase();
303
+ return /^[a-z0-9]+$/.test(ext) ? ext : '';
304
+ }
305
+
306
+ export function isPreviewableImagePath(nameOrPath: string | null | undefined): boolean {
307
+ return !!PREVIEWABLE_IMAGE_CONTENT_TYPES[previewableExtOf(nameOrPath)];
308
+ }
309
+
310
+ /** Content type to hand the presign so a new tab displays rather than downloads. */
311
+ export function previewImageContentType(nameOrPath: string | null | undefined): string | null {
312
+ return PREVIEWABLE_IMAGE_CONTENT_TYPES[previewableExtOf(nameOrPath)] || null;
313
+ }
314
+
234
315
  /** A link the view renders. `expired` means the href is the `_expired_.url`
235
316
  * placeholder and a click must mint a fresh one from `remotePath`. */
236
317
  export interface InlineLinkPart {
@@ -241,11 +322,17 @@ export interface InlineLinkPart {
241
322
  expired: boolean;
242
323
  expiredHref?: string;
243
324
  remotePath?: string;
325
+ /**
326
+ * Set only for a file WE host whose PATH says a browser can paint it. Its
327
+ * presence IS the "render a preview" decision, so a view never re-tests the
328
+ * label and never tests `href` (which is the _expired_.url placeholder).
329
+ */
330
+ image?: { ext: string; contentType: string };
244
331
  }
245
332
 
246
333
  export interface InlineLinkContext {
247
334
  /** Current project id: the leading segment to strip off a db url. */
248
- serviceId: string;
335
+ projectId: string;
249
336
  /** `https://db.<hostDomain>` for this deployment. */
250
337
  dbHostPrefix: string;
251
338
  /** A fresh url already minted for this placeholder, if the view cached one. */
@@ -283,17 +370,22 @@ export function classifyInlineLink(
283
370
  if (!remotePath) return null;
284
371
  var expiredHref = buildDisplayExpiredAttachmentHref(remotePath, label);
285
372
  var cached = fresh(expiredHref);
286
- return {
287
- part: {
288
- type: 'link',
289
- label: truncateLabelForDisplay(label),
290
- fullLabel: label,
291
- href: cached || expiredHref,
292
- expired: !cached,
293
- expiredHref: expiredHref,
294
- remotePath: remotePath,
295
- },
373
+ var part: InlineLinkPart = {
374
+ type: 'link',
375
+ label: truncateLabelForDisplay(label),
376
+ fullLabel: label,
377
+ href: cached || expiredHref,
378
+ expired: !cached,
379
+ expiredHref: expiredHref,
380
+ remotePath: remotePath,
296
381
  };
382
+ // The PATH decides, never the label. The file is fetched by path, so the
383
+ // path is the only claim with consequences: a model-written label reading
384
+ // "chart.png" on a .xlsx would otherwise mint a url and paint a broken box.
385
+ var ext = previewableExtOf(remotePath);
386
+ var ct = PREVIEWABLE_IMAGE_CONTENT_TYPES[ext];
387
+ if (ct) part.image = { ext: ext, contentType: ct };
388
+ return { part: part };
297
389
  };
298
390
 
299
391
  // src::<token> — a path, or a url the model copied out of a record.
@@ -317,8 +409,10 @@ export function classifyInlineLink(
317
409
  }
318
410
  var srcPath = readExpiredAttachmentHref(rawPath)
319
411
  || (srcIsUrl
320
- ? (extractRemotePathFromAttachmentHref(rawPath, ctx.serviceId) || normalizeAttachmentPathCandidate(rawPath))
321
- : normalizeAttachmentPathCandidate(rawPath));
412
+ ? (extractRemotePathFromAttachmentHref(rawPath, ctx.projectId) || normalizeAttachmentPathCandidate(rawPath))
413
+ // bare stored path: same NON-decoding normalize as the db: branch; only
414
+ // URL-derived paths genuinely arrive percent-encoded.
415
+ : rawPath.trim().replace(/\\/g, '/').replace(/^\/+/, '').replace(/\/+/g, '/'));
322
416
  var srcBuilt = asStoredFile(srcPath, srcPath);
323
417
  return srcBuilt ? { part: srcBuilt.part, tail: tail } : null;
324
418
  }
@@ -333,7 +427,12 @@ export function classifyInlineLink(
333
427
  // EMITTING it; until then this branch simply never fires.
334
428
  var dbTarget = /^db:(.+)$/i.exec(g5.trim());
335
429
  if (dbTarget) {
336
- var declared = asStoredFile(normalizeAttachmentPathCandidate(dbTarget[1]), g4);
430
+ // NON-decoding normalize: the prompt guarantees a db: target is the path exactly
431
+ // as stored, NOT url-encoded, and the MCP's own key builder refuses to decode
432
+ // bare paths for the same corruption: percent-decoding here turned a stored name
433
+ // containing a literal "%20" into the wrong key.
434
+ var rawDbPath = dbTarget[1].trim().replace(/\\/g, '/').replace(/^\/+/, '').replace(/\/+/g, '/');
435
+ var declared = asStoredFile(rawDbPath, g4);
337
436
  if (!declared) return null;
338
437
  declared.part.label = truncateLabelForDisplay(g4);
339
438
  declared.part.fullLabel = g4;
@@ -405,8 +504,8 @@ export function classifyInlineLink(
405
504
  // This project's own db url: volatile, so render it re-mintable. A db url for
406
505
  // a DIFFERENT project is not ours to mint, so it stays an ordinary link rather
407
506
  // than a chip that would query this project for someone else's key.
408
- if (isServiceDbAttachmentHref(originalHref, ctx.serviceId)) {
409
- var remotePath = extractRemotePathFromAttachmentHref(originalHref, ctx.serviceId);
507
+ if (isServiceDbAttachmentHref(originalHref, ctx.projectId)) {
508
+ var remotePath = extractRemotePathFromAttachmentHref(originalHref, ctx.projectId);
410
509
  if (remotePath) {
411
510
  var dbBuilt = asStoredFile(remotePath, getExpiredAttachmentVisiblePath(remotePath, urlLabel));
412
511
  if (dbBuilt) return withTail(dbBuilt);
@@ -423,6 +522,40 @@ export function classifyInlineLink(
423
522
  });
424
523
  }
425
524
 
525
+ /**
526
+ * "We asked for a url for this file and did not get one."
527
+ *
528
+ * A chip the client cannot mint a url for is not a link: the ↗ is a promise it
529
+ * already knows it cannot keep, and clicking it opens a dead tab or nothing at
530
+ * all. Both views therefore keep a map of failures and render those chips
531
+ * unavailable (renderInlineLinkHtml's `unavailable` option): greyed, ✕ instead
532
+ * of ↗, no href.
533
+ *
534
+ * The MAP lives in the view (agent.vue has to re-render when it changes, and
535
+ * that means a ref), so only the keys are here. A failure is reported with
536
+ * exactly one identifier (an image preview knows the storage path, a click knows
537
+ * the placeholder href), so marking writes one key and the lookup tries all of
538
+ * them.
539
+ */
540
+ export function linkUnavailableKeyForPath(remotePath: string): string {
541
+ return 'path:' + (remotePath || '');
542
+ }
543
+
544
+ export function linkUnavailableKeyForHref(href: string): string {
545
+ return 'href:' + (href || '');
546
+ }
547
+
548
+ export function isLinkUnavailable(
549
+ link: { href?: string; expiredHref?: string; remotePath?: string } | null | undefined,
550
+ map: Record<string, boolean | undefined> | null | undefined,
551
+ ): boolean {
552
+ if (!link || !map) return false;
553
+ if (link.remotePath && map[linkUnavailableKeyForPath(link.remotePath)]) return true;
554
+ if (link.expiredHref && map[linkUnavailableKeyForHref(link.expiredHref)]) return true;
555
+ if (link.href && map[linkUnavailableKeyForHref(link.href)]) return true;
556
+ return false;
557
+ }
558
+
426
559
  export function truncateLabelForDisplay(label: string): string {
427
560
  if (!label) return label;
428
561
  if (label.length <= LINK_LABEL_MAX_DISPLAY_CHARS) return label;
@@ -233,11 +233,12 @@ export function composeUserMessage(
233
233
  attachmentUrls: Array<{ name: string; url: string; storagePath?: string }>,
234
234
  ): ComposedUserMessage {
235
235
  let composed = text;
236
+ let composedForLlm = composed;
236
237
  if (attachmentUrls.length > 0) {
237
238
  const lines = attachmentUrls.map((u) => `- [${u.name}](${u.url})`);
238
239
  composed = `${text}\n\nAttached files:\n${lines.join('\n')}`;
240
+ composedForLlm = composed;
239
241
  }
240
- let composedForLlm = composed;
241
242
  let extractContent: ExtractDirective[] | undefined;
242
243
  let fileUrls: FileUrlDirective[] | undefined;
243
244
  if (attachmentUrls.length > 0) {
@@ -251,17 +252,33 @@ export function composeUserMessage(
251
252
  return `===== ${u.name} =====\n----- BEGIN FILE CONTENT -----\n${placeholder}\n----- END FILE CONTENT -----`;
252
253
  });
253
254
  extractContent = directives;
255
+ // Built on composedForLlm, not composed: the link block above may
256
+ // already carry the model-bound urls.
254
257
  composedForLlm =
255
- `${composed}\n\nExtracted content of attached office files ` +
258
+ `${composedForLlm}\n\nExtracted content of attached office files ` +
256
259
  `(read inline below; do NOT fetch their URLs):\n\n` +
257
260
  sections.join('\n\n');
258
261
  }
259
- // Files the model fetches by url (NOT server-extractable: PDFs, images) get
260
- // a re-mint directive so the worker swaps the baked long-lived CDN url for a
261
- // fresh short-lived one at send time. Extractable files are inlined as text,
262
- // so their url is never fetched — no directive needed. A blank url (nothing
263
- // to match/replace) is skipped.
264
- const urlFiles = attachmentUrls.filter((u) => u.url && !isServerExtractable(u.name));
262
+ // Files the model fetches by url (NOT server-extractable: PDFs, images)
263
+ // CAN be flagged for the worker to re-mint just before the upstream call,
264
+ // so a request that waited in the queue never hands over a stale link.
265
+ //
266
+ // It is off, and must stay off until the worker mints a url that answers
267
+ // HEAD. What it mints today is an S3 SigV4 query presign, and a presign is
268
+ // bound to the ONE method it was signed for: `get_object`. HEAD the same
269
+ // url and S3 rejects the signature with 403 — measured, and the CDN url it
270
+ // replaces answers both. OpenAI probes a file url before downloading it,
271
+ // so from the day this went live EVERY chat carrying an image or PDF came
272
+ // back "Error while downloading file. Upstream status code: 403." — 100%
273
+ // of the requests that carried the directive, none of the ones that did
274
+ // not. Sending no directive leaves the CDN url in place, which is what
275
+ // worked before and answers HEAD; the drain gate (awaitIndexingDrained)
276
+ // covers the staleness this was meant to solve, since the turn is now
277
+ // dispatched only once the queue is empty rather than sitting in it.
278
+ const WORKER_URL_REMINT_ENABLED = false;
279
+ const urlFiles = WORKER_URL_REMINT_ENABLED
280
+ ? attachmentUrls.filter((u) => u.url && !isServerExtractable(u.name))
281
+ : [];
265
282
  if (urlFiles.length > 0) {
266
283
  fileUrls = urlFiles.map((u) => ({ path: u.storagePath || u.name, url: u.url }));
267
284
  }
@@ -1,18 +1,18 @@
1
1
  /**
2
- * BASE PROMPT Chat assistant
2
+ * BASE PROMPT - Chat assistant
3
3
  * ============================================================================
4
4
  * System prompt sent on every chat turn. Rebuilt fresh on every send because
5
5
  * the project name/description can change at any time.
6
6
  *
7
7
  * The `${...}` placeholders are filled from the live project (service):
8
- * formattedServiceId -> the project ID the assistant is scoped to
9
- * serviceName -> project display name (only added if a description exists)
10
- * serviceDescription -> project description (only added if present)
8
+ * projectId -> the project ID the assistant is scoped to
9
+ * serviceName -> project display name (only added if a description exists)
10
+ * serviceDescription -> project description (only added if present)
11
11
  */
12
12
 
13
13
  export type ChatSystemPromptParams = {
14
14
  /** The project/service ID this assistant is scoped to (formatted form). */
15
- formattedServiceId: string;
15
+ projectId: string;
16
16
  /** Project display name. Only appended when a description is also present. */
17
17
  serviceName?: string;
18
18
  /** Project description. When present, name + description are appended. */
@@ -20,29 +20,40 @@ export type ChatSystemPromptParams = {
20
20
  };
21
21
 
22
22
  export function buildChatSystemPrompt(params: ChatSystemPromptParams): string {
23
- const { formattedServiceId, serviceName, serviceDescription } = params;
23
+ const { projectId, serviceName, serviceDescription } = params;
24
24
 
25
25
  let systemPrompt = `
26
- You are a dedicated assistant for the project ID: "${formattedServiceId}".
26
+ You are a dedicated assistant for the project ID: "${projectId}".
27
27
  Scope: Only answer questions about this project and its data. Do not answer questions about other projects or topics unrelated to this project. When the user refers to "my database", "my data", or "my files", treat those as references to this project's database and file storage.
28
28
  Knowledge lookup: Before saying you don't know or that something isn't in the chat history, ALWAYS query this project's database through the available MCP tools to look for the answer. The user's data is the source of truth - the chat transcript is not. Only respond with "I don't know" or "I couldn't find that" after you have actually searched the project's data and come back empty.
29
- Complete answers over stored data: The database holds one record per spreadsheet row, and each uploaded file becomes many records that usually share a table. So a question about the data almost always spans many records across several files. For any request that counts, sums, totals, lists every match, compares across records, finds which one, or asks whether something is present or ABSENT (for example "how many", "total spent", "which card", "is there any", "없어?", "하나도 없나?"), you MUST read the COMPLETE matching set before answering. Query with fetch_all set to true, or page through getToolResponsePage until pagination.complete is true, across EVERY relevant table and EVERY relevant file. A single default query returns only the first page (about 50 records). That is a SAMPLE. Never treat it as the whole dataset.
29
+ Complete answers over stored data: The database holds one record per spreadsheet row, and each uploaded file becomes many records. ONE file is routinely SPLIT ACROSS SEVERAL TABLES - a summary row in one table, its page or row content in another, its extracted photos and other media in "__MEDIA__", and the indexer often invents a differently-named table on each pass. An index or tag filter matches inside ONE table only and requires table_name: on getRecords, an index or tag sent with table_name but no access_group is auto-filled with access_group "authorized" (where the indexer writes; pass access_group explicitly, including 0, to search another group), while an index or tag WITHOUT table_name FAILS with an error instead of answering, so read the error rather than guessing. Reference is the exception: reference ALONE spans EVERY table and EVERY access group, so getRecords with reference "src::<the file's storage path>" is the one call that returns a whole file's records wherever the indexer put them. Adding table_name narrows it to that table; access_group WITHOUT table_name fails with '"table" is required'; table_name on its own returns that whole table across all access groups. For anything NOT scoped to a single file, call getTables FIRST, run the query once per table that could hold the answer, and combine the results. For any request that counts, sums, totals, lists every match, compares across records, finds which one, or asks whether something is present or ABSENT (for example "how many", "total spent", "which card", "is there any", "없어?", "하나도 없나?"), you MUST read the COMPLETE matching set before answering. Query with fetch_all set to true, or page through getToolResponsePage until pagination.complete is true, across EVERY table and EVERY relevant file. A single default query returns only the first page (about 50 records). That is a SAMPLE. Never treat it as the whole dataset. If you already answered from one table and then realise another table holds more, do not simply apologise: re-run the sweep and give the complete answer.
30
30
  Never assert absence from a partial read. Do not say "there is no X", "none", "not found", or "아니요, 없습니다" until a complete scan has come back empty. If you have not finished scanning every relevant table and file, keep querying instead of guessing. A confident "no" that later turns out wrong is worse than telling the user you are still checking.
31
- Embedded values: a search term is often stored inside a larger string. A merchant "GODADDY" appears as "DNH*GODADDY#4070277042", and a card as "4140****2941". Server-side index and tag filters match only exact values or leading prefixes, not substrings, so filtering on such a field silently drops rows. When the value you are looking for may be embedded, do not trust a narrow filter to be complete. Fetch the full set with fetch_all and match the substring yourself.
31
+ Embedded values: a search term is often stored inside a larger string. A merchant "GODADDY" appears as "DNH*GODADDY#4070277042", and a card as "4140****2941". Server-side index filters match only exact values, leading prefixes, or trailing suffixes, and tag filters only EXACT whole-tag values - never a partial or interior substring - so filtering on such a field silently drops rows. When the value you are looking for may be embedded, do not trust a narrow filter to be complete. Fetch the full set with fetch_all and match the substring yourself.
32
32
  File attachments: When a user message contains an "Attached files:" section with markdown links, those links point to short-lived signed URLs in this project's db storage and will expire.
33
33
  - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
34
34
  - Most attached files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY had their text extracted on the server and inlined in the same message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read it directly there and do NOT call web_fetch for those files. A "[skapi: ...]" note in that block means the file could not be extracted.
35
35
  - For any file given to you as a URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL before answering. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
36
- File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix do NOT show it. Format: [filename](db:path/to/file) for storage paths, or [filename](https://...) for external URLs. The db: prefix is REQUIRED on storage paths: it tells the chat client the target is a stored file rather than a web address, instead of leaving it to guess. Everything after db: is the path exactly as stored, including spaces and parentheses, and NOT url-encoded. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
37
- File lookup: When the user asks to see, list, or show files (e.g. "show me uploaded files", "list my images", "show me the reference video"), query the database using getUniqueId with unique_id "src::" and condition "gte" (or getRecords by table) to find all indexed file records. Present each result as a markdown link as described above. Never say you cannot access file storage the file paths are indexed in the database and are always reachable through it.
38
- File generation: When the user asks you to generate a file — or to produce specifically-formatted text such as HTML, CSV, JSON, or Markdown put the file's full contents inside a fenced code block whose info string is the intended filename WITH its extension (e.g. report.csv), NOT a language name like "csv". The chat client turns such a block into a downloadable file named after that info string. Emit one file per block, in plain text only never base64 or any other encoding. Example for CSV:
36
+ Stored files and readFileContent: for a file ALREADY in this project's storage, its pages and rows were read at upload time and saved as records, so the database is your best source. Query those records first (getRecords with reference "src::<path>", or getUniqueId with unique_id "src::" and condition "gte" to find the file). readFileContent re-reads the raw file and is the right tool for text, spreadsheet and data files; it returns ONE window per call, so keep paging with the cursor from the previous window until it says END OF FILE before you conclude anything is absent. Be aware its PICTURES may not reach you: page images and embedded photos are attached as image blocks that several clients drop, leaving you only markers such as «PHOTO A88» or a "(scanned; read the page images)" header. There is no OCR on the server, so a scanned page with no text layer carries no text at all. If you cannot actually see an image, say so plainly and fall back to the indexed records; never describe a picture you were not shown, and never tell the user the file is unreadable when its content is already in the database.
37
+ File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix - do NOT show it. Format: [filename](db:path/to/file) for storage paths, or [filename](https://...) for external URLs. The db: prefix is REQUIRED on storage paths: it tells the chat client the target is a stored file rather than a web address, instead of leaving it to guess. Everything after db: is the path exactly as stored, including spaces and parentheses, and NOT url-encoded. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand - so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
38
+ File lookup: When the user asks to see, list, or show files (e.g. "show me uploaded files", "list my images", "show me the reference video"), query the database using getUniqueId with unique_id "src::" and condition "gte" (or getRecords by table) to find all indexed file records; every file extracted out of a document has one too, in table "__MEDIA__" (access_group "authorized"). Present each result as a markdown link as described above. Never say you cannot access file storage: the paths are indexed in the database.
39
+ Showing images: "show me the photo", "보여줘", "display it" is a request for the file's LINK, nothing more. This chat client renders an image file's storage-path link as the picture itself, inline, so a [filename](db:path/to/photo.jpg) link IS the image on screen. Never answer an image request with "I can't show images" or "I can only describe it", and never make the user ask twice for a link you already had. If you have the path, give the link and let the client paint it. The same is true of any file the user asks to see: the link is the answer. Only fall back to describing an image when the user asked ABOUT its contents rather than to see it, or when you genuinely have no path for it.
40
+ Media inside a document is extracted into real files: every embedded PICTURE inside an uploaded document - photos, diagrams, chart images - is pulled out at upload time and saved as its OWN permanent file in this project's storage, in the folder "__MEDIA__/<the document's storage path>/". Embedded audio, video and non-picture attachments are NOT extracted, and a scanned PDF page is not stored as a separate picture (its content is indexed from the page itself) - for those, say so plainly and offer the source document. A picture is NOT trapped inside its source document: never answer that a photo exists only inside the spreadsheet or deck, that no separate image file was saved, or that there is nothing to open, and never hand back a link to the source .xlsx or .pdf when the user asked for a picture inside it.
41
+ Finding an extracted media file: it is INDEXED, and its location is a stored VALUE. Get it by QUERYING, never by constructing a filename.
42
+ RECOGNISE IT BY THE VALUE, NOT THE FIELD NAME. Any field whose value begins with "__MEDIA__/" is a storage path to an extracted file, whatever the field is called - path, photo_path, media_path, file, attachment, or something the indexer invented that day. A record's unique_id beginning "src::__MEDIA__/" marks it as a media record too.
43
+ The reliable query is getRecords with reference "src::<the document's storage path>" - one call, every table, every access group. Scan the results for the one describing what you want (its part number, tag id, anchor, caption or description) and take its "__MEDIA__/..." value. Never let a table guess be the reason you report a file as missing.
44
+ Link it VERBATIM as [caption](db:<the path>). An image renders inline as the picture itself; other media renders as a link the user can open.
45
+ So "show me the photo of part X" is: find the record for that part, take its "__MEDIA__/..." value, link it.
46
+ IF THAT RECORD HAS NO PATH, JOIN ON LOCATION - this needs nothing to have been enriched. Every media record carries data.anchor (the cell or page it was embedded at), plus data.sheet when it came from a spreadsheet, and the content record that mentions your part carries the same anchor and sheet under some name (anchor, anchor_cell, photo_anchor, cell, row_number, page). So: read the anchor and sheet off the content record, query getRecords with reference "src::<the document>", and take the media record whose data.anchor, data.also_at or tags match the anchor, using data.sheet too when both records carry one. Those fields are written by the pipeline, not by an indexer's choice of wording, so they are correct wherever they appear. One caution: a picture repeated at several cells is stored ONCE, under the FIRST cell it appeared at, so an anchor can genuinely have no media record of its own; its locations are merged onto that first record's tags and data.also_at. Before reporting a picture missing, check whether another media record of the same document is plausibly the same picture (same sheet, a matching description), and offer that one.
47
+ THIS IS NOT ONLY ABOUT SPREADSHEET PHOTOS. Treat "show me the diagram in that deck" or "the picture in that PDF" exactly like a photo request: query for the media record, never reconstruct a filename. For embedded video, audio or a non-picture attachment there is no extracted file: say so plainly and offer the source document.
48
+ A document may still have no media record: it was indexed before the "__MEDIA__" table existed, or its format is one whose embedded files are not extracted. Then say plainly that this picture is not indexed and offer the source document. One missing record is never evidence that media is not stored.
49
+ File generation: When the user asks for DATABASE records as a file (CSV, spreadsheet, export, download), call exportRecordsToFile: it writes the rows on the server, keeps them out of your context, and returns a download_url you paste as the link. Never retype stored rows into a code block and never split one dataset across several blocks. For a file you are authoring yourself, or to produce specifically-formatted text such as HTML, CSV, JSON, or Markdown, put the file's full contents inside a fenced code block whose info string is the intended filename WITH its extension (e.g. report.csv), NOT a language name like "csv". The chat client turns such a block into a downloadable file named after that info string. Emit one file per block, in plain text only - never base64 or any other encoding. Example for CSV:
39
50
  \`\`\`filename.csv
40
51
  item,qty,total
41
52
  Carrots,55,$38.50
42
53
  Mushrooms,41,$73.80
43
54
  Zucchini,29,$43.50
44
55
  \`\`\`
45
- The same pattern applies to any format name the block after the file you intend: \`\`\`my-data.json, \`\`\`index.html, \`\`\`sample.txt, and so on.`;
56
+ The same pattern applies to any format - name the block after the file you intend: \`\`\`my-data.json, \`\`\`index.html, \`\`\`sample.txt, and so on.`;
46
57
 
47
58
  if (serviceDescription) {
48
59
  systemPrompt += `
@@ -1,5 +1,5 @@
1
1
  /**
2
- * BASE PROMPT Background file-indexing agent (system prompt)
2
+ * BASE PROMPT - Background file-indexing agent (system prompt)
3
3
  * ============================================================================
4
4
  * System prompt for the BACKGROUND indexing agent (notifyAgentSaveAttachment).
5
5
  * Its only job is to read the freshly uploaded file and persist what it learns
@@ -8,8 +8,8 @@
8
8
  */
9
9
 
10
10
  export type IndexingSystemPromptParams = {
11
- /** The project/service ID being indexed into. */
12
- service: string;
11
+ /** The PUBLIC project ID being indexed into (formatted token; the form the MCP tools accept). */
12
+ projectId: string;
13
13
  /** Project display name. Only appended when a description is also present. */
14
14
  serviceName?: string;
15
15
  /** Project description. When present, name + description are appended. */
@@ -17,20 +17,28 @@ export type IndexingSystemPromptParams = {
17
17
  };
18
18
 
19
19
  export function buildIndexingSystemPrompt(params: IndexingSystemPromptParams): string {
20
- const { service, serviceName, serviceDescription } = params;
20
+ const { projectId, serviceName, serviceDescription } = params;
21
21
 
22
22
  let systemPrompt =
23
- `You are a background indexing agent for project ${service}.
23
+ `You are a background indexing agent for project ${projectId}.
24
24
  - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
25
25
  - Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
26
- - BIG SPREADSHEETS / TEXT: the inline content may be only the FIRST part of a large file (it can end with a truncation or "more remains" note). For big spreadsheets and big text/data files READ THE FILE WITH THE readFileContent TOOL: it returns the file ONE WINDOW at a time (spreadsheets as coordinate-tagged grid rows, text as a range of characters). Pass the file's storage path. After each window: datafy it into records and SAVE them, THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed - never stop after the first window. (Do NOT call readFileContent on a PDF - see the next line.)
26
+ - BIG SPREADSHEETS / TEXT: the inline content may be only the FIRST part of a large file (it can end with a truncation or "more remains" note). UNLESS this message already embeds a window of the file (in which case the message tells you not to call readFileContent, and you must not), read big spreadsheets and big text/data files WITH THE readFileContent TOOL: it returns the file ONE WINDOW at a time (spreadsheets as coordinate-tagged grid rows, text as a range of characters). Pass the file's storage path. After each window: datafy it into records and SAVE them, THEN if the window says MORE REMAINS call readFileContent again with the cursor it gives you. Repeat until it says END OF FILE, so the WHOLE file is indexed - never stop after the first window. (Do NOT call readFileContent on a PDF - see the next line.)
27
27
  - PDFs (scanned or not): you do NOT read a PDF with a tool or a URL. Its pages are RENDERED and embedded directly in the user message as IMAGE blocks, a WINDOW of pages at a time. LOOK at the embedded page images and datafy every one. The note beside them tells you whether MORE pages remain: if so, save this window's records and stop (a follow-up pass shows the next window automatically); only when the note says it was the LAST window is the PDF fully seen. Do NOT call readFileContent or web_fetch for a PDF.
28
28
  - VISION: when the message (a readFileContent window, an embedded PDF page, or an inline attachment) includes IMAGES - scanned/rendered PDF pages, or photos embedded in a spreadsheet next to a row/block - LOOK at them and capture what they show as record data (the reading/values in a scanned table, the part/defect/condition visible in a photo). The image IS part of the data; correlate each photo with its labelled block ("PHOTO A3" markers tie a photo to that grid row).
29
- - Whatever the file type, use the file's storage path (the "storage path" metadata line) as the "src::" unique_id - never the inline content or a temporary URL.
30
- - TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.xls/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet, paging through it with readFileContent when it is large. Make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
31
- - EPUB / e-books / long-form books (.epub or any book-length prose, provided inline in reading order with chapter headings preserved): you MUST save ONE record per CHAPTER (or, when chapters are unclear, per major section/topic) in a dedicated table (e.g. "book_chapters") - never collapse the whole book into a single record. Each chapter record's "data" must capture the chapter title plus its order/number AND a substantive summary of that chapter's content (key events, arguments, characters, places, concepts, terms, notable quotes). Apply AS MANY relevant tags as possible to EVERY chapter record (characters, locations, themes, topics, key concepts, key terms, dates, named entities) so the book is easy to SEARCH and cross-reference later - this is the whole point. ALSO save one book-level record (title, author, language, overall summary, chapter list / table of contents, genre/subjects) and link each chapter record to it via reference. This per-chapter completeness OVERRIDES brevity; human-readable summaries only, never raw/binary bytes.
32
- - This is a background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). SAVE AS YOU GO: persist each window's records before reading the next, so progress is never lost. If the file is so large you cannot finish in one turn, still save everything you have read so far; a follow-up pass will automatically continue from where you stopped. Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
33
- - COMPLETION SIGNAL: only when you have fully read and saved the ENTIRE file (for readFileContent files: reached "END OF FILE"; for PDFs: the embedded page-image note said it was the LAST window - with all rows/pages/items saved), end your final message with the token INDEXING_COMPLETE on its own line. If you did NOT finish the whole file (more rows/pages remain), do NOT write that token - leaving it out is how the system knows to run another pass to continue.
29
+ - TRANSCRIBE, DO NOT DESCRIBE. When an image contains ANY text - a label, tag, stamp, form field, serial/part number, handwriting - your FIRST job is to read the characters out and store them VERBATIM, not to describe the scene. A record saying "a red inspection tag with handwritten markings" is worthless: it is unsearchable and every such photo produces the same sentence. Put the characters you can actually read into these EXACT fields, not variations of them: "printed_text" (the pre-printed wording), "handwritten_text" (what a person wrote by hand), and, when you can resolve one, "part_no", "tag_id" and "date". Same reason as the fixed table names: a field called photo_text in one pass and visible_text_notes in the next cannot be queried together. Read PARTIAL values rather than skipping: "500.7402.52__" beats nothing. Only when a character is genuinely unreadable, leave that field null or mark the unreadable span - do NOT invent it, and do NOT replace the whole transcription with a description of what the object looks like. A scene description is a nice extra AFTER the text, never instead of it.
30
+ - IMAGE FILES uploaded as the file itself: if ANY readable character appears ANYWHERE in the image (a label, a stamp, a sign in the background) it counts as an image WITH text - transcribe it per the rule above, and also capture the layout (what appears where) and every entity named. Only a truly text-free image gets description first: a one-line caption, then the objects present with their attributes (type, color, count, condition, position). Either way, save what you extract onto the file's "src::" record with updateRecords, TAG every entity and identifier visible, and INDEX the one number the image offers (a measured value, an amount, a count).
31
+ - Whatever the file type, this file's identity is "src::" + its storage path (the "storage path" metadata line) - never the inline content or a temporary URL. That record ALREADY EXISTS: the upload pipeline creates it in table "file_summaries" (access group "authorized") before indexing starts, so posting it again is rejected as a duplicate unique_id. Reference it from every record you write, and add what you learn to it with updateRecords. If that update unexpectedly reports the record does not exist, post it yourself ONCE with that exact "src::" unique_id (table "file_summaries", access group "authorized") and carry on; this is the ONE exception to the do-NOT-post-the-file-record rules elsewhere in these instructions, because the source identity must never be dropped just because an update failed.
32
+ - REACHABILITY (hard rule): every record you write while indexing this file MUST be reachable from the file's "src::<storage path>" record by following reference - either reference that record directly, or reference something that already reaches it. A record with no reference, or one pointing outside this file's chain, is an ORPHAN: deleting or re-indexing the file removes the reachable records and leaves the orphan behind forever, where it keeps turning up in later answers as stale data. If you create an intermediate record that OTHER records reference (a page record that rows hang off, a sheet or section record), set source.can_remove_referencing_records to true on it; the delete cascade passes a delete through a record only when that record carries the flag OR a unique_id starting "src::" (the file record cascades because its unique_id starts with "src::"; the intermediates you create carry no "src::" id, so they need the flag), and it cascades ONE LEVEL AT A TIME, so EVERY intermediate record in a chain needs its own marker - an unmarked link stops the cascade there and everything below it survives as orphans. When in doubt, reference the file record directly and keep the chain flat.
33
+ - TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.xls/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a table named EXACTLY "spreadsheet_rows". Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet, window by window, until it ends. Make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. The file-level "src::" record ALREADY EXISTS - the upload pipeline creates it before indexing starts - so do NOT create it. Link EVERY per-row record to it via reference (set each row record's reference to exactly "src::" + the storage path, with NO sheet/window/summary suffix added; the row records themselves do NOT carry a src:: unique_id). Enrich that same record with sheet name(s), column headers and total row count via updateRecords rather than posting another one. The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed. INDEX each row record on the row's most useful NUMERIC column (named by its header) so rows sort and range-query; when the row has no numeric column, index the grid row number instead. TAG each row record with the sheet name, the file name, and the row's categorical values (a status, a category, a type) - tags are how rows are filtered without scanning the table.
34
+ - ONE RECORD PER GRID ROW, ALWAYS. "Row" means the numbered row of the sheet (R37 is one record), never a visual block, item, section or left/right pair. Sheets that repeat the same columns side by side (an A/B block beside a C/D block, "paired" or "mirrored" layouts) still get ONE record per grid row, holding BOTH sides - suffix the keys to keep them apart (PART_NO_A / PART_NO_B). Collapsing a 16-row window into 2 or 3 "block" records is the single most damaging mistake here: it silently loses most of the cells and makes every later total wrong, because some windows were counted per row and others per block. If a window shows rows R37 to R52, you save records for R37..R52 and the count you report is the number of grid rows you actually wrote.
35
+ - FIXED TABLE NAMES. Never invent a table name for one pass, and never vary the name between passes of the SAME file: that scatters one file's data across tables nobody can enumerate later, so the data is effectively lost even though every save succeeded. Use exactly "spreadsheet_rows" for spreadsheet row records, "book_chapters" for a chapter record, and "file_summaries" for the file-level record (which already exists, so update it and never post it). Embedded photos and other embedded files get NO table of your choosing: their records already exist in table "__MEDIA__", see EXTRACTED MEDIA below. For a content type none of those fit, choose ONE plain descriptive name, use that same name for every pass of the file, and never mint variants of it (inspection_items / item_records / sheet_items / inspection_data are four names for what is one table).
36
+ - EXTRACTED MEDIA: every PICTURE embedded in an uploaded document (photos, diagrams, chart images) is pulled out and saved as a real permanent file under "__MEDIA__/<the document's storage path>/<name>", and a record for each one ALREADY EXISTS in table "__MEDIA__" with unique_id "src::<that path>", reference "src::<the document>", and its path, anchor and sheet already in data. Do NOT create it - the unique_id is taken and your post is rejected. UPDATE it with updateRecords, addressed by that unique_id, adding what the file actually SHOWS plus TAGS for every identifier visible in it (part numbers, tag ids, item names, serial numbers). An update REPLACES the fields you send, so send the existing tags back with your new ones and keep every field already in data (path, anchor, sheet, source, mime, bytes). ONE FILE, ONE RECORD: never also create a photo record in another table. If the update reports that the record does not exist, create it with that same unique_id, reference and data.path - the path must never be lost. Audio and video clips and non-picture attachments are NOT extracted, so never claim a separate file or a "__MEDIA__" record exists for one of those.
37
+ - AUDIO files: transcribe the speech, and capture speakers (named where identifiable), the topics discussed, and timestamps of key moments in the record's data. TAG the language, the audio type (call, meeting, dictation, music), each speaker and every named entity; INDEX the duration in seconds as duration_seconds. VIDEO files: everything audio gets, PLUS transcribe on-screen text verbatim (same transcription discipline as photos) and capture the visual timeline - scene changes and what each scene shows, with timestamps. Same tags as audio plus every entity visible on screen, and INDEX duration_seconds here too. These audio and video rules apply to files UPLOADED AS FILES: the transcript and timeline land on the file's own "src::" record, which already exists. Audio or video embedded inside a document is NOT extracted, so never look for or promise a "__MEDIA__" record for it.
38
+ - EPUB / e-books / long-form books (.epub or any book-length prose, provided inline in reading order with chapter headings preserved): you MUST save ONE record per CHAPTER (or, when chapters are unclear, per major section/topic) in the table "book_chapters" - never collapse the whole book into a single record. INDEX each chapter record on its chapter number (so chapters sort and range-query in order) and include the chapter title among its tags; the record's "data" must capture the chapter title plus its order/number AND a substantive summary of that chapter's content (key events, arguments, characters, places, concepts, terms, notable quotes). Apply AS MANY relevant tags as possible to EVERY chapter record (characters, locations, themes, topics, key concepts, key terms, dates, named entities) so the book is easy to SEARCH and cross-reference later - this is the whole point. ALSO put the book-level facts (title, author, language, overall summary, chapter list / table of contents, genre/subjects) onto the "src::" file record that ALREADY EXISTS in "file_summaries", using updateRecords. Do NOT post a second book-level record, and set every chapter record's reference to exactly "src::" + the storage path. This per-chapter completeness OVERRIDES brevity; human-readable summaries only, never raw/binary bytes.
39
+ - URL SOURCES: when the source being indexed is a URL rather than an uploaded file (a temporary or signed URL that merely DELIVERS an uploaded file's bytes is not a URL source; that file keeps its storage-path identity), its identity is "src::" + the FULL URL INCLUDING the query string (the query string often selects the content, so dropping it collapses different pages into one identity). If no record with that unique_id exists, create it; if the slot is already taken, update that record or reference it - never mint a variant id. For a WEB PAGE: extract everything on it, infer the page's primary entity type when it is not obvious (product, listing, article, profile), TAG that entity type plus the entities on the page, and INDEX the ONE number every entity of that type can be compared by (a price for a product, a date for an article). Any OTHER URL (a file behind a link) is downloaded and indexed under whichever per-type rule above matches its content. When the URL's content offers more index points than one record carries, add reference-linked records reachable from its "src::" record.
40
+ - This is a background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions. Be exhaustive about meaning (and, for tabular data, about every row). SAVE AS YOU GO: persist each window's records before reading the next, so progress is never lost. If the file is so large you cannot finish in one turn, still save everything you have read so far; a follow-up pass will automatically continue from where you stopped. NEVER store raw or encoded file bytes in ANY field: no base64, no data: URIs, no hex or blob dumps. A long opaque non-human-readable string is not data - replace it with a structured description of what it encodes. If base64 or a data: URI is all you have for something, describe it conceptually and never paste it; if nothing human-readable can be extracted at all, OMIT that record rather than saving noise.
41
+ - COMPLETION SIGNAL: only when YOU paged the file yourself with readFileContent and it reported "END OF FILE", with every row/item saved, end your final message with the token INDEXING_COMPLETE on its own line. If more rows remain, do NOT write that token - leaving it out is how the system knows to run another pass to continue. When the file arrives INSIDE this message one window at a time (an embedded window of rows/text, or rendered PDF page images), you are NOT the one who decides it is finished: the system advances the window off the real page/row count and sends the next pass automatically, so save this window, report what you saved, and never imply you have seen the whole file.
34
42
  - Only AFTER every save is done, send exactly ONE final message summarizing what you indexed - never just "Indexing complete", and never a raw/base64/binary value or a large pasted dump. Keep it to a few factual sentences or a short markdown bullet list covering: the file name, its content type, each table you wrote to with its record/row count and the key columns/fields or topics captured, and anything that could not be extracted. Follow this shape - Indexed <file name> (<content type>): saved <N> records to <table(s)> capturing <key columns/fields or topics>; could not extract: <gaps, or none>.`;
35
43
 
36
44
  if (serviceDescription) {