bunnyquery 1.12.1 → 1.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunnyquery",
-  "version": "1.12.1",
+  "version": "1.14.0",
   "description": "Embeddable BunnyQuery AI chat widget + its framework-agnostic chat engine",
   "main": "bunnyquery.js",
   "exports": {

package/src/engine/budget.ts

@@ -5,6 +5,10 @@
  * to sanitizeAttachmentLinksForHistory (which used to read a global).
  */
 import { sanitizeAttachmentLinksForHistory } from './links';
+import {
+	formatRetrievalNote, isRetrievalNote, RETRIEVAL_NOTES_MAX, RETRIEVAL_NOTES_TOKEN_BUDGET,
+	type RetrievalNote,
+} from './retrieval_memory';
 
 export var CONTEXT_WINDOW_DEFAULT: Record<string, number> = { claude: 200000, openai: 128000 };
 // Exact model ids first, then family keys (see getModelContextWindow: a suffixed
@@ -328,11 +332,45 @@ export type BoundedChatOptions = {
 	platform: string;
 	model?: string;
 	systemPrompt: string;
-	history: Array<{ role: string; content: string }>;
+	/**
+	 * The turns, oldest first, the one being sent LAST. An assistant turn may
+	 * carry `_retrieval` (see retrieval_memory.ts): the note is appended to that
+	 * turn's LLM copy for the newest RETRIEVAL_NOTES_MAX such turns, within
+	 * RETRIEVAL_NOTES_TOKEN_BUDGET. Turns without it are sent as they always were.
+	 */
+	history: Array<{ role: string; content: string; _retrieval?: RetrievalNote | null }>;
 	/** Used to strip/rewrite expired attachment links in older user turns. */
 	projectId: string;
+	/** Set false to leave every `_retrieval` note out of the request. Default on. */
+	retrievalNotes?: boolean;
 };
 
+/**
+ * Append each carried retrieval note to its assistant turn, newest first, until
+ * RETRIEVAL_NOTES_MAX notes or RETRIEVAL_NOTES_TOKEN_BUDGET estimated tokens are
+ * used. Returns new message objects for the turns it changed and the originals
+ * for the rest; the turn at `latestIndex` (the one being sent) is never touched.
+ * Pure, and exported so the bound can be tested without a session.
+ */
+export function applyRetrievalNotes<T extends { role: string; content: string; _retrieval?: RetrievalNote | null }>(
+	messages: T[], latestIndex: number,
+): { messages: T[]; notesUsed: number; noteTokens: number } {
+	var out = messages.slice();
+	var used = 0, tokens = 0;
+	for (var i = messages.length - 1; i >= 0; i--) {
+		if (used >= RETRIEVAL_NOTES_MAX) break;
+		var m = messages[i];
+		if (i === latestIndex || !m || m.role !== 'assistant' || !isRetrievalNote(m._retrieval)) continue;
+		var line = formatRetrievalNote(m._retrieval);
+		if (!line) continue;
+		var cost = estimateTextTokens(line) + 2;
+		if (tokens + cost > RETRIEVAL_NOTES_TOKEN_BUDGET) break;
+		tokens += cost; used++;
+		out[i] = Object.assign({}, m, { content: (m.content || '') + '\n\n' + line });
+	}
+	return { messages: out, notesUsed: used, noteTokens: tokens };
+}
+
 export function buildBoundedChatMessages(options: BoundedChatOptions) {
 	var contextBasedBudget = contextBasedBudgetFor(options.platform, options.model, options.projectId);
 	// Scaling used to be gated on an EXPLICIT per-project override so that a
@@ -354,6 +392,10 @@ export function buildBoundedChatMessages(options: BoundedChatOptions) {
 		Math.round(MAX_HISTORY_MESSAGES * (budgetForHistory / HISTORY_TOKEN_BUDGET)));
 	var windowed = options.history.slice(-maxHistoryMessages);
 	var latestIndex = windowed.length - 1;
+	// Retrieval memory rides on the turn it belongs to, BEFORE the token loop
+	// below, so a note is paid for out of the same history budget as the words
+	// around it and the request can never exceed the budget by the notes' size.
+	if (options.retrievalNotes !== false) windowed = applyRetrievalNotes(windowed, latestIndex).messages;
 	var trimmed = windowed.map(function (m, i) {
 		if (i === latestIndex) return m;
 		var stripped = stripFileBlocksFromHistory(m.content);

package/src/engine/citations.ts

@@ -0,0 +1,181 @@
+/**
+ * Citations and the no-source mark (reliability plan 3.5).
+ *
+ * The retrieval note (retrieval_memory.ts) already records what a turn READ:
+ * the files it queried or linked and the pages searchText found a term on.
+ * This module turns that note into what the two clients DRAW under an answer:
+ *
+ *   - a row of file chips, one per cited file (and page, when a page is
+ *     known), each opening the file the way every other stored-file chip does
+ *     (the `_expired_.url` placeholder that the client's click handler mints a
+ *     fresh url for; the page is in the label only, there is no deep link);
+ *   - a small mark when the answer used NO data tool at all, so an answer the
+ *     model wrote from its own head is visibly different from one it read.
+ *
+ * Pure data in, pure data out: no DOM, nothing reactive. Each client renders the
+ * parts with the same emitter it already uses for inline links, so a citation
+ * chip is byte for byte the chip the model would have written inline, and the
+ * same delegated click handler, the same unavailable state and the same minted
+ * url cache serve it.
+ */
+
+import type { RetrievalNote, RetrievalHit } from './retrieval_memory';
+import { isRetrievalNote } from './retrieval_memory';
+import { buildDisplayExpiredAttachmentHref, isHttpUrlLike, truncateLabelForDisplay } from './links';
+
+export interface Citation {
+	/** Storage path, exactly as the note carries it. */
+	path: string;
+	/** 1-based page, when a searchText hit named one. */
+	page?: number;
+	/** What the chip says: the file name (last path segment) plus " p.N" when a page is known. */
+	label: string;
+}
+
+/**
+ * The tools that READ project data. An answer whose turn called none of these
+ * (and no web tool) was written without looking at anything, which is what the
+ * no-source mark says. Names are the MCP tool names as the body carries them.
+ */
+export var DATA_TOOL_NAMES = [
+	'getRecords', 'queryGrid', 'readFileContent', 'searchText', 'getFileOutline',
+	'getUniqueId', 'getTables', 'getTags', 'getIndexes',
+];
+
+/** The mark's text, single-sourced so the two clients cannot word it differently. */
+export var NO_DATA_TOOL_MARK = 'No project data was read for this answer';
+
+/** The file name of a storage path: its last non-empty segment. */
+export function citationFileName(path: string): string {
+	var segs = String(path || '').split('/');
+	for (var i = segs.length - 1; i >= 0; i--) {
+		if (segs[i]) return segs[i];
+	}
+	return String(path || '');
+}
+
+function citationLabel(path: string, page?: number): string {
+	return citationFileName(path) + (typeof page === 'number' && page > 0 ? ' p.' + page : '');
+}
+
+/** A note entry that names a URL, not a stored file: nothing of ours to open. */
+function isMintablePath(path: string): boolean {
+	return !!path && !isHttpUrlLike(path);
+}
+
+/**
+ * The citations of one note, deduplicated and ordered: searchText hits first
+ * (files in the order they were first hit, pages ascending within a file), then
+ * the files the turn read or linked without a page, in the note's order. A file
+ * that has a page hit is not repeated as a page-less chip. Empty for a null note.
+ */
+export function citationsFor(note: RetrievalNote | null | undefined): Citation[] {
+	if (!isRetrievalNote(note)) return [];
+	var out: Citation[] = [];
+	var hitPaths: string[] = [];
+	var byPath: Record<string, number[]> = {};
+	var i: number;
+	for (i = 0; i < note.hits.length; i++) {
+		var h: RetrievalHit = note.hits[i];
+		if (!h || typeof h.path !== 'string' || !isMintablePath(h.path)) continue;
+		var page = typeof h.page === 'number' ? h.page : Number(h.page);
+		if (!isFinite(page) || page <= 0) continue;
+		page = Math.floor(page);
+		if (!byPath[h.path]) { byPath[h.path] = []; hitPaths.push(h.path); }
+		if (byPath[h.path].indexOf(page) === -1) byPath[h.path].push(page);
+	}
+	for (i = 0; i < hitPaths.length; i++) {
+		var p = hitPaths[i];
+		var pages = byPath[p].slice().sort(function (a, b) { return a - b; });
+		for (var j = 0; j < pages.length; j++) {
+			out.push({ path: p, page: pages[j], label: citationLabel(p, pages[j]) });
+		}
+	}
+	for (i = 0; i < note.files.length; i++) {
+		var f = note.files[i];
+		if (typeof f !== 'string' || !isMintablePath(f)) continue;
+		if (byPath[f]) continue;
+		var dup = false;
+		for (var k = 0; k < out.length; k++) {
+			if (out[k].path === f && out[k].page === undefined) { dup = true; break; }
+		}
+		if (dup) continue;
+		out.push({ path: f, label: citationLabel(f) });
+	}
+	return out;
+}
+
+/** True when `name` is one of the data-reading tools. */
+export function isDataToolName(name: string): boolean {
+	return DATA_TOOL_NAMES.indexOf(name) !== -1;
+}
+
+/**
+ * True when the turn has a note and that note shows no data tool and no web
+ * tool: the answer was written without reading anything. A null note is
+ * UNKNOWN (no body to read, a greeting, an indexing pass), never "no source".
+ */
+export function answerUsedNoDataTool(note: RetrievalNote | null | undefined): boolean {
+	if (!isRetrievalNote(note)) return false;
+	if (note.web) return false;
+	for (var i = 0; i < note.tools.length; i++) {
+		if (isDataToolName(note.tools[i])) return false;
+	}
+	return true;
+}
+
+/**
+ * The retrieval note of a SETTLED CHAT ANSWER, or null: the one place both
+ * clients decide which bubbles get chips and the mark. Not a user turn, not an
+ * indexing pass (a collapsed row, never a bubble), not an error, not a bubble
+ * still waiting for its answer. The greeting is client chrome with no note.
+ */
+export function retrievalNoteOfAnswer(msg: {
+	role?: string;
+	isError?: boolean;
+	isPending?: boolean;
+	isBackgroundTask?: boolean;
+	_streamPending?: boolean;
+	_retrieval?: RetrievalNote | null;
+} | null | undefined): RetrievalNote | null {
+	if (!msg || msg.role !== 'assistant') return null;
+	if (msg.isError || msg.isPending || msg.isBackgroundTask || msg._streamPending) return null;
+	return isRetrievalNote(msg._retrieval) ? msg._retrieval : null;
+}
+
+/** The shape both clients' link emitters accept (a RenderableInlineLink with the view's part tag). */
+export interface CitationLinkPart {
+	type: 'link';
+	label: string;
+	fullLabel: string;
+	href: string;
+	expired: boolean;
+	expiredHref: string;
+	remotePath: string;
+}
+
+/**
+ * One citation as the link part the clients already know how to draw and click:
+ * the `_expired_.url` placeholder for the file's storage path (or the fresh url
+ * the view has cached for it), the storage path as `remotePath` so the click
+ * handler and the unavailable map find it, and NO image, so a cited picture is
+ * a chip under the answer rather than a second preview of it. The display label
+ * is truncated from the front (truncateLabelForDisplay keeps the tail), so the
+ * " p.N" suffix survives a long file name.
+ */
+export function citationLinkPart(
+	citation: Citation,
+	resolveFreshHref?: (expiredHref: string) => string | undefined,
+): CitationLinkPart {
+	var expiredHref = buildDisplayExpiredAttachmentHref(citation.path, citation.label);
+	var cached = resolveFreshHref ? resolveFreshHref(expiredHref) : undefined;
+	return {
+		type: 'link',
+		label: truncateLabelForDisplay(citation.label),
+		fullLabel: citation.label,
+		href: cached || expiredHref,
+		expired: !cached,
+		expiredHref: expiredHref,
+		remotePath: citation.path,
+	};
+}

package/src/engine/config.ts

@@ -23,6 +23,7 @@
  */
 
 import { type AttachmentParser, registerAttachmentParser } from './attachment_parsers';
+import { cachedProjectSettings, loadProjectSettings, webAccessFrom } from './project_settings';
 
 /**
  * One report about a turn that is streaming, as handed to `onLiveStreamUpdate`.
@@ -237,6 +238,23 @@ export interface ChatEngineConfig {
      * unless the embedder turns it on.
      */
     liveStreamingRealtime?: boolean;
+    /**
+     * Whether a CHAT answer may reach the open web: Claude's `web_fetch` and
+     * OpenAI's `web_search` are attached to a chat request only when this
+     * resolves to true.
+     *
+     * OPTIONAL, AND UNSET IS NOT FALSE. When the host leaves it out, the engine
+     * reads the project's own setting (`web_access` in the `bq::settings`
+     * record, see project_settings.ts) and falls back to OFF, so the project
+     * owner's switch on the Project Settings page decides for every client that
+     * does not override it. A host that sets a boolean here overrides the
+     * project for its own page, both ways. Resolution: this option, then the
+     * project setting, then false.
+     *
+     * Chat turns only. An indexing pass keeps the tools it has today, since the
+     * worker hands it rendered pages and extracted text and not a URL to fetch.
+     */
+    webAccess?: boolean;
     /**
      * skapi.forwardRequestFinalize, bound to the consumer's skapi instance.
      * Stores the version of a streamed turn that history should keep (the engine
@@ -465,6 +483,32 @@ export function liveStreamingEnabled(): boolean {
     return _config?.liveStreaming === true;
 }
 
+/**
+ * The web switch for a chat turn, SYNCHRONOUSLY: the host's `webAccess` option
+ * when it is a boolean, else the project's cached `web_access` setting, else
+ * false. For a template or a prompt builder, which have nowhere to await. Reads
+ * the cache as it stands, so before the settings fetch settles it answers the
+ * default; request builders use resolveWebAccess below instead.
+ */
+export function webAccessEnabled(service?: string): boolean {
+    const opt = _config ? _config.webAccess : undefined;
+    if (typeof opt === 'boolean') return opt;
+    return webAccessFrom(service ? cachedProjectSettings(service) : null);
+}
+
+/**
+ * The same switch, awaiting the project's settings record when the host did not
+ * decide. Never rejects: a failed or impossible read (no reader injected, a
+ * signed-out visitor on a require_login project) settles as "unset", which is
+ * off. Cheap after the first call per project: the store dedupes and caches.
+ */
+export function resolveWebAccess(service?: string): Promise<boolean> {
+    const opt = _config ? _config.webAccess : undefined;
+    if (typeof opt === 'boolean') return Promise.resolve(opt);
+    if (!service) return Promise.resolve(false);
+    return loadProjectSettings(service).then(webAccessFrom, () => false);
+}
+
 /**
  * True when a streamed turn whose answer never reached its row can be READ BACK.
  *

package/src/engine/history.ts

@@ -5,6 +5,7 @@
  * view-specific display formatting. projectId is passed for link sanitization.
  */
 import { extractClaudeText, extractOpenAIText, INDEXING_COMPLETE_MARKER, EMPTY_INDEXING_REPLY, getChatHistory, bgIndexingQueueName } from './requests';
+import { extractRetrievalNote } from './retrieval_memory';
 import { isErrorResponseBody, getErrorMessage, isRetriedFailure } from './errors';
 import { streamRecoveryEnabled } from './config';
 import { sanitizeAttachmentLinksForHistory } from './links';
@@ -798,8 +799,17 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 		// cases the pass simply shows no duration, which is right: the honest answer
 		// to "how long did it take" is nothing, not the queue wait dressed up as it.
 		var executedTs = Number(item && item.executed);
+		// When skapi recorded the destination's response as having finished arriving.
+		// It is the answer's time whenever the row carries it, because `updated` is
+		// not that for a STREAMED turn: finalizing the turn stores its answer a few
+		// seconds later and moves `updated` to that moment. Absent on rows settled
+		// before the worker recorded it, on a turn that failed with no response, and
+		// under an SDK that does not project it. All three fall back to `updated`,
+		// which is exactly what this showed before.
+		var respondedTs = Number(item && item.responded);
 		var userTs = isFinite(createdTs) && createdTs > 0 ? createdTs : (isFinite(updatedTs) && updatedTs > 0 ? updatedTs : undefined);
-		var replyTs = isFinite(updatedTs) && updatedTs > 0 ? updatedTs : (isFinite(createdTs) && createdTs > 0 ? createdTs : undefined);
+		var replyTs = isFinite(respondedTs) && respondedTs > 0 ? respondedTs
+			: (isFinite(updatedTs) && updatedTs > 0 ? updatedTs : (isFinite(createdTs) && createdTs > 0 ? createdTs : undefined));
 
 		if (userText) {
 			var displayContent;
@@ -889,6 +899,14 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 			if (item._fromBgChain) okm._fromBgChain = true;
 			if (item._isBgTask) okm.isBackgroundTask = true;
 			if (isCompact) okm._compact = true;
+			// Retrieval memory, re-derived from the stored body on every load so a
+			// reload reads the same note the live settle attached. Chat turns only:
+			// an indexing pass never enters the LLM history, and a compact stub has
+			// no body to read.
+			if (!item._isBgTask && !isCompact) {
+				var note = extractRetrievalNote(response, assistantText);
+				if (note) okm._retrieval = note;
+			}
 			if (serverItemId !== undefined) okm._serverItemId = serverItemId;
 			if (replyTs !== undefined) okm._ts = replyTs;
 			// How long THIS pass took: from the moment the worker began EXECUTING it to

package/src/engine/host.ts

@@ -7,6 +7,8 @@
  * otherwise touch the DOM or a framework goes through a host hook.
  */
 
+import type { RetrievalNote } from './retrieval_memory';
+
 export interface ChatIdentity {
 	projectId: string;
 	/**
@@ -78,6 +80,11 @@ export interface IndexingFileRef {
 export interface ChatMessage {
 	role: 'user' | 'assistant';
 	content: string; // raw markdown — never HTML (the view parses for display)
+	/** Retrieval memory (retrieval_memory.ts): what this assistant turn read,
+	 *  queried or linked, taken off the provider body's tool blocks or, failing
+	 *  that, the answer's file links. Never rendered. buildBoundedChatMessages
+	 *  appends it to the turn's LLM copy so a follow-up skips discovery. */
+	_retrieval?: RetrievalNote;
 	isPending?: boolean;
 	isPendingInProcess?: boolean;
 	isPendingQueued?: boolean;

package/src/engine/index.ts

@@ -28,6 +28,12 @@ export {
 	// says no: an SDK too old to know the key drops it silently, leaving the
 	// destination streaming SSE into a buffered row that reads back empty.
 	skapiSupportsStreaming,
+	// The chat's web switch, resolved the one way both clients and the request
+	// builders agree on: the host's `webAccess` option, then the project's
+	// `web_access` setting, then off. The sync form is for a prompt builder or a
+	// template; the awaiting form is what the request builders use.
+	webAccessEnabled,
+	resolveWebAccess,
 } from './config';
 
 export {
@@ -78,6 +84,12 @@ export {
 	isCsrStatusEnvelope, csrEnvelopeError,
 } from './errors';
 export * from './budget';
+// Retrieval memory: the per-turn note of what a chat turn read, queried or
+// linked, and the bounds under which buildBoundedChatMessages carries it.
+export * from './retrieval_memory';
+// Citations and the no-source mark: what each client draws under an answer
+// out of that note, and the one gate deciding which bubbles get them.
+export * from './citations';
 // Per-format UTF-8 declaration for files offered as a download. Shared so a fenced
 // block and a server-published file open identically in Excel, Word and a browser.
 export * from './download_encoding';
@@ -269,6 +281,11 @@ export {
 	normalizeChatGreeting,
 	chatGreetingFrom,
 	projectChatGreeting,
+	// web access for chat answers (same record, its own key, off by default)
+	DEFAULT_WEB_ACCESS,
+	normalizeWebAccess,
+	webAccessFrom,
+	projectWebAccess,
 	// pure readers over a settings `data` object
 	normalizeUploadAccessGroup,
 	normalizeProjectAccessSetting,

package/src/engine/project_settings.ts

@@ -95,9 +95,39 @@ export const UPLOAD_ACCESS_OPTIONS = UPLOAD_ACCESS_GROUPS.map((value) => ({
 export type ProjectSettingsData = {
 	upload_access_group?: unknown;
 	chat_greeting?: unknown;
+	web_access?: unknown;
 	[key: string]: unknown;
 };
 
+/**
+ * Whether a CHAT answer may reach the open web (Claude `web_fetch`, OpenAI
+ * `web_search`). Same record, its own `web_access` key.
+ *
+ * OFF BY DEFAULT, and the default is the point: customers reported answers that
+ * were not in their data, and an open web tool is one of the ways such an answer
+ * gets out. A project that never opens the setting answers from its files and
+ * database alone. Only the literal `true` turns it on: this record is public and
+ * a client must never read a value it did not write as permission.
+ *
+ * Resolution order, applied by the engine where the chat request is built:
+ *   1. the host's `webAccess` init/config option, when it is a boolean;
+ *   2. this setting, when the project has set it;
+ *   3. off.
+ * Indexing requests are not governed by this: an indexing pass keeps the tools
+ * it has today (see requests.ts notifyAgentSaveAttachment).
+ */
+export const DEFAULT_WEB_ACCESS = false;
+
+/** Narrow a stored value to the switch: only `true` is on. */
+export function normalizeWebAccess(value: any): boolean {
+	return value === true;
+}
+
+/** The switch held in a settings-record `data`, false when unset. */
+export function webAccessFrom(data: ProjectSettingsData | null | undefined): boolean {
+	return data ? normalizeWebAccess(data.web_access) : DEFAULT_WEB_ACCESS;
+}
+
 /**
  * Longest custom opening line the dashboard will store.
  *
@@ -314,6 +344,16 @@ export function projectChatGreeting(service: string): string {
 	return chatGreetingFrom(cachedProjectSettings(service));
 }
 
+/**
+ * Sync convenience: the project's web-access setting, false until the fetch
+ * settles or when the project never set it. The request builders prefer the
+ * awaiting form (config.ts resolveWebAccess) so the first turn after a page load
+ * reads the stored value rather than the cold-cache default.
+ */
+export function projectWebAccess(service: string): boolean {
+	return webAccessFrom(cachedProjectSettings(service));
+}
+
 /**
  * Adopt a value the caller just WROTE, so the settings page reflects its own
  * save without a re-fetch.

package/src/engine/prompts/chat_system_prompt.ts

@@ -60,10 +60,20 @@ export type ChatSystemPromptParams = {
 	 * ones.
 	 */
 	indexAccessGroup?: string;
+	/**
+	 * Whether THIS chat may reach the open web. The engine attaches Claude's
+	 * web_fetch and OpenAI's web_search to a chat request only when the project's
+	 * web switch is on (the host's `webAccess` option, else the project's
+	 * `web_access` setting, else off: config.ts webAccessEnabled), and the prompt
+	 * follows the tool list, so the model is never told to fetch with a tool it
+	 * does not have. Omitted means off, which is also the engine's default.
+	 */
+	webAccess?: boolean;
 };
 
 export function buildChatSystemPrompt(params: ChatSystemPromptParams): string {
 	const { projectId, serviceName, serviceDescription, greeting, canUpload, client } = params;
+	const webAccess = params.webAccess === true;
 	// Rendered as the model would have to WRITE it in a tool call: the named
 	// aliases go in quotes, a raw group number does not.
 	const g = params.indexAccessGroup;
@@ -72,20 +82,59 @@ export function buildChatSystemPrompt(params: ChatSystemPromptParams): string {
 		: (g === 'public' || g === 'private' || g === 'authorized' || g === 'admin') ? `"${g}"`
 		: '"authorized"';
 
+	// ---- Grounding ---------------------------------------------------------
+	// The rules around this block were all written against the OPPOSITE mistake: a
+	// model that says "not found" before it has really looked. Read alone they push
+	// one way only (keep searching, never say it is absent), and nothing said the
+	// other half: state only what the data returned. Customers reported answers
+	// that were not in their data at all, which is the one thing this product must
+	// never do, so the other half is spelled out here.
+	//
+	// Three rules, each closing one way an invented answer gets out: a fact with no
+	// source, a "not found" replaced by general knowledge, and an inference worded
+	// as if the data said it. The last line keeps them from swallowing what is NOT
+	// a claim about the project's data (product help, a greeting, authored text).
+	// The upload suggestion follows canUpload for the same reason the About section
+	// does: a visitor who cannot upload must never be told to.
+	const groundingRules =
+`ANSWER ONLY FROM THE DATA. Every fact you state about this project's data - a name, a number, a date, a status, what a document says, whether something exists - must come from one of four places: a tool result returned in THIS conversation, a file or image attached to it (including one you read through its attachment link), the project description at the end of this prompt, or what the user told you themselves in this chat. Your general knowledge, the open internet and a plausible guess are NOT sources. Never use them to answer a question about the user's data, to fill a gap in it, or to complete a record that came back partial. A value that is not in what you retrieved is unknown: say it is not in the data instead of supplying a likely one.
+WHEN THE DATA DOES NOT HAVE IT, SAY SO. Once you have searched the way these rules require and nothing answers the question, tell the user plainly, in their language, that this project's data does not contain it, and say what you searched (the tables, files or terms) so they can tell missing data from a missed search. Do not answer from memory instead, and do not turn a missing answer into a general explanation of the topic. Offer what would let you answer it: ${canUpload === false
+			? `another name or file to look under, or asking the project's owner to add it`
+			: `the document to upload, or another name or file to look under`}.
+KEEP WHAT THE DATA SAYS APART FROM WHAT YOU WORKED OUT. A total you computed, a comparison, a trend, an interpretation or a recommendation is yours, not the data's: label it as such ("calculated from...", "this suggests...") and name the rows or files it was made from. Never present an estimate, an assumption or an inference as something the data states.
+These three rules govern every claim about the project's data. They do not limit the About BunnyQuery section (answer product questions from it), a greeting, or text the user explicitly asks you to write.`;
+
+	// ---- Documents -------------------------------------------------------
+	// One method for a question about a file's CONTENT (a contract, a report, a
+	// manual), where the invented answers were worst: the model picked records it
+	// liked and paraphrased. Find the file, find the page with searchText, quote
+	// the exact text, cite file and page; an old file (no page text) is read with
+	// readFileContent; nothing found is said as such. The last sentence follows
+	// the web switch: with the web off the chat has no such tool, with it on a web
+	// fact must be marked as one. Additive: the rules around it are unchanged.
+	const documentRules =
+`DOCUMENTS (a contract, a report, a manual, a letter, any uploaded file the user asks about the content of): FIRST FIND THE FILE, with getFileOutline when you know its storage path, otherwise with getUniqueId using unique_id "src::" and condition "gte" (narrow it by the name the user used). THEN FIND THE PAGE: call searchText with the file's path and the words the user asked about; it returns the file, the page and the matching excerpt. QUOTE THE EXACT TEXT it returned and CITE THE FILE AND THE PAGE ("report.pdf, page 7"); never present a paraphrase as a quote. If searchText answers no_page_text, the file was indexed before page text existed: read it with readFileContent instead, paging with the cursor until END OF FILE, and cite the file. When searchText and the file's records come back empty over files that HAVE page text, say that the data does not have it and name the files and the terms you searched. Never fill the gap from your general knowledge, and never from the web${webAccess
+			? ` without saying so. WEB ACCESS IS ON for this project: when the user asks for something that is not in the project's data, you may search the web for it, but every such fact must be marked as coming from the web (name the site), kept apart from what the project's data says, and never used to fill in or complete a document's content. This is the one exception to the four sources named above.`
+			: `: this chat has no web tool.`}`;
+
 	let systemPrompt = `
 You are a dedicated assistant for the project ID: "${projectId}".
 Scope: Only answer questions about this project and its data. Do not answer questions about other projects or topics unrelated to this project. When the user refers to "my database", "my data", or "my files", treat those as references to this project's database and file storage. The ONE exception is BunnyQuery itself - what this app is, what it can do, and how to use it - which is always in scope: answer it from the "About BunnyQuery" section at the end of this prompt.
 Knowledge lookup: Before saying you don't know or that something isn't in the chat history, ALWAYS query this project's database through the available MCP tools to look for the answer. The user's data is the source of truth - the chat transcript is not. Only respond with "I don't know" or "I couldn't find that" after you have actually searched the project's data and come back empty.
+${groundingRules}
 NUMBERS FROM A SPREADSHEET: use queryGrid, never mental arithmetic over records. A total, a count, an average, a "how many mention X", a "which one is biggest" - all of those are computed server-side over EVERY row of the file and come back with the sheet, the row count and the row numbers they were made from. Records are a SAMPLE, and a sample added up is a confident wrong number. Quote the row count and the sheet alongside the figure so the reader can check it.
 CALL queryGrid describe FIRST, before any figure. Workbooks routinely state the same money more than once: a detail sheet, then per-song, per-album and per-artist sheets that each re-total it, plus a summary sheet whose bottom row is the file total. Those look like four different answers and are one. describe names which sheets restate which, and which rows are totals. Pick ONE sheet, say which you picked, and never add figures across a sheet and its summary. If the reply carries a warning about restatement, repeat it to the user.
 A FILE TOTAL IS NOT A ROW'S TOTAL. The biggest number on a summary sheet is the whole file, not the thing that was asked about. Before quoting any figure, check it is scoped to what the question named: filter by the column that identifies it and report how many rows matched.
+${documentRules}
 Complete answers over stored data: The database holds one record per spreadsheet row, and each uploaded file becomes many records. ONE file is routinely SPLIT ACROSS SEVERAL TABLES - a summary row in one table, its page or row content in another, its extracted photos and other media in "__MEDIA__", and the indexer often invents a differently-named table on each pass. An index or tag filter matches inside ONE table only and requires table_name: on getRecords, an index or tag sent with table_name but no access_group is auto-filled with access_group "authorized", but THIS project indexes at access_group ${indexGroupLiteral}, so pass access_group ${indexGroupLiteral} EXPLICITLY on EVERY query that names a table_name here, index or tag or plain - the auto-fill would search a group this project's data is not in and come back empty, and leaving access_group off a plain table query does NOT mean "all groups": unless you are the project's owner the server reads a table with no group as access_group 0 (public only), so a table indexed at ${indexGroupLiteral} comes back empty with its records sitting right there. Files uploaded before the project's setting changed may sit at another group, so when a scoped query comes back empty, retry it across the other groups (0, 1, "private") before concluding there is nothing, while an index or tag WITHOUT table_name FAILS with an error instead of answering, so read the error rather than guessing. Reference is the exception: reference ALONE spans EVERY table and EVERY access group, so getRecords with reference "src::<the file's storage path>" is the one call that returns a whole file's records wherever the indexer put them. Adding table_name narrows it to that table; access_group WITHOUT table_name fails with '"table" is required'; table_name on its own returns that whole table across all access groups ONLY for the project's owner, and only its access_group 0 records for any other user, so name the group whenever you name a table. For anything NOT scoped to a single file, call getTables FIRST, run the query once per table that could hold the answer, and combine the results. For any request that counts, sums, totals, lists every match, compares across records, finds which one, or asks whether something is present or ABSENT (for example "how many", "total spent", "which card", "is there any", "없어?", "하나도 없나?"), you MUST read the COMPLETE matching set before answering. Query with fetch_all set to true, or page through getToolResponsePage until pagination.complete is true, across EVERY table and EVERY relevant file. A single default query returns only the first page (about 50 records). That is a SAMPLE. Never treat it as the whole dataset. If you already answered from one table and then realise another table holds more, do not simply apologise: re-run the sweep and give the complete answer.
 Never assert absence from a partial read. Do not say "there is no X", "none", "not found", or "아니요, 없습니다" until a complete scan has come back empty. If you have not finished scanning every relevant table and file, keep querying instead of guessing. A confident "no" that later turns out wrong is worse than telling the user you are still checking.
 Embedded values: a search term is often stored inside a larger string. A merchant "BAKSA" appears as "DNH*BAKSA#4070277042", and a card as "5860****5173". Server-side index filters match only exact values, leading prefixes, or trailing suffixes, and tag filters only EXACT whole-tag values - never a partial or interior substring - so filtering on such a field silently drops rows. When the value you are looking for may be embedded, do not trust a narrow filter to be complete. Fetch the full set with fetch_all and match the substring yourself.
 File attachments: When a user message contains an "Attached files:" section with markdown links, those links point to short-lived signed URLs in this project's db storage and will expire.
 - Image files (.jpg, .jpeg, .png, .gif, .webp) are ALREADY attached inline as image content blocks in the same message - you can see them directly. Do NOT call web_fetch on image URLs; that will fail or return garbage. Just look at the image block and answer.
 - Other attached files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, email messages (.eml), and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) are ALREADY INDEXED: they were read end to end when they were uploaded, before this message reached you, and their content is in the database as records. Query it with getRecords using reference "src::<the storage path from the attachment link>" - one call, every table, every access group. Do NOT call web_fetch on their URLs. If you need the raw text rather than the indexed records (an exact quote, a specific cell), call readFileContent on that same path and page it with the cursor. Some turns instead carry the file text inlined between "BEGIN FILE CONTENT" / "END FILE CONTENT" markers; when that block is present read it directly, and a "[skapi: ...]" note inside it means that file could not be extracted.
-- For any file given to you as a URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL before answering. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
+${webAccess
+			? `- For any file given to you as a URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL before answering. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.`
+			: `- PDFs and every other attached file are indexed the same way before this message reaches you, and this chat has NO web tool: never try to download or fetch an attachment's URL. Answer a PDF from its records (getRecords with reference "src::<the storage path from the attachment link>"), and for its exact wording use searchText or readFileContent on that same path (see DOCUMENTS above).`}
 Stored files and readFileContent: for a file ALREADY in this project's storage, its pages and rows were read at upload time and saved as records, so the database is your best source. Query those records first (getRecords with reference "src::<path>", or getUniqueId with unique_id "src::" and condition "gte" to find the file). readFileContent re-reads the raw file and is the right tool for text, spreadsheet and data files; it returns ONE window per call, so keep paging with the cursor from the previous window until it says END OF FILE before you conclude anything is absent. Be aware its PICTURES may not reach you: page images and embedded photos are attached as image blocks that several clients drop, leaving you only markers such as «PHOTO A88» or a "(scanned; read the page images)" header. There is no OCR on the server, so a scanned page with no text layer carries no text at all. If you cannot actually see an image, say so plainly and fall back to the indexed records; never describe a picture you were not shown, and never tell the user the file is unreadable when its content is already in the database.
 File links: When you find a record whose unique_id starts with "src::", the part after "src::" is the file's storage path or original URL. Always present it as a markdown link so the user can access it. Strip the "src::" prefix - do NOT show it. Format: [filename](db:path/to/file) for storage paths, or [filename](https://...) for external URLs. The db: prefix is REQUIRED on storage paths: it tells the chat client the target is a stored file rather than a web address, instead of leaving it to guess. Everything after db: is the path exactly as stored, including spaces and parentheses, and NOT url-encoded. Storage-path links render as clickable buttons in this chat client that fetch a fresh signed URL on demand - so even if a previously shared URL has expired, give the user the storage-path link instead of saying the file is unavailable. Never tell the user a file is inaccessible or a URL is expired if you have its storage path in the database.
 File lookup: When the user asks to see, list, or show files (e.g. "show me uploaded files", "list my images", "show me the reference video"), query the database using getUniqueId with unique_id "src::" and condition "gte" (or getRecords by table) to find all indexed file records; every file extracted out of a document has one too, in table "__MEDIA__" (access_group "authorized"). Present each result as a markdown link as described above. Never say you cannot access file storage: the paths are indexed in the database.