bunnyquery 1.5.1 → 1.5.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bunnyquery",
-  "version": "1.5.1",
+  "version": "1.5.2",
   "description": "Embeddable BunnyQuery AI chat widget + its framework-agnostic chat engine",
   "main": "bunnyquery.js",
   "exports": {

package/src/engine/budget.ts

@@ -58,7 +58,10 @@ export function buildBoundedChatMessages(options: BoundedChatOptions) {
 	var trimmed = windowed.map(function (m, i) {
 		if (i === latestIndex) return m;
 		var stripped = stripFileBlocksFromHistory(m.content);
-		var sanitized = m.role === 'user' ? sanitizeAttachmentLinksForHistory(stripped, options.serviceId) : stripped;
+		// Sanitize BOTH roles: user turns via the "Attached files:" block, assistant
+		// turns via the safe db-only path (forAssistant=true) so a volatile db url the
+		// model emitted doesn't get replayed into the LLM context as a dead link.
+		var sanitized = sanitizeAttachmentLinksForHistory(stripped, options.serviceId, m.role !== 'user');
 		return Object.assign({}, m, { content: sanitized });
 	});
 	var bounded: Array<{ role: string; content: string }> = [], used = 0;

package/src/engine/history.ts

@@ -107,7 +107,9 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 			if (serverItemId !== undefined) em._serverItemId = serverItemId;
 			mapped.push(em);
 		} else if (assistantText) {
-			var okm: any = { role: 'assistant', content: assistantText };
+			// Safe db-only sanitize (forAssistant) so a volatile db url the model
+			// emitted renders as a re-mintable `_expired_.url` link, not a dead one.
+			var okm: any = { role: 'assistant', content: sanitizeAttachmentLinksForHistory(assistantText, opts.serviceId, true) };
 			if (item._isBgTask) okm.isBackgroundTask = true;
 			if (serverItemId !== undefined) okm._serverItemId = serverItemId;
 			mapped.push(okm);

package/src/engine/host.ts

@@ -96,6 +96,13 @@ export interface ChatHost {
 	}): Promise<any>;
 	/** Mint a temporary CDN URL for a stored file. */
 	getTemporaryUrl(storagePath: string): Promise<string>;
+	/** Delete a file's AI-index record ("src::<storagePath>") ahead of a
+	 *  reindex/overwrite so the agent re-creates it fresh instead of colliding/
+	 *  duplicating. The skapi backend cascades a src:: delete to the record's
+	 *  reference-linked children. OPTIONAL — hosts that don't implement it fall
+	 *  through to a plain re-index. Implementations must be best-effort (swallow
+	 *  "not found" / permission errors so indexing still proceeds). */
+	deleteExistingFileRecord?(storagePath: string): Promise<any>;
 	/** Map a relative path to the consumer's db storage key (e.g. uid-prefixed). */
 	storagePathFor(relPath: string): string;
 	getMimeType(name: string): string | null;

package/src/engine/links.ts

@@ -56,13 +56,35 @@ export function buildDisplayExpiredAttachmentHref(remotePath: string, fallback?:
 	return EXPIRED_ATTACHMENT_URL_ORIGIN + '/' + encodePathSegments(getExpiredAttachmentVisiblePath(remotePath, fallback));
 }
 
-export function sanitizeAttachmentLinksForHistory(content: string, serviceId: string): string {
-	if (!content || content.indexOf('Attached files:') === -1) return content;
+// Does `href` point at THIS service's db attachment storage? A db attachment URL's
+// path always begins with the serviceId segment (…/<serviceId>/<hash>/<path>). Used
+// to SAFELY sanitize assistant messages — where an arbitrary external citation URL
+// must never be rewritten, only the service's own volatile db links.
+export function isServiceDbAttachmentHref(href: string, serviceId: string): boolean {
+	if (!serviceId) return false;
+	try {
+		var parsed = new URL(href);
+		if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return false;
+		var segs = normalizeAttachmentPathCandidate(parsed.pathname || '').split('/').filter(Boolean);
+		return segs.length > 0 && segs[0] === serviceId;
+	} catch (e) { return false; }
+}
+
+// Replace volatile attachment URLs with their durable `_expired_.url/<path>`
+// placeholder so a stored/replayed copy re-mints on demand instead of going stale.
+// USER turns: the "Attached files:" block gates the (broad) rewrite — every url
+// there is a db link. ASSISTANT turns (forAssistant=true): no marker exists and the
+// text may contain external citation urls, so restrict the rewrite to THIS
+// service's db attachment urls and leave every other link untouched.
+export function sanitizeAttachmentLinksForHistory(content: string, serviceId: string, forAssistant?: boolean): string {
+	if (!content) return content;
+	if (!forAssistant && content.indexOf('Attached files:') === -1) return content;
 	return content.replace(/\[([^\]\n]+)\]\((https?:\/\/[^\s)]+)\)/g, function (_m: string, label: string, href: string) {
+		if (forAssistant && !isServiceDbAttachmentHref(href, serviceId)) return _m;
 		var remotePath = extractRemotePathFromAttachmentHref(href, serviceId);
 		var labelPath = normalizeAttachmentPathCandidate(label);
 		var fullPath = remotePath || labelPath;
-		if (!fullPath) return '[' + label + '](' + EXPIRED_ATTACHMENT_URL_ORIGIN + '/file)';
+		if (!fullPath) return forAssistant ? _m : '[' + label + '](' + EXPIRED_ATTACHMENT_URL_ORIGIN + '/file)';
 		return '[' + label + '](' + buildDisplayExpiredAttachmentHref(fullPath, label) + ')';
 	});
 }

package/src/engine/office.ts

@@ -21,6 +21,18 @@ export type ExtractDirective = {
 	mime?: string;
 };
 
+// A directive telling the proxy worker to re-mint a fresh, short-lived URL for a
+// file the model fetches by url (PDFs, images — anything NOT server-extractable)
+// right before the upstream call, so a queued request never hands the model a
+// stale link. The worker mints from `path` and swaps `url` (the exact baked url
+// string) everywhere it appears in the request body.
+export type FileUrlDirective = {
+	/** db storage path of the file, e.g. "folder/report.pdf" (also the src:: value). */
+	path: string;
+	/** The exact baked url string in the request body to replace with a fresh one. */
+	url: string;
+};
+
 // Files whose text the worker extracts SERVER-SIDE and inlines for indexing,
 // instead of handing the agent a URL to fetch. Two groups:
 //   (1) BINARY document formats the model can't read at all (OOXML, Hancom
@@ -106,6 +118,8 @@ export interface ComposedUserMessage {
 	composedForLlm: string;
 	/** Office-extraction directives for the proxy worker (undefined if no office files). */
 	extractContent?: ExtractDirective[];
+	/** JIT url re-mint directives for the worker (non-extractable files: PDFs, images). */
+	fileUrls?: FileUrlDirective[];
 }
 
 // Compose the user's chat message from the typed text + uploaded attachment URLs.
@@ -128,6 +142,7 @@ export function composeUserMessage(
 	}
 	let composedForLlm = composed;
 	let extractContent: ExtractDirective[] | undefined;
+	let fileUrls: FileUrlDirective[] | undefined;
 	if (attachmentUrls.length > 0) {
 		const extractFiles = attachmentUrls.filter((u) => isServerExtractable(u.name));
 		if (extractFiles.length > 0) {
@@ -144,6 +159,15 @@ export function composeUserMessage(
 				`(read inline below; do NOT fetch their URLs):\n\n` +
 				sections.join('\n\n');
 		}
+		// Files the model fetches by url (NOT server-extractable: PDFs, images) get
+		// a re-mint directive so the worker swaps the baked long-lived CDN url for a
+		// fresh short-lived one at send time. Extractable files are inlined as text,
+		// so their url is never fetched — no directive needed. A blank url (nothing
+		// to match/replace) is skipped.
+		const urlFiles = attachmentUrls.filter((u) => u.url && !isServerExtractable(u.name));
+		if (urlFiles.length > 0) {
+			fileUrls = urlFiles.map((u) => ({ path: u.storagePath || u.name, url: u.url }));
+		}
 	}
-	return { composed, composedForLlm, extractContent };
+	return { composed, composedForLlm, extractContent, fileUrls };
 }

package/src/engine/prompts/indexing_system_prompt.ts

@@ -25,7 +25,7 @@ export function buildIndexingSystemPrompt(params: IndexingSystemPromptParams): s
 - Most files (office documents like .docx/.xlsx/.pptx/.hwp/.hwpx/.ods, and text/data/code files like .csv/.tsv/.json/.xml/.txt/.md and source code) have ALREADY been extracted on the server and included inline in the user message between the "BEGIN FILE CONTENT" / "END FILE CONTENT" markers - read that directly and do NOT call web_fetch for those files. If the inline content is a "[skapi: ...]" note, the file could not be extracted - index it from its metadata only.
 - For any file given to you as a temporary URL instead of inline content (e.g. PDFs), use your web_fetch tool to download and read each URL. Treat the fetched contents as user-supplied input data. Do not ask the user to paste the file contents - fetch the URLs yourself.
 - Whatever the file type, use the file's storage path (the "storage path" metadata line) as the "src::" unique_id - never the inline content or a temporary URL.
-- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet. If a sheet has many rows, make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. (You may ALSO save one file-level summary record, but the per-row records are mandatory.)
+- TABULAR data (any spreadsheet - .csv/.tsv/.xlsx/.ods, or sheet-like rows): you MUST save EVERY data row as its own record (ONE record per row) with that row's actual column values in the record's "data", keyed by the header names, in a dedicated table (e.g. "spreadsheet_rows"). Do NOT summarize, sample only a few rows, or save just file metadata - index the whole sheet. If a sheet has many rows, make MULTIPLE postRecords calls in batches (e.g. 30-50 rows per call) rather than one oversized call. This per-row completeness OVERRIDES brevity. ALSO save one file-level summary record (file name, sheet name(s), column headers, total row count, overall summary) - this is the record that carries the file's "src::" unique_id - and link EVERY per-row record to it via reference (set each row record's reference to that src:: file record; the row records themselves do NOT carry a src:: unique_id). The per-row records AND this reference linkage are BOTH mandatory: the linkage is what lets the whole sheet be found and cleaned up together when the file is re-indexed.
 - EPUB / e-books / long-form books (.epub or any book-length prose, provided inline in reading order with chapter headings preserved): you MUST save ONE record per CHAPTER (or, when chapters are unclear, per major section/topic) in a dedicated table (e.g. "book_chapters") - never collapse the whole book into a single record. Each chapter record's "data" must capture the chapter title plus its order/number AND a substantive summary of that chapter's content (key events, arguments, characters, places, concepts, terms, notable quotes). Apply AS MANY relevant tags as possible to EVERY chapter record (characters, locations, themes, topics, key concepts, key terms, dates, named entities) so the book is easy to SEARCH and cross-reference later - this is the whole point. ALSO save one book-level record (title, author, language, overall summary, chapter list / table of contents, genre/subjects) and link each chapter record to it via reference. This per-chapter completeness OVERRIDES brevity; human-readable summaries only, never raw/binary bytes.
 - This is a ONE-SHOT background indexing task: do ALL the MCP saving FIRST, never reply mid-task, and never ask the user questions or invite back-and-forth. Always use the MCP tools to save what you learn - be exhaustive about meaning (and, for tabular data, about every row). Never store raw or binary bytes (base64, blobs); describe them in human-readable text instead.
 - Only AFTER every save is done, send exactly ONE final message summarizing what you indexed - never just "Indexing complete", and never a raw/base64/binary value or a large pasted dump. Keep it to a few factual sentences or a short markdown bullet list covering: the file name, its content type, each table you wrote to with its record/row count and the key columns/fields or topics captured, and anything that could not be extracted. Follow this shape - Indexed <file name> (<content type>): saved <N> records to <table(s)> capturing <key columns/fields or topics>; could not extract: <gaps, or none>.`;

package/src/engine/requests.ts

@@ -11,7 +11,7 @@
  *                                    only the BgTaskEntry TYPE lives here)
  */
 import { buildIndexingSystemPrompt, buildIndexingUserMessage } from './prompts';
-import { isServerExtractable, makeExtractPlaceholder, type ExtractDirective } from './office';
+import { isServerExtractable, makeExtractPlaceholder, type ExtractDirective, type FileUrlDirective } from './office';
 import { chatEngineConfig, pollOpt } from './config';
 
 export const ANTHROPIC_MESSAGES_API_URL = 'https://api.anthropic.com/v1/messages';
@@ -206,6 +206,7 @@ export type CallClaudeWithMcpParams = {
 	system?: string;
 	mcpServer: ClaudeMcpServerRequest;
 	extractContent?: ExtractDirective[];
+	fileUrls?: FileUrlDirective[];
 	onResponse?: (res: any) => void;
 	onError?: (err: any) => void;
 };
@@ -221,6 +222,7 @@ export async function callClaudeWithMcp({
 	system,
 	mcpServer,
 	extractContent,
+	fileUrls,
 }: CallClaudeWithMcpParams) {
 	const mcpServerDefinition: Record<string, any> = {
 		type: 'url',
@@ -252,6 +254,9 @@ export async function callClaudeWithMcp({
 			...(extractContent && extractContent.length
 				? { _skapi_extract: extractContent }
 				: {}),
+			...(fileUrls && fileUrls.length
+				? { _skapi_file_urls: fileUrls }
+				: {}),
 			...(system
 				? {
 						system: [
@@ -306,6 +311,7 @@ export async function callClaudeWithPublicMcp(
 	model?: string,
 	userId?: string,
 	extractContent?: ExtractDirective[],
+	fileUrls?: FileUrlDirective[],
 	onResponse?: (res: any) => void,
 	onError?: (err: any) => void,
 ) {
@@ -319,6 +325,7 @@ export async function callClaudeWithPublicMcp(
 		maxTokens: MAX_TOKENS,
 		system,
 		extractContent,
+		fileUrls,
 		mcpServer: {
 			name: MCP_NAME,
 			url: mcpUrl(),
@@ -338,6 +345,7 @@ export async function callOpenAIWithPublicMcp(
 	model?: string,
 	userId?: string,
 	extractContent?: ExtractDirective[],
+	fileUrls?: FileUrlDirective[],
 	onResponse?: (res: any) => void,
 	onError?: (err: any) => void,
 ) {
@@ -386,6 +394,9 @@ export async function callOpenAIWithPublicMcp(
 			...(extractContent && extractContent.length
 				? { _skapi_extract: extractContent }
 				: {}),
+			...(fileUrls && fileUrls.length
+				? { _skapi_file_urls: fileUrls }
+				: {}),
 			input: responseInput,
 			tools: [
 				{

package/src/engine/session.ts

@@ -108,11 +108,11 @@ export class ChatSession {
 		};
 	}
 
-	private _callProviderFor(platform: string, prompt: string, messages: any, system: string, model: string | undefined, userId: string, extractContent: any) {
+	private _callProviderFor(platform: string, prompt: string, messages: any, system: string, model: string | undefined, userId: string, extractContent: any, fileUrls?: any) {
 		var id = this.host.getIdentity();
 		return platform === 'openai'
-			? callOpenAIWithPublicMcp(prompt, id.serviceId, id.owner, messages, system, model, userId, extractContent)
-			: callClaudeWithPublicMcp(prompt, id.serviceId, id.owner, messages, system, model, userId, extractContent);
+			? callOpenAIWithPublicMcp(prompt, id.serviceId, id.owner, messages, system, model, userId, extractContent, fileUrls)
+			: callClaudeWithPublicMcp(prompt, id.serviceId, id.owner, messages, system, model, userId, extractContent, fileUrls);
 	}
 
 	dispatchAgentRequest(params: any) {
@@ -126,7 +126,7 @@ export class ChatSession {
 		var dispatchItemId: string | undefined;
 		var sendAndPoll = function () {
 			return Promise.resolve(
-				self._callProviderFor(params.aiPlatform, params.text, params.boundedMessages, params.systemPrompt, params.aiModel, params.userId, params.extractContent)
+				self._callProviderFor(params.aiPlatform, params.text, params.boundedMessages, params.systemPrompt, params.aiModel, params.userId, params.extractContent, params.fileUrls)
 			).then(function (initial: any) {
 				if (initial && initial.poll && (initial.status === 'pending' || initial.status === 'running')) {
 					if (initial.id) {
@@ -208,7 +208,7 @@ export class ChatSession {
 	// composed = clean display text; composedForLlm carries office-extraction
 	// placeholders for the provider only. useBgQueue routes a post-attachment turn
 	// onto the "-bg" queue so it runs after indexing.
-	dispatchComposedMessage(composed: string, useBgQueue?: boolean, composedForLlm?: string, extractContent?: any): void {
+	dispatchComposedMessage(composed: string, useBgQueue?: boolean, composedForLlm?: string, extractContent?: any, fileUrls?: any): void {
 		var self = this;
 		if (!composed) return;
 		var id = this.host.getIdentity();
@@ -241,7 +241,7 @@ export class ChatSession {
 			this.host.notify(); this.updateHistoryCache(); this.host.scrollToBottom(true);
 
 			var capturedComposed = composed, capturedPlatform = aiPlatform;
-			Promise.resolve(this._callProviderFor(aiPlatform, composed, boundedQ.messages, systemPrompt, aiModel, chatQueue, extractContent))
+			Promise.resolve(this._callProviderFor(aiPlatform, composed, boundedQ.messages, systemPrompt, aiModel, chatQueue, extractContent, fileUrls))
 				.then(function (result: any) {
 					var sendingIdx = self.state.messages.findIndex(function (m) {
 						return m.isSendingToServer && (m.isPendingQueued || m.isPendingInProcess) && m.role === 'user';
@@ -292,7 +292,7 @@ export class ChatSession {
 		var run = this.dispatchAgentRequest({
 			key: key, serviceId: id.serviceId, owner: id.owner, aiPlatform: aiPlatform, aiModel: aiModel,
 			systemPrompt: systemPrompt, text: composed, boundedMessages: bounded.messages, userId: chatQueue,
-			extractContent: extractContent,
+			extractContent: extractContent, fileUrls: fileUrls,
 		});
 		// Render the reply into the "Thinking..." bubble whenever the chatbox is
 		// CURRENTLY showing this chat — even after an unmount/remount. The old
@@ -1014,6 +1014,12 @@ export class ChatSession {
 			chain = chain.then(function () {
 				var hadExists = false;
 				var skipped = false;
+				// True when the file already existed and we are re-indexing over it
+				// (user chose Reindex, OR Overwrite replaced the bytes). Drives the
+				// delete-then-repost of the stale "src::" record below. Kept separate
+				// from hadExists, which must keep driving ONLY the Reindexing/Indexing
+				// label (isReindex) — overwrite still labels as "Indexing:".
+				var existedBefore = false;
 				var onProg = function (p: any) {
 					if (p && p.total) {
 						att.progress = Math.floor(((idx + p.loaded / p.total) / total) * 100);
@@ -1032,9 +1038,9 @@ export class ChatSession {
 					var isExists = code === 'EXISTS' || (msg && /exist/i.test(msg));
 					if (!isExists) throw err; // a member upload failed → whole attachment fails (red)
 					return self.host.promptOverwrite(member.file.name).then(function (choice) {
-						if (choice === 'overwrite') return doMemberUpload(false); // replace the existing file
+						if (choice === 'overwrite') { existedBefore = true; return doMemberUpload(false); } // replace the existing file
 						if (choice === 'skip') { skipped = true; return; } // leave it untouched; no upload/index
-						hadExists = true; // keep it; Reindex
+						hadExists = true; existedBefore = true; // keep it; Reindex
 					});
 				}).then(function () {
 					if (skipped) return; // user skipped this member — no url, no index request
@@ -1044,12 +1050,23 @@ export class ChatSession {
 					urls.push({ name: member.relPath, url: url, storagePath: member.storagePath });
 					if (att.kind !== 'folder') { att.uploadedUrl = url; att.storagePath = member.storagePath; }
 					var mime = member.file.type || self.host.getMimeType(member.file.name);
+					// Delete-then-repost: when the file already existed (Reindex or
+					// Overwrite), delete the stale "src::<storagePath>" index record
+					// first — the skapi backend cascades the delete to its
+					// reference-linked children — so re-indexing REPLACES rather than
+					// colliding/duplicating. Awaited before the index request is
+					// enqueued. Best-effort + optional-hook guarded: a missing record,
+					// a permission error, or a host without the hook must not block
+					// indexing.
+					var preIndex = (existedBefore && typeof self.host.deleteExistingFileRecord === 'function')
+						? Promise.resolve(self.host.deleteExistingFileRecord(member.storagePath)).catch(function () { })
+						: Promise.resolve();
 					// Run a client-side attachment parser (e.g. .hwp) if one matches; its
 					// output is inlined into the indexing request (falls back to office
 					// extraction / web_fetch when no parser matches or it yields nothing).
-					return Promise.resolve(
-						parseAttachmentContent(member.file, member.file.name, mime || undefined),
-					).then(function (parsedContent: string | null) {
+					return preIndex.then(function () {
+						return parseAttachmentContent(member.file, member.file.name, mime || undefined);
+					}).then(function (parsedContent: string | null) {
 					return notifyAgentSaveAttachment({
 						platform: id.platform as 'claude' | 'openai',
 						model: id.model,