bunnyquery 1.8.5 → 1.8.8

package/src/engine/history.ts

@@ -4,7 +4,7 @@
  * formatIndexingLabel callback) so the engine touches neither localStorage nor
  * view-specific display formatting. projectId is passed for link sanitization.
  */
-import { extractClaudeText, extractOpenAIText, INDEXING_COMPLETE_MARKER, EMPTY_INDEXING_REPLY } from './requests';
+import { extractClaudeText, extractOpenAIText, INDEXING_COMPLETE_MARKER, EMPTY_INDEXING_REPLY, getChatHistory, bgIndexingQueueName } from './requests';
 import { isErrorResponseBody, getErrorMessage } from './errors';
 import { sanitizeAttachmentLinksForHistory } from './links';
 
@@ -80,6 +80,513 @@ export function parseIndexingRequestText(userText: any): IndexingRequestRef | nu
 	};
 }
 
+// One page of the status probe below. Matches ChatSession's own
+// WORKER_PASS_ADOPT_LIMIT: the queue holds one live pass per file (plus a
+// handful pending), so a FULL page means the answer may be incomplete and the
+// probe reports checked=false rather than letting a miss read as "finished".
+const LIVE_INDEX_PROBE_LIMIT = 20;
+
+// ─── Shared bg-queue probe (write-through memo) ──────────────────────────────
+// The SAME two queries — getChatHistory({queue, status: 'pending'|'running'})
+// — are fired by four independent callers: the session's live-index snapshot,
+// its worker-pass adoption ladder, its drain wait, and fetchLiveIndexingKeys
+// (the db-files page's badge probe). Navigating chat → files re-asked the queue
+// the chat had answered seconds earlier.
+//
+// This memo dedupes them WITHOUT weakening anyone's evidence:
+//   • progress loops (adopt ladder, drain wait) call with maxAgeMs 0 — they
+//     always fetch fresh, because their whole point is to observe a CHANGE —
+//     but their answers write through into the memo;
+//   • snapshot readers pass a maxAge and reuse a young answer.
+// Every entry carries `at`, the time the underlying fetch actually ran. A
+// caller that needs two INDEPENDENT looks (dbfile's green-confirm ladder) must
+// compare `at`, never its own receipt time: a memo re-serve has the same `at`
+// and therefore counts as the single look it really is.
+export const BG_PROBE_TTL_MS = 4000;
+type BgProbeEntry = { result: any; at: number };
+const bgProbeCache: { [key: string]: BgProbeEntry } = {};
+const bgProbeInflight: { [key: string]: Promise<BgProbeEntry> } = {};
+
+export function probeBgQueue(
+	params: {
+		service: string;
+		owner: string;
+		platform: 'claude' | 'openai';
+		queue: string;
+		status: 'pending' | 'running';
+		limit: number;
+	},
+	opts?: { maxAgeMs?: number },
+): Promise<BgProbeEntry> {
+	const key = [params.service, params.owner, params.platform, params.queue, params.status, params.limit].join('|');
+	const maxAge = opts && typeof opts.maxAgeMs === 'number' ? opts.maxAgeMs : 0;
+	const cached = bgProbeCache[key];
+	if (maxAge > 0 && cached && Date.now() - cached.at < maxAge) {
+		return Promise.resolve(cached);
+	}
+	const inflight = bgProbeInflight[key];
+	if (inflight) return inflight;
+	const p = Promise.resolve(getChatHistory(
+		{ service: params.service, owner: params.owner, platform: params.platform, queue: params.queue, status: params.status },
+		{ limit: params.limit, fetchMore: false },
+	)).then(function (result: any) {
+		const entry: BgProbeEntry = { result: result, at: Date.now() };
+		bgProbeCache[key] = entry;
+		return entry;
+	});
+	bgProbeInflight[key] = p;
+	// Clear the in-flight slot on either outcome; failures are NOT cached.
+	p.then(function () { delete bgProbeInflight[key]; }, function () { delete bgProbeInflight[key]; });
+	return p;
+}
+
+/**
+ * One bounded look at the background-indexing queue: which files still have a
+ * pass pending or running? This is the same negative signal ChatSession's
+ * display layer relies on - for a worker-driven (auto_continue) run, only the
+ * queue can say the run is over, because the worker enqueues continuation
+ * passes the client never dispatched.
+ *
+ * Returns every storage path AND file name found on live passes (both, because
+ * older prompts may lack the storage-path line), plus `checked`: false when a
+ * page came back full, in which case absence from `keys` proves nothing and
+ * the caller must keep whatever state it already had.
+ *
+ * SCOPE: the probed queue is "<userId>-bg" - THIS user's dispatches only. A
+ * chain launched by another collaborator or a widget end-user lives on their
+ * queue and is invisible here, so "idle" must never be read as "nobody is
+ * indexing this file", only as "this user's runs are over". The durable done::
+ * marker (indexDoneUniqueId) is the cross-user signal.
+ */
+export async function fetchLiveIndexingKeys(params: {
+	service: string;
+	owner: string;
+	platform: 'claude' | 'openai';
+	/** Same value the dispatch used - see bgIndexingQueueName. */
+	userId?: string;
+}): Promise<{ keys: Set<string>; checked: boolean; at: number }> {
+	const queue = bgIndexingQueueName(params.userId, params.service);
+	const base = { service: params.service, owner: params.owner, platform: params.platform, queue };
+	const [pending, running] = await Promise.all([
+		probeBgQueue({ ...base, status: 'pending', limit: LIVE_INDEX_PROBE_LIMIT }, { maxAgeMs: BG_PROBE_TTL_MS }),
+		probeBgQueue({ ...base, status: 'running', limit: LIVE_INDEX_PROBE_LIMIT }, { maxAgeMs: BG_PROBE_TTL_MS }),
+	]);
+	const keys = new Set<string>();
+	let truncated = false;
+	for (const entry of [pending, running]) {
+		const res = entry.result;
+		const list: any[] = (res && Array.isArray((res as any).list)) ? (res as any).list : [];
+		if (list.length >= LIVE_INDEX_PROBE_LIMIT) truncated = true;
+		for (const item of list) {
+			const text = extractLastUserTextFromRequest(item && item.request_body);
+			if (!text || !isIndexingRequestText(text)) continue;
+			const ref = parseIndexingRequestText(text);
+			if (!ref) continue;
+			if (ref.path) keys.add(ref.path);
+			if (ref.name) keys.add(ref.name);
+		}
+	}
+	// `at` = when the OLDER of the two underlying fetches ran. Consumers that
+	// need two independent looks (dbfile's green ladder) key on this, so a
+	// memo re-serve is correctly seen as the same single look.
+	return { keys, checked: !truncated, at: Math.min(pending.at, running.at) };
+}
+
+// ─── Split history fetch ─────────────────────────────────────────────────────
+// One conversation, two queues: ordinary chat lives on the user's queue (or a
+// legacy/random one), indexing passes on "<userId>-bg" — whose stored request
+// bodies carry whole file windows and dominated every history page. The split
+// fetches the SURFACE (id-prefix listing minus the bg queue, full bodies) and
+// the BG queue (exact, compact stubs — the lambda stubs ONLY indexing-shaped
+// items, ordinary chats deferred onto the bg queue keep full bodies) and
+// returns a getChatHistory-shaped page.
+//
+// This is v2, redesigned around the 2026-08-11 review findings:
+//   • The SDK ignores explicit fetchOptions.startKeyHistory — paging rides its
+//     internal per-params-hash cursors. So ALL split state lives HERE, module-
+//     level, keyed by (service|owner|platform|userId): the bg overflow buffer,
+//     the undelivered surface page, end flags. Callers keep echoing
+//     startKeyHistory opaquely; it is bookkeeping only, on both sides.
+//   • Pages TILE exactly: each page emits only items at-or-newer-than the
+//     surface page's oldest item (the boundary); bg items older than that are
+//     BUFFERED for the next page. Consumers' raw prepend, retainedOlder
+//     pruning and clear-horizon early-end all rely on this invariant.
+//   • Retry-safe: the fetched surface page is held as pendingSurface until the
+//     merged page is actually returned, and every bg page lands in the buffer
+//     the moment it arrives — a mid-call failure + caller retry resumes with
+//     nothing skipped (the SDK's internal cursors have already advanced; the
+//     state carries what they advanced past).
+//   • Empty-but-not-end pages are eliminated SERVER-side (the lambda re-queries
+//     past fully-filtered windows). A defensive client cap remains.
+//
+// Backwards-safe: an old backend ignores queue_exclude/compact — the surface
+// then includes bg items with bodies (exactly today's single fetch), the bg
+// call returns duplicates, and dedup-by-id (surface wins) degrades to current
+// behavior plus one redundant query, never to data loss.
+// Per-call bg fetches. Depth is LAZY: the consumers merge prepended pages by
+// timestamp, so older stubs can arrive on any later page and still land at
+// their true position — there is no need to drain the bg queue eagerly, and
+// doing so is exactly what made first paint slow (bg windows are ~1MB of RAW
+// rows, i.e. a handful of stubs per round trip). Two fetches keep the nearby
+// stubs in the same paint; scroll-up/viewport-fill pages pull the rest.
+const BG_COVERAGE_MAX_PAGES = 2;
+
+type SplitHistoryState = {
+	/** Fetched bg items not yet emitted (older than the last page's boundary). */
+	bgBuffer: any[];
+	bgEnd: boolean;
+	/** Whether the bg chain has fetched at least once (first call resets the
+	 *  SDK's internal bg cursor with fetchMore:false; later calls continue). */
+	bgStarted: boolean;
+	surfaceEnd: boolean;
+	/** Surface page fetched but not yet delivered in a returned merged page —
+	 *  reused on retry so the SDK's advanced cursor cannot skip it. Stamped
+	 *  with the fetchMore mode it was fetched under: a page-1 fetch abandoned
+	 *  mid-coverage must never be replayed as an OLDER page (or vice versa). */
+	pendingSurface: { list: any[]; endOfList: boolean; startKeyHistory: any[]; forFetchMore: boolean } | null;
+	/** Surface items fetched but HELD BACK because the bg coverage loop's hop
+	 *  cap fired before the bg side reached this page's boundary: emitting
+	 *  them would let the uncovered bg stubs land on a LATER page above
+	 *  strictly-newer surface turns. Prepended to the next page's surface. */
+	surfaceCarry: any[];
+	lastSurfaceKeys: any[];
+	/** Newest bg item id this chain has ever seen. The head refresh compares it
+	 *  against the fresh head page: a FULL head page that no longer contains it
+	 *  means more than one page of bg rows landed while nobody was polling, and
+	 *  the walk must reopen or the gap is unfetchable forever. */
+	newestBgId: string;
+};
+const splitHistoryStates: { [key: string]: SplitHistoryState } = {};
+// Per-key serialization: a token-bumped reload can start a NEW split call
+// while an old one is mid-flight, and both would interleave on the SDK's
+// shared internal cursor chains (silently skipping pages). The newcomer waits
+// for the incumbent to settle; its stale result is discarded by the callers'
+// own token guards, and the newcomer's fetchMore:false reset then starts from
+// a clean chain.
+const splitHistoryLocks: { [key: string]: Promise<void> } = {};
+
+function freshSplitState(): SplitHistoryState {
+	return { bgBuffer: [], bgEnd: false, bgStarted: false, surfaceEnd: false, pendingSurface: null, surfaceCarry: [], lastSurfaceKeys: [], newestBgId: '' };
+}
+
+/** Track the newest bg id the chain has seen (ids are creation-ordered). */
+function noteBgIds(state: SplitHistoryState, list: any[]): void {
+	for (const it of list) {
+		const id = it && typeof it.id === 'string' ? it.id : '';
+		if (id && id > state.newestBgId) state.newestBgId = id;
+	}
+}
+
+/** Test hook: drop split-fetch state (all keys, or one). */
+export function __resetSplitHistoryState(key?: string): void {
+	if (key !== undefined) { delete splitHistoryStates[key]; delete splitHistoryLocks[key]; return; }
+	for (const k in splitHistoryStates) delete splitHistoryStates[k];
+	for (const k in splitHistoryLocks) delete splitHistoryLocks[k];
+}
+
+const createdOf = (it: any): number => {
+	const c = Number(it && it.created);
+	return isFinite(c) && c > 0 ? c : NaN;
+};
+const oldestCreated = (lst: any[]): number => {
+	let m = Infinity;
+	for (const it of lst) {
+		const c = createdOf(it);
+		if (!isNaN(c) && c < m) m = c;
+	}
+	return m; // Infinity when no item carries a usable timestamp
+};
+
+// A pure-bg band can outlast the LAMBDA's own re-query cap (its raw windows
+// are 1MB and inline indexing bodies run to ~300KB, so one lambda call may
+// cross only a few dozen rows) — so the client must ALSO loop past
+// empty-but-not-end surface pages, or the fill loops upstream read them as
+// exhausted and strand older history (and an empty page 1 would blank the
+// chat). Each hop here is one more lambda call.
+const SURFACE_EMPTY_MAX_PAGES = 10;
+
+export type SplitHistoryResult = {
+	list: any[];
+	endOfList: boolean;
+	startKeyHistory: any[];
+	/** True when this chat had never been walked in this session — the first
+	 *  paint. Consumers gate the "Loading indexing history" hint on it: a
+	 *  mid-walk tab return restarts the walk for cursor safety but must stay
+	 *  silent (flashing the hint on every return was the reported bug). */
+	firstLoad?: boolean;
+	/** Present only when `deferBg` was requested AND bg work remains: resolves
+	 *  with the stub batch fetched in the background (the per-key lock is held
+	 *  until it settles, so no other history call can interleave). The caller
+	 *  merges the batch by timestamp — the same path older pages use. */
+	bgPending?: Promise<{ list: any[]; endOfList: boolean }>;
+};
+
+export async function getSplitChatHistory(
+	params: { service: string; owner: string; platform: 'claude' | 'openai'; userId?: string },
+	fetchOptions: Record<string, any>,
+	/** Test seam: replaces getChatHistory. Not for production callers. */
+	_fetchImpl?: typeof getChatHistory,
+): Promise<SplitHistoryResult> {
+	const key = [params.service, params.owner, params.platform, params.userId || ''].join('|');
+	const prev = splitHistoryLocks[key] || Promise.resolve();
+	// The lock resolves when the WHOLE call — including a deferred bg batch —
+	// has settled, so queued calls never interleave with background work. A
+	// call that returns WITHOUT a bgPending (or throws) releases immediately;
+	// a deferred call releases in the bg batch's own finally.
+	let releaseLock: () => void;
+	const lockTail = new Promise<void>((r) => { releaseLock = r; });
+	const run = () => _getSplitChatHistoryLocked(key, params, fetchOptions, releaseLock!, _fetchImpl);
+	const p = prev.then(run, run);
+	p.then((res: any) => { if (!res || !res.bgPending) releaseLock(); }, () => releaseLock());
+	splitHistoryLocks[key] = p.then(() => lockTail, () => lockTail);
+	return p;
+}
+
+async function _getSplitChatHistoryLocked(
+	key: string,
+	params: { service: string; owner: string; platform: 'claude' | 'openai'; userId?: string },
+	fetchOptions: Record<string, any>,
+	releaseLock: () => void,
+	_fetchImpl?: typeof getChatHistory,
+): Promise<SplitHistoryResult> {
+	const fetch = _fetchImpl || getChatHistory;
+	const bgQueue = bgIndexingQueueName(params.userId, params.service);
+	const base = { service: params.service, owner: params.owner, platform: params.platform };
+	const fetchMore = !!(fetchOptions && fetchOptions.fetchMore);
+	const limit = fetchOptions && fetchOptions.limit;
+
+	// HEAD refresh: a repeat first-page call on a chain that already walked BOTH
+	// queues to their end (tab return, poll-side page-1 refresh). Page 1 must be
+	// re-fetched for new settles, but what the chain LEARNED about the older
+	// tail — "all history fetched" — survives. Wiping it here was the tab-return
+	// bug: every return re-pulled the bg stub pages from scratch, reported
+	// endOfList false, and that un-gated the viewport fill into re-walking every
+	// older page the user had already exhausted. Only a FULLY-ended chain is
+	// safe to preserve: its fetchMore path short-circuits without touching the
+	// SDK cursors this page-1 refresh rewinds. A mid-walk chain still restarts.
+	// True when this chat has never been walked in this session at all — the
+	// consumers gate the "Loading indexing history" hint on it, so a mid-walk
+	// tab return (which restarts the walk for cursor safety) stays SILENT.
+	const firstLoad = !splitHistoryStates[key];
+	let headRefresh = false;
+	if (!splitHistoryStates[key]) {
+		splitHistoryStates[key] = freshSplitState();
+	} else if (!fetchMore) {
+		const prev = splitHistoryStates[key];
+		if (prev.surfaceEnd && prev.bgEnd) {
+			headRefresh = true;
+			prev.pendingSurface = null;
+			prev.surfaceCarry = [];
+			prev.bgBuffer = []; // fully drained by construction; defensive
+		} else {
+			// Mid-walk chain: the page-1 refresh genuinely restarts the walk
+			// (the underlying fetchMore:false calls below reset the SDK's own
+			// cursors to match), exactly as before.
+			splitHistoryStates[key] = freshSplitState();
+		}
+	}
+	const state = splitHistoryStates[key];
+
+	// A held page fetched under the OTHER mode is not this call's page —
+	// discard it rather than replay page 1 as an "older" page (or vice versa).
+	if (state.pendingSurface && state.pendingSurface.forFetchMore !== fetchMore) {
+		state.pendingSurface = null;
+	}
+
+	// ── surface page ────────────────────────────────────────────────────────
+	if (!state.pendingSurface) {
+		// The ended-chain short-circuit serves FETCHMORE only: a head refresh
+		// re-fetches page 1 for real (new settles live there), and the
+		// preserved surfaceEnd keeps every later fetchMore off the wire.
+		if (state.surfaceEnd && !headRefresh) {
+			state.pendingSurface = { list: [], endOfList: true, startKeyHistory: state.lastSurfaceKeys, forFetchMore: fetchMore };
+		} else {
+			const sOpts: any = { fetchMore };
+			if (limit) sOpts.limit = limit;
+			let s = await fetch({ ...base, queue_exclude: bgQueue }, sOpts);
+			// Loop past empty-but-not-end pages (see SURFACE_EMPTY_MAX_PAGES).
+			let hops = 0;
+			while (s && !s.endOfList && !((s.list || []).length) && hops < SURFACE_EMPTY_MAX_PAGES) {
+				hops++;
+				const nOpts: any = { fetchMore: true };
+				if (limit) nOpts.limit = limit;
+				s = await fetch({ ...base, queue_exclude: bgQueue }, nOpts);
+			}
+			state.pendingSurface = {
+				list: (s && Array.isArray(s.list)) ? s.list : [],
+				endOfList: !!(s && s.endOfList),
+				startKeyHistory: (s && Array.isArray(s.startKeyHistory)) ? s.startKeyHistory : [],
+				forFetchMore: fetchMore,
+			};
+		}
+	}
+	const surface = state.pendingSurface;
+
+	// ── deferred bg (first-paint mode) ──────────────────────────────────────
+	// The caller paints the CONVERSATION from this return immediately; the
+	// stub fetch happens behind the resolved promise, under the same lock.
+	// Anything already buffered ships now (it costs nothing).
+	if (fetchOptions && fetchOptions.deferBg && (!state.bgEnd || headRefresh)) {
+		const surfaceList0 = state.surfaceCarry.length ? state.surfaceCarry.concat(surface.list) : surface.list.slice();
+		state.surfaceCarry = [];
+		const emitNow: any[] = surfaceList0.concat(state.bgBuffer);
+		state.bgBuffer = [];
+		// A head refresh must not let page 1's "more pages exist" clobber the
+		// preserved end-knowledge: those pages are the tail already walked.
+		if (!headRefresh) state.surfaceEnd = surface.endOfList;
+		state.lastSurfaceKeys = surface.startKeyHistory;
+		state.pendingSurface = null;
+		const bgPending = (async () => {
+			try {
+				const batch: any[] = [];
+				if (headRefresh) {
+					// One HEAD page only: the walked tail is complete; this
+					// catches bg rows that settled/minted while nobody was
+					// polling. bgEnd/bgStarted are normally left alone — the
+					// end-knowledge survives, and the coverage loop (gated on
+					// !bgEnd) never touches the cursor this rewinds. ONE
+					// exception: a FULL head page that no longer contains the
+					// newest bg id this chain had seen means more than a page
+					// of bg rows landed while hidden — the gap is unreachable
+					// unless the walk reopens (id-dedup makes re-walking the
+					// tail harmless).
+					const bOpts: any = { fetchMore: false };
+					if (limit) bOpts.limit = limit;
+					const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
+					const bList: any[] = (b && Array.isArray(b.list)) ? b.list : [];
+					for (const it of bList) { if (it && typeof it === 'object') (it as any)._fromBgChain = true; batch.push(it); }
+					const prevNewest = state.newestBgId;
+					noteBgIds(state, bList);
+					if (prevNewest && !(b && b.endOfList) &&
+						!bList.some((it: any) => it && it.id === prevNewest)) {
+						state.bgEnd = false;
+						state.bgStarted = true;
+					}
+				} else {
+					let hops = 0;
+					while (!state.bgEnd && hops < BG_COVERAGE_MAX_PAGES) {
+						hops++;
+						const bOpts: any = { fetchMore: state.bgStarted };
+						if (limit) bOpts.limit = limit;
+						const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
+						state.bgStarted = true;
+						const bList: any[] = (b && Array.isArray(b.list)) ? b.list : [];
+						for (const it of bList) { if (it && typeof it === 'object') (it as any)._fromBgChain = true; batch.push(it); }
+						noteBgIds(state, bList);
+						state.bgEnd = !!(b && b.endOfList);
+						if (!bList.length && !state.bgEnd) break;
+						if (state.bgEnd) break;
+					}
+				}
+				return { list: batch, endOfList: state.surfaceEnd && state.bgEnd };
+			} finally {
+				releaseLock();
+			}
+		})();
+		return {
+			list: emitNow,
+			// A head-refreshed ended chain KNOWS it is still ended — reporting
+			// the hardcoded false here was what un-gated the fill loop on every
+			// tab return. Mid-walk it computes to false exactly as before (this
+			// branch is only entered with bgEnd false then); the bg batch still
+			// carries the final word for that case.
+			endOfList: state.surfaceEnd && state.bgEnd,
+			startKeyHistory: surface.startKeyHistory,
+			firstLoad,
+			bgPending,
+		};
+	}
+
+	// Carried-over surface items (held back by an uncovered boundary on the
+	// previous page) lead this page's surface list.
+	const surfaceList = state.surfaceCarry.length ? state.surfaceCarry.concat(surface.list) : surface.list.slice();
+
+	// The tiling line: everything at-or-newer than this is emitted now, older
+	// bg items wait in the buffer. A finished surface (endOfList) opens the
+	// line all the way (-Infinity) so the bg side drains over the next pages.
+	// An empty surface page past the loop above (a pathological pure-bg band
+	// beyond both the lambda's and our own hop caps) yields Infinity — no bg
+	// drain, an empty page, and the consumers' empty-page guards take over.
+	const boundary = surface.endOfList ? -Infinity : oldestCreated(surfaceList);
+
+	// ── bg coverage ─────────────────────────────────────────────────────────
+	if (headRefresh) {
+		// Non-deferred head refresh: the same single HEAD page the deferred
+		// branch fetches, under the same rules — new settles are caught, the
+		// end-knowledge (bgEnd/bgStarted) is left untouched, EXCEPT when a full
+		// head page no longer contains the newest bg id this chain had seen:
+		// more than a page landed while hidden, and the walk must reopen or the
+		// gap is unfetchable (id-dedup makes re-walking the tail harmless).
+		const hOpts: any = { fetchMore: false };
+		if (limit) hOpts.limit = limit;
+		const hb = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, hOpts);
+		const hbList: any[] = (hb && Array.isArray(hb.list)) ? hb.list : [];
+		for (const it of hbList) { if (it && typeof it === 'object') (it as any)._fromBgChain = true; state.bgBuffer.push(it); }
+		const prevNewestH = state.newestBgId;
+		noteBgIds(state, hbList);
+		if (prevNewestH && !(hb && hb.endOfList) &&
+			!hbList.some((it: any) => it && it.id === prevNewestH)) {
+			state.bgEnd = false;
+			state.bgStarted = true;
+		}
+	} else if (boundary !== Infinity || surface.endOfList) {
+		let hops = 0;
+		while (!state.bgEnd && hops < BG_COVERAGE_MAX_PAGES) {
+			const bufOldest = state.bgBuffer.length ? oldestCreated(state.bgBuffer) : Infinity;
+			if (state.bgBuffer.length && bufOldest <= boundary) break;
+			hops++;
+			const bOpts: any = { fetchMore: state.bgStarted };
+			if (limit) bOpts.limit = limit;
+			const b = await fetch({ ...base, queue: bgQueue, queue_exact: true, compact: true }, bOpts);
+			state.bgStarted = true;
+			const bList: any[] = (b && Array.isArray(b.list)) ? b.list : [];
+			// Buffer IMMEDIATELY: the SDK cursor has advanced; this is what a
+			// retry resumes from. Tagged as bg-chain items so the consumers'
+			// surface-frontier logic (retention boundary, clear-horizon) can
+			// tell deep stubs from the conversation's own paging frontier.
+			for (const it of bList) { if (it && typeof it === 'object') (it as any)._fromBgChain = true; state.bgBuffer.push(it); }
+			noteBgIds(state, bList);
+			state.bgEnd = !!(b && b.endOfList);
+			if (!bList.length && !state.bgEnd) break; // defensive: server loops past empties
+			if (state.bgEnd) break;
+		}
+	}
+
+	// ── emit ────────────────────────────────────────────────────────────────
+	// Everything fetched ships NOW — surface items are never withheld and the
+	// bg buffer drains into every page. Ordering across pages is the
+	// consumers' job (stable timestamp merge on prepend), which is what
+	// removed the old tiling/withholding machinery: it delayed the
+	// CONVERSATION behind a full bg drain on every first visit.
+	const emitSurface: any[] = surfaceList;
+	state.surfaceCarry = [];
+	const emitBg: any[] = state.bgBuffer;
+	state.bgBuffer = [];
+
+	// Dedup by item id, surface copy (full bodies) winning — the old-backend
+	// degradation path, and a guard against any range overlap.
+	const seen: { [id: string]: boolean } = {};
+	for (const it of emitSurface) { if (it && typeof it.id === 'string') seen[it.id] = true; }
+	const merged = emitSurface.concat(emitBg.filter((it: any) => !(it && typeof it.id === 'string' && seen[it.id])));
+
+	// ── deliver ─────────────────────────────────────────────────────────────
+	// Same head-refresh rule as the deferred branch: page 1's own "more pages
+	// exist" must not clobber a preserved end-knowledge.
+	if (!headRefresh) state.surfaceEnd = surface.endOfList;
+	state.lastSurfaceKeys = surface.startKeyHistory;
+	state.pendingSurface = null;
+
+	return {
+		list: merged,
+		endOfList: state.surfaceEnd && state.bgEnd && state.bgBuffer.length === 0 && state.surfaceCarry.length === 0,
+		// Bookkeeping only (both the consumers and the SDK treat it opaquely);
+		// the real cursors are the SDK's internal ones plus this module's state.
+		startKeyHistory: surface.startKeyHistory,
+		firstLoad,
+	};
+}
+
+
 export type MapHistoryOptions = {
 	clearedAt: number;
 	projectId: string;
@@ -100,9 +607,19 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 		var isFailed = item && item.status === 'failed';
 		var response = isFailed ? (item.error != null ? item.error : item.response_body)
 			: (item && item.response_body != null ? item.response_body : item && item.error);
-		var userText = extractLastUserTextFromRequest(requestBody);
-		var assistantText = isPending ? '' : ((extractAssistantText(response) || '').trim() || '');
-		var isErrorResponse = !isPending && (isFailed || isErrorResponseBody(response));
+		// COMPACT listing stubs (bg-queue pages fetched with `compact: true`):
+		// bodies never left the server; the label line, the response head, and
+		// the completion-marker bit arrive as dedicated fields. Everything the
+		// COLLAPSED row needs is here; expanding a row fetches the real bodies
+		// per item (buildHistoryItemFullId point lookups) and remaps.
+		var isCompact = !!(item && item.compact);
+		var userText = isCompact
+			? (typeof item.request_text === 'string' ? item.request_text : '')
+			: extractLastUserTextFromRequest(requestBody);
+		var assistantText = isPending ? '' : (isCompact
+			? ((typeof item.response_text === 'string' ? item.response_text : '').trim())
+			: ((extractAssistantText(response) || '').trim() || ''));
+		var isErrorResponse = !isPending && (isFailed || (!isCompact && isErrorResponseBody(response)));
 		// Record the completion marker, then STRIP it — both, and in that order.
 		// Recording gives the display layer a structured signal instead of a substring
 		// search over model prose. Stripping matches the live resolution path: without
@@ -112,8 +629,12 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 		//
 		// Gated on _isBgTask: only an INDEXING pass has a protocol token to hide. An
 		// ordinary reply that merely mentions it keeps its own words.
-		var reportedComplete = !!(item && item._isBgTask) && !isErrorResponse && !!assistantText &&
-			assistantText.indexOf(INDEXING_COMPLETE_MARKER) !== -1;
+		var reportedComplete = !!(item && item._isBgTask) && !isErrorResponse && (isCompact
+			? item.response_complete_marker === true
+			: (!!assistantText && assistantText.indexOf(INDEXING_COMPLETE_MARKER) !== -1));
+		// Still gated on the recorded marker (i.e. an INDEXING pass): an ordinary
+		// reply that merely mentions the token keeps its own words. A compact
+		// head can carry the literal token too, so the strip covers both forms.
 		if (reportedComplete) assistantText = assistantText.split(INDEXING_COMPLETE_MARKER).join('').trim();
 		var serverItemId = item && typeof item.id === 'string' && item.id ? item.id : undefined;
 		// A USER bubble shows when the request was made (`created`); an ASSISTANT
@@ -152,9 +673,11 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 				displayContent = sanitizeAttachmentLinksForHistory(userText, opts.projectId);
 			}
 			var userMsg: any = { role: 'user', content: displayContent };
+			if (item._fromBgChain) userMsg._fromBgChain = true;
 			if (isInProcess) userMsg.isPendingInProcess = true;
 			if (isQueued) userMsg.isPendingQueued = true;
 			if (isCancelledItem) userMsg.isCancelled = true;
+			if (isCompact) userMsg._compact = true;
 			if (item._isBgTask) userMsg.isBackgroundTask = true;
 			if (indexFile) userMsg._indexFile = indexFile;
 			if (item._isOnBgQueue) userMsg._useBgQueue = true;
@@ -165,12 +688,17 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 		if (isCancelledItem) { /* no assistant bubble */ }
 		else if (isInProcess) {
 			var ph: any = { role: 'assistant', content: '', isPending: true, isPendingInProcess: true };
+			// _ts matters even on a placeholder: the consumers' merge and any
+			// display fallback must be able to place it without its user bubble.
+			if (userTs !== undefined) ph._ts = userTs;
+			if (item._fromBgChain) ph._fromBgChain = true;
 			if (item._isBgTask) ph.isBackgroundTask = true;
 			if (serverItemId !== undefined) { ph._serverItemId = serverItemId; runningItemIds.push(serverItemId); }
 			mapped.push(ph);
 		} else if (isQueued) { /* no assistant placeholder */ }
 		else if (isErrorResponse) {
 			var em: any = { role: 'assistant', content: getErrorMessage(response), isError: true };
+			if (item._fromBgChain) em._fromBgChain = true;
 			if (item._isBgTask) em.isBackgroundTask = true;
 			if (serverItemId !== undefined) em._serverItemId = serverItemId;
 			if (replyTs !== undefined) em._ts = replyTs;
@@ -183,12 +711,24 @@ export function mapHistoryListToMessages(list: any[], platform: 'claude' | 'open
 			// Safe db-only sanitize (forAssistant) so a volatile db url the model
 			// emitted renders as a re-mintable `_expired_.url` link, not a dead one.
 			var okm: any = { role: 'assistant', content: sanitizeAttachmentLinksForHistory(assistantText, opts.projectId, true) || EMPTY_INDEXING_REPLY };
+			if (item._fromBgChain) okm._fromBgChain = true;
 			if (item._isBgTask) okm.isBackgroundTask = true;
+			if (isCompact) okm._compact = true;
 			if (serverItemId !== undefined) okm._serverItemId = serverItemId;
 			if (replyTs !== undefined) okm._ts = replyTs;
 			if (reportedComplete) okm._indexComplete = true;
 			mapped.push(okm);
 		}
 	});
+	// Stamp the CHAT every bubble belongs to. Every cross-project guard in the
+	// consumers reads `_ownerKey` and is written "undefined means unknown, keep
+	// it" — and the mapper never set it, so SERVER-history bubbles (which is all
+	// of them, including every indexing row) passed every one of those filters
+	// unchallenged. That is what let one project's transcript survive on screen
+	// into another project and be persisted under its key.
+	if (opts.projectId) {
+		var ownerKey = opts.projectId + '#' + platform;
+		for (var oi = 0; oi < mapped.length; oi++) mapped[oi]._ownerKey = ownerKey;
+	}
 	return { messages: mapped, runningItemIds: runningItemIds };
 }

package/src/engine/host.ts

@@ -97,6 +97,10 @@ export interface ChatMessage {
 	 *  which is how an 88-page file once "finished" at page 15. */
 	_indexComplete?: boolean;
 	_useBgQueue?: boolean;
+	/** Mapped from an item delivered by the bg chain of the split history fetch
+	 *  (stubs or deferred chats). Surface-frontier logic (retention boundary,
+	 *  clear-horizon) skips these — their ids reach arbitrarily deep. */
+	_fromBgChain?: boolean;
 	/** Local id of a turn STAGED at Send time while its attachments upload. The
 	 *  bubble exists before any server request does, so it is never matched by
 	 *  _serverItemId and is never promoted/cancelled by the queue machinery —
@@ -143,6 +147,9 @@ export interface ChatState {
 	typingAbort: boolean;
 	loadingHistory: boolean;
 	loadingOlderHistory: boolean;
+	/** A deferred bg stub batch (first-paint split fetch) is still in flight;
+	 *  views show a small 'loading indexing history' hint while true. */
+	bgHistoryLoading: boolean;
 	historyEndOfList: boolean;
 	historyStartKeyHistory: string[];
 	historyRequestToken: number;