onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/retrieval/compact.js

@@ -0,0 +1,810 @@
+/**
+ * Slice 1: Compact retrieval contract.
+ *
+ * Turns reranked results into a compact DTO that is safe to return to
+ * agents: bounded snippet/summary, no raw content by default, and a
+ * self-describing `expand` pointer for `detail=full` retrieval.
+ *
+ * Locked caps:
+ * - snippet  <= 400 characters
+ * - summary  <= 800 characters
+ */
+import { getTable, TABLE_NAMES } from "../storage/tables.js";
+import { tokenize } from "./lexical.js";
+import { extractPassages, passageQueryTerms } from "./passages.js";
+import { parentKeyForRow, readChunkMarker, } from "../ingestion/chunk-meta.js";
+import { rankedKey } from "./group-by-parent.js";
+import { stitchChunkContents } from "./stitch-chunks.js";
+import { eqFilter, inFilter } from "../storage/filter.js";
+import { stableEvidenceIdentityForRow } from "./evidence-key.js";
+export const SNIPPET_MAX = 400;
+export const SUMMARY_MAX = 800;
+export const TITLE_MAX = 120;
+/**
+ * Token threshold above which `renderCompactMarkdown` annotates a hit with a
+ * compact `~Nk tokens` warning. The point is to make the agent see the size
+ * before it commits to a `detail=full` expansion that would blow context.
+ */
+export const LARGE_HIT_TOKEN_THRESHOLD = 10_000;
+/**
+ * Middle-detail excerpt caps (characters, matching SNIPPET_MAX / SUMMARY_MAX).
+ * Targets ~2k–4k tokens total for an evidence-focused view that sits between
+ * `summary` (≤800 chars) and `full` (entire row content):
+ *
+ * - rows at or below MIDDLE_INLINE_THRESHOLD_CHARS get the full content inline
+ *   (the excerpt would be the same content with no truncation, so skip the
+ *   elision overhead);
+ * - larger rows get the first MIDDLE_HEAD_CHARS + last MIDDLE_TAIL_CHARS with
+ *   a single elision marker between, preserving the bookends most useful for
+ *   fact-checking (initial question, final answer/decision) while bounding
+ *   cost.
+ *
+ * Char-based bounds are an intentional approximation: for typical mixed text,
+ * 1 token ≈ 4 chars, so 8000 head + 4000 tail ≈ 3000 tokens.
+ */
+export const MIDDLE_HEAD_CHARS = 8_000;
+export const MIDDLE_TAIL_CHARS = 4_000;
+export const MIDDLE_INLINE_THRESHOLD_CHARS = 12_000;
+/**
+ * Neighbor chunks to include on each side of the matched chunk for a
+ * `detail="middle"` expand of a chunked source. The window is `[i-K, i+K]`
+ * clamped to the set, so K=1 yields up to 3 contiguous chunks (~5k chars /
+ * ~1.4k tokens at the default 1800-char target) — enough surrounding context to
+ * read the matched section in place without re-dumping the whole conversation.
+ */
+export const MIDDLE_CHUNK_NEIGHBORS = 1;
+/**
+ * Pre-call size advert for an expansion: every compact hit carries this on
+ * both the top-level fields and inside `expand.params`, so an agent that
+ * copy-pastes the expand pointer verbatim still sees the size warning at the
+ * exact moment it chooses whether to spend context on `detail="full"`.
+ */
+export function computeContentSize(content) {
+    const content_bytes = Buffer.byteLength(content, "utf8");
+    const content_tokens_estimate = Math.ceil(content_bytes / 4);
+    return { content_bytes, content_tokens_estimate };
+}
+/**
+ * Build the middle-detail content excerpt for a row body.
+ *
+ * - For content at or below MIDDLE_INLINE_THRESHOLD_CHARS, returns the content
+ *   as-is (no truncation, `truncated: false`) — at that size the head+tail
+ *   excerpt would equal the input anyway, and skipping the elision marker
+ *   keeps the response identical to detail="full" for small rows.
+ * - For larger content, returns `head + elision marker + tail` where head is
+ *   the first MIDDLE_HEAD_CHARS and tail is the last MIDDLE_TAIL_CHARS. The
+ *   marker reports how many characters were elided so a reader knows the gap
+ *   size without having to subtract.
+ */
+export function buildMiddleExcerpt(content, query) {
+    if (content.length <= MIDDLE_INLINE_THRESHOLD_CHARS) {
+        return { text: content, truncated: false };
+    }
+    if (query) {
+        const anchored = buildQueryAnchoredExcerpt(content, query);
+        if (anchored)
+            return { text: anchored, truncated: true };
+    }
+    const head = content.slice(0, MIDDLE_HEAD_CHARS);
+    const tail = content.slice(content.length - MIDDLE_TAIL_CHARS);
+    const elidedChars = content.length - MIDDLE_HEAD_CHARS - MIDDLE_TAIL_CHARS;
+    const elidedTokens = Math.ceil(Buffer.byteLength(content, "utf8") / 4);
+    const marker = `\n\n[…${elidedChars.toLocaleString("en-US")} chars elided — full row is ~${formatTokensShort(elidedTokens)} tokens, request detail="full" to expand…]\n\n`;
+    return { text: head + marker + tail, truncated: true };
+}
+/** Query-anchored middle excerpt: opening, then the best-matching spans. */
+const ANCHORED_HEAD_CHARS = 2_000;
+const ANCHORED_PASSAGE_CHARS = 3_000;
+const ANCHORED_PASSAGES = 3;
+function buildQueryAnchoredExcerpt(content, query) {
+    const passages = extractPassages(content, query, {
+        max: ANCHORED_PASSAGES,
+        windowChars: ANCHORED_PASSAGE_CHARS,
+    }).filter((p) => p.end > ANCHORED_HEAD_CHARS);
+    if (passages.length === 0)
+        return null;
+    const parts = [content.slice(0, ANCHORED_HEAD_CHARS)];
+    let cursor = ANCHORED_HEAD_CHARS;
+    for (const p of passages) {
+        const start = Math.max(p.start, cursor);
+        if (start >= p.end)
+            continue;
+        parts.push(`\n\n[…${(start - cursor).toLocaleString("en-US")} chars elided — match at char ${start.toLocaleString("en-US")}…]\n\n`, content.slice(start, p.end));
+        cursor = p.end;
+    }
+    const rest = content.length - cursor;
+    if (rest > 0) {
+        parts.push(`\n\n[…${rest.toLocaleString("en-US")} chars to the end — request detail="full" for the whole row…]`);
+    }
+    return parts.join("");
+}
+function formatTokensShort(tokens) {
+    if (tokens >= 1000)
+        return `${Math.round(tokens / 1000)}k`;
+    return String(tokens);
+}
+/**
+ * Pointer fields that let an agent re-cite this exact evidence for retrieval
+ * feedback without the route having to re-derive identity from a stale/partial
+ * pointer. `stable_evidence_key` is authoritative; `source` + `parent_id` are
+ * carried so conversation identity survives even if the key is ever dropped.
+ * Empty when no backing row is available (the expand path can still fetch by id).
+ */
+function evidencePointerParams(sourceTable, row) {
+    if (!row)
+        return {};
+    const identity = stableEvidenceIdentityForRow(sourceTable, row);
+    const source = row["source"];
+    const parentId = row["parent_id"];
+    return {
+        stable_evidence_key: identity.stable_evidence_key,
+        ...(typeof source === "string" && source ? { source } : {}),
+        ...(typeof parentId === "string" && parentId ? { parent_id: parentId } : {}),
+    };
+}
+/**
+ * Batch-fetch the raw source rows for reranked results.
+ * Groups ids by source table and issues one `id IN (...)` query per table.
+ *
+ * Keyed by {@link rankedKey} — `(source_table, id)`, NOT bare id — so a row id
+ * shared across two tables (cloud vs local docs, hot vs cold) can never have one
+ * table's row overwrite the other's and produce a dishonest snippet/title/source
+ * for the surviving hit. Callers must look up via `get(rankedKey(r))`.
+ */
+export async function fetchRawRowsForRanked(ranked) {
+    const byTable = new Map();
+    for (const r of ranked) {
+        const arr = byTable.get(r.source_table) ?? [];
+        arr.push(r.id);
+        byTable.set(r.source_table, arr);
+    }
+    const out = new Map();
+    for (const [tableName, ids] of byTable) {
+        if (ids.length === 0)
+            continue;
+        const rows = (await getTable(tableName)
+            .query()
+            .where(inFilter("id", ids, { validateAsRowId: true }))
+            .limit(ids.length)
+            .toArray());
+        for (const row of rows) {
+            out.set(rankedKey({ source_table: tableName, id: row.id }), row);
+        }
+    }
+    return out;
+}
+/**
+ * The minimal column projections each table family needs for
+ * {@link parentKeyForRow}. Deliberately excludes `content` and the embedding
+ * vector: chunk-grouping must inspect the WHOLE reranked pool (siblings beyond
+ * top_k still have to collapse before the top_k slice), and pulling full rows —
+ * embeddings included — for the entire deep pool on every query was a real
+ * hot-path regression. These projections let us key the pool cheaply, then fetch
+ * full rows only for the surviving representatives.
+ */
+const CONV_MARKER_COLUMNS = [
+    "id",
+    "parent_id",
+    "chunk_index",
+    "chunk_count",
+    "source",
+];
+const DOC_MARKER_COLUMNS = ["id", "source_path", "metadata"];
+/**
+ * Compute the parent grouping key for every reranked result using a
+ * projection-only query (no `content`, no embedding vector) per source table.
+ *
+ * Returns `rankedKey(r) → parentKey | null`, keyed by `(source_table, id)` so a
+ * row id shared across two tables (cloud vs local docs, hot vs cold) can never
+ * collide into one entry. `null` for any row with no explicit chunk marker (the
+ * flag-off / legacy case — never grouped) and for any id with no backing row.
+ * Digests never chunk, so their ids map to `null` without a query. Pair with
+ * {@link groupByParent} (which looks up via {@link rankedKey}), then call
+ * {@link fetchRawRowsForRanked} on the post-group top-k representatives only.
+ */
+export async function fetchParentKeysForRanked(ranked) {
+    const byTable = new Map();
+    for (const r of ranked) {
+        const arr = byTable.get(r.source_table) ?? [];
+        arr.push(r.id);
+        byTable.set(r.source_table, arr);
+    }
+    const out = new Map();
+    for (const [tableName, ids] of byTable) {
+        if (ids.length === 0)
+            continue;
+        const keyOf = (id) => rankedKey({ source_table: tableName, id });
+        // Digests can never be chunked — skip the query entirely.
+        if (tableName === "digests") {
+            for (const id of ids)
+                out.set(keyOf(id), null);
+            continue;
+        }
+        const columns = tableName === "conversations_hot" || tableName === "conversations_cold"
+            ? CONV_MARKER_COLUMNS
+            : DOC_MARKER_COLUMNS;
+        const rows = (await getTable(tableName)
+            .query()
+            .where(inFilter("id", ids, { validateAsRowId: true }))
+            .select([...columns])
+            .limit(ids.length)
+            .toArray());
+        for (const row of rows) {
+            const id = row.id;
+            out.set(keyOf(id), parentKeyForRow(tableName, row));
+        }
+        // Any id with no returned row reads as ungrouped.
+        for (const id of ids)
+            if (!out.has(keyOf(id)))
+                out.set(keyOf(id), null);
+    }
+    return out;
+}
+/** Derive the compact `tier` label from source_table (+ source_scope for docs). */
+export function deriveTier(sourceTable, row) {
+    switch (sourceTable) {
+        case "conversations_hot":
+            return "hot";
+        case "conversations_cold":
+            return "cold";
+        case "digests":
+            return "digest";
+        case "structured_docs":
+        case "structured_docs_local": {
+            const scope = row?.source_scope;
+            return scope === "document" ? "document" : "structured_doc";
+        }
+    }
+}
+/**
+ * Candidate source tables that can hold a row with the given compact tier.
+ * The `document` and `structured_doc` tiers map to either the cloud or local
+ * structured-docs table — callers should try them in order and keep the first hit.
+ */
+export function tablesForTier(tier) {
+    switch (tier) {
+        case "hot":
+            return ["conversations_hot"];
+        case "cold":
+            return ["conversations_cold"];
+        case "digest":
+            return ["digests"];
+        case "document":
+        case "structured_doc":
+            return ["structured_docs", "structured_docs_local"];
+    }
+}
+/**
+ * Fetch a single row by id for the id-based expansion path on /search.
+ *
+ * When `sourceTableHint` is provided, only that table is consulted — this is
+ * the authoritative path for compact hits, which carry their backing table
+ * identity in `expand.params.source_table` precisely to disambiguate rows
+ * that may share an id across the cloud / local structured-doc tables.
+ *
+ * When `sourceTableHint` is omitted, falls back to iterating the candidate
+ * tables for the tier (first-match wins). This fallback is only used for
+ * hand-crafted expansion requests that don't carry `source_table`.
+ */
+export async function fetchRowByIdForTier(id, tier, sourceTableHint) {
+    const candidates = sourceTableHint
+        ? [sourceTableHint]
+        : tablesForTier(tier);
+    for (const tableName of candidates) {
+        const rows = (await getTable(tableName)
+            .query()
+            // `id` is user-supplied via /search → escape only (no charset throw, so a
+            // malformed id stays a clean 404 rather than a 500).
+            .where(eqFilter("id", id))
+            .limit(1)
+            .toArray());
+        if (rows.length > 0) {
+            return { row: rows[0], sourceTable: tableName };
+        }
+    }
+    return null;
+}
+/**
+ * Fetch every row matching `id` across all supported tables.
+ *
+ * Used by the no-tier exact-ID path on /search: callers that pass only `id`
+ * deserve a deterministic answer — exactly one row, an explicit not-found,
+ * or an explicit ambiguous-ID error. Never silently fall back to ranked
+ * search.
+ *
+ * Returns an array of {row, sourceTable} hits — empty if nothing matches.
+ */
+export async function fetchRowsByIdAcrossTables(id) {
+    return fetchRowsByIdInTables(id, TABLE_NAMES);
+}
+/**
+ * Fetch every row matching `id` across the given tables. Used by the
+ * tier-without-source_table path: when a tier maps to multiple backing
+ * tables (document / structured_doc → cloud + local), callers deserve the
+ * same ambiguity guarantee as the bare-id path rather than silent
+ * first-match-wins.
+ */
+export async function fetchRowsByIdInTables(id, tables) {
+    const matches = [];
+    for (const tableName of tables) {
+        const rows = (await getTable(tableName)
+            .query()
+            // `id` is user-supplied via /search → escape only (see fetchRowByIdForTier).
+            .where(eqFilter("id", id))
+            .limit(2)
+            .toArray());
+        for (const row of rows) {
+            matches.push({ row, sourceTable: tableName });
+        }
+    }
+    return matches;
+}
+/**
+ * Build a CompactHit directly from a raw row (bypassing retrieval/ranking).
+ * Used by the id-based expansion path: score is fixed at 1.0 and
+ * retrieval_method is labelled "direct" to signal a non-search fetch.
+ */
+export function buildCompactHitFromRow(row, sourceTable, opts) {
+    const tier = deriveTier(sourceTable, row);
+    const content = row.content || "";
+    const summary = deriveSummary(row, content);
+    const title = deriveTitle(sourceTable, row, content, summary);
+    const snippet = deriveSnippet(content, opts.query ?? "", "direct");
+    const timestamp = row.indexed_at ||
+        row.created_at ||
+        "";
+    const id = row.id;
+    const { content_bytes, content_tokens_estimate } = computeContentSize(content);
+    const detail = opts.detail ?? "full";
+    const [passage] = extractPassages(content, opts.query ?? "", { max: 1, windowChars: SNIPPET_MAX });
+    const hit = {
+        id,
+        source: deriveSource(sourceTable, row),
+        title,
+        score: 1.0,
+        retrieval_method: "direct",
+        timestamp,
+        tier,
+        snippet,
+        summary,
+        content_bytes,
+        content_tokens_estimate,
+        expand: {
+            route: opts.expandRoute ?? "/search",
+            params: {
+                id,
+                tier,
+                source_table: sourceTable,
+                detail: "full",
+                content_bytes,
+                content_tokens_estimate,
+                ...evidencePointerParams(sourceTable, row),
+            },
+        },
+    };
+    if (opts.includePassage) {
+        hit.match_terms = passageQueryTerms(opts.query ?? "");
+        if (passage) {
+            hit.passage_start = passage.start;
+            hit.passage_end = passage.end;
+        }
+    }
+    if (detail === "full") {
+        hit.content = content;
+    }
+    else if (detail === "middle") {
+        const excerpt = buildMiddleExcerpt(content, opts.query);
+        hit.content = excerpt.text;
+        if (excerpt.truncated)
+            hit.content_truncated = true;
+    }
+    return hit;
+}
+/** Projection columns needed to stitch a conversation / document chunk set. */
+const CONV_STITCH_COLUMNS = [
+    "id",
+    "content",
+    "parent_id",
+    "chunk_index",
+    "chunk_count",
+    "source",
+];
+const DOC_STITCH_COLUMNS = [
+    "id",
+    "content",
+    "metadata",
+    "source_path",
+];
+/**
+ * Fetch the WHOLE parent chunk set of `matchedRow`. Reads only the columns
+ * needed to reassemble content (never the embedding vector). Best-effort: a
+ * sibling missing from the table (a partially-written set) simply doesn't appear,
+ * and the stitcher renders the gap honestly.
+ *
+ * The caller always needs the full set: even a bounded `middle` window must
+ * advertise the full-parent size on its `detail="full"` escalation pointer, so we
+ * stitch the whole set for that advert and a sub-window for the returned body.
+ * Server-side IO/memory therefore scales with parent length regardless of detail
+ * — fine for the bounded response, but not free for very long Codex/Claude
+ * threads. We read each sibling's ordinal from the authoritative column /
+ * metadata because that is simpler and more robust than an index-range predicate
+ * over the stringified `chunk_index`; a range/ID-targeted fetch is the follow-up
+ * if long-parent expand cost shows up in practice.
+ */
+async function fetchChunkSiblings(sourceTable, matchedRow, marker) {
+    const isConv = sourceTable === "conversations_hot" || sourceTable === "conversations_cold";
+    const limit = marker.count + 4; // small slack to surface any stray duplicate
+    const where = isConv
+        ? // A conversation chunk set shares parent_id AND source (the namespaced
+            // retrieval key includes source), so scope to both.
+            `${eqFilter("parent_id", marker.parentId)} AND ${eqFilter("source", typeof matchedRow["source"] === "string" ? matchedRow["source"] : "")}`
+        : eqFilter("source_path", marker.parentId);
+    const rows = (await getTable(sourceTable)
+        .query()
+        .where(where)
+        .select([...(isConv ? CONV_STITCH_COLUMNS : DOC_STITCH_COLUMNS)])
+        .limit(limit)
+        .toArray());
+    const out = [];
+    for (const row of rows) {
+        const m = readChunkMarker(sourceTable, row);
+        if (!m)
+            continue;
+        out.push({ index: m.index, content: row.content ?? "" });
+    }
+    return out;
+}
+/**
+ * Build a CompactHit for the exact-ID expand path, reassembling chunk siblings
+ * when the matched row belongs to a chunk set.
+ *
+ * - Unchunked rows, and any `detail="summary"` request, are returned verbatim by
+ *   {@link buildCompactHitFromRow} — so with `RIFT_CHUNKING` off (no marked rows)
+ *   this is provably the pre-stitching behavior.
+ * - For a chunked row, `detail="full"` reassembles the whole parent in
+ *   `chunk_index` order; `detail="middle"` reassembles the matched chunk plus a
+ *   bounded neighbor window (`MIDDLE_CHUNK_NEIGHBORS` each side). Both ranges are
+ *   contiguous, and seams are de-overlapped so duplicated text appears once.
+ *
+ * The representative identity (id, title, source, timestamp, snippet, score)
+ * comes from the matched row. `content` and the TOP-LEVEL size fields are
+ * replaced with the stitched body actually returned, while the `expand.params`
+ * size fields advertise the FULL parent (what the `detail="full"` escalation
+ * pointer would return), plus `chunk_count` / `stitched_chunks` for observability.
+ */
+export async function buildExpandedHit(row, sourceTable, opts) {
+    const detail = opts.detail ?? "full";
+    const marker = readChunkMarker(sourceTable, row);
+    if (!marker || detail === "summary") {
+        return buildCompactHitFromRow(row, sourceTable, opts);
+    }
+    // Fetch the whole parent set once, then stitch it twice: the full ordered set
+    // sizes the `detail="full"` escalation pointer (expand.params), and a bounded
+    // sub-window is the body we actually return for `detail="middle"`.
+    const allSiblings = await fetchChunkSiblings(sourceTable, row, marker);
+    // Expected range = the whole parent [0, count-1]; passing it lets the stitcher
+    // mark a missing leading/trailing section (a partially-written set) instead of
+    // returning a partial body that reads as the complete parent.
+    const fullStitched = stitchChunkContents(allSiblings, {
+        trackPositions: !!opts.includePassage,
+        expectedStart: 0,
+        expectedEnd: marker.count - 1,
+    });
+    let returned = fullStitched;
+    if (detail === "middle") {
+        const start = Math.max(0, marker.index - MIDDLE_CHUNK_NEIGHBORS);
+        const end = Math.min(marker.count - 1, marker.index + MIDDLE_CHUNK_NEIGHBORS);
+        // Expected range = the requested window [start, end]; a sibling missing
+        // inside it (including its first/last index) renders as an omission marker.
+        // The deliberately out-of-window chunks are not "missing" and aren't marked.
+        returned = stitchChunkContents(allSiblings.filter((c) => c.index >= start && c.index <= end), { expectedStart: start, expectedEnd: end, trackPositions: !!opts.includePassage });
+    }
+    // Reuse all the title/source/snippet/timestamp derivation from the matched
+    // row, but start from "summary" so buildCompactHitFromRow doesn't also emit
+    // the single-chunk content — we replace content + size with the stitched body.
+    const hit = buildCompactHitFromRow(row, sourceTable, {
+        ...opts,
+        detail: "summary",
+    });
+    // Top-level size = the body actually returned (middle window or full parent).
+    // The escalation pointer keeps detail="full" (set by buildCompactHitFromRow)
+    // and must advertise the FULL parent body re-expanding would return — so a
+    // bounded `middle` hit never under-states the cost of escalating to `full`.
+    const returnedSize = computeContentSize(returned.text);
+    const fullSize = computeContentSize(fullStitched.text);
+    hit.content = returned.text;
+    // Keep the passage of the ranked chunk. Searching the whole parent again
+    // can select an earlier duplicate that was never shown in the result row.
+    delete hit.passage_start;
+    delete hit.passage_end;
+    const [passage] = extractPassages(row.content ?? "", opts.query ?? "", { max: 1, windowChars: SNIPPET_MAX });
+    const chunkStart = returned.chunkStarts?.get(marker.index);
+    if (opts.includePassage && passage && chunkStart !== undefined) {
+        hit.passage_start = chunkStart + passage.start;
+        hit.passage_end = chunkStart + passage.end;
+    }
+    hit.content_bytes = returnedSize.content_bytes;
+    hit.content_tokens_estimate = returnedSize.content_tokens_estimate;
+    hit.expand.params["content_bytes"] = fullSize.content_bytes;
+    hit.expand.params["content_tokens_estimate"] = fullSize.content_tokens_estimate;
+    hit.chunk_count = marker.count;
+    hit.stitched_chunks = returned.sectionsUsed;
+    // A returned body that omits any sibling is an incomplete view of the parent.
+    // `content_truncated` is the existing contract downstream telemetry/agents use
+    // to know the body is partial: a bounded `middle` window trips it, and so does
+    // a `full` reassembly missing a sibling (a partially-written set). When a small
+    // parent's window happens to cover every chunk, sectionsUsed === count and it
+    // is correctly NOT flagged.
+    if (returned.sectionsUsed < marker.count)
+        hit.content_truncated = true;
+    return hit;
+}
+/** Derive a stable `source` string (what produced this row). */
+export function deriveSource(sourceTable, row) {
+    if (sourceTable === "conversations_hot" || sourceTable === "conversations_cold") {
+        return row?.source || "conversation";
+    }
+    if (sourceTable === "digests")
+        return "digest";
+    return row?.source_type || "filesystem";
+}
+/** Derive a compact title. Prefers stored metadata, then first non-empty line. */
+export function deriveTitle(sourceTable, row, content, summary) {
+    // Explicit stored title in metadata JSON takes precedence.
+    const metaTitle = readMetadataTitle(row);
+    if (metaTitle)
+        return truncate(metaTitle, TITLE_MAX);
+    if (sourceTable === "digests" && row) {
+        const start = row.period_start || "";
+        const end = row.period_end || "";
+        if (start || end)
+            return truncate(`Digest ${start}→${end}`, TITLE_MAX);
+    }
+    // Basename of a document path is a decent fallback for structured_docs.
+    if (sourceTable === "structured_docs" || sourceTable === "structured_docs_local") {
+        const p = row?.source_path;
+        if (p) {
+            const base = p.split("/").filter(Boolean).pop();
+            if (base)
+                return truncate(base, TITLE_MAX);
+        }
+    }
+    const firstLine = firstNonEmptyLine(summary) || firstNonEmptyLine(content);
+    if (firstLine)
+        return truncate(firstLine, TITLE_MAX);
+    return "(untitled)";
+}
+/**
+ * Derive a bounded snippet.
+ * - lexical/hybrid: up to 400 chars centered on the best query-term hit.
+ * - vector-only: first 400 chars of the first user turn (or content).
+ */
+export function deriveSnippet(content, query, retrievalMethod) {
+    if (!content)
+        return "";
+    if (retrievalMethod === "lexical" || retrievalMethod === "hybrid") {
+        const centered = centerOnQueryMatch(content, query);
+        if (centered !== null)
+            return truncate(centered, SNIPPET_MAX);
+    }
+    const firstTurn = extractFirstUserTurn(content);
+    return truncate(collapseWhitespace(firstTurn || content), SNIPPET_MAX);
+}
+/** Derive a bounded summary. Prefers stored summary, then content fallback. */
+export function deriveSummary(row, content) {
+    const stored = row?.summary ?? "";
+    if (stored.trim())
+        return truncate(collapseWhitespace(stored), SUMMARY_MAX);
+    const metaSummary = readMetadataSummary(row);
+    if (metaSummary)
+        return truncate(collapseWhitespace(metaSummary), SUMMARY_MAX);
+    const firstTurn = extractFirstUserTurn(content);
+    return truncate(collapseWhitespace(firstTurn || content), SUMMARY_MAX);
+}
+export function buildCompactHit(ranked, row, opts) {
+    const tier = deriveTier(ranked.source_table, row);
+    const content = ranked.content || row?.content || "";
+    const summary = deriveSummary(row, content);
+    const title = deriveTitle(ranked.source_table, row, content, summary);
+    // Best query-matching span, with no speaker attribution.
+    const [best] = extractPassages(content, opts.query, { max: 1, windowChars: SNIPPET_MAX });
+    const snippet = best
+        ? truncate(best.text, SNIPPET_MAX)
+        : deriveSnippet(content, opts.query, ranked.retrieval_method);
+    const timestamp = row?.indexed_at ||
+        row?.created_at ||
+        "";
+    const { content_bytes, content_tokens_estimate } = computeContentSize(content);
+    const hit = {
+        id: ranked.id,
+        source: deriveSource(ranked.source_table, row),
+        title,
+        score: ranked.score,
+        retrieval_method: ranked.retrieval_method,
+        timestamp,
+        tier,
+        snippet,
+        summary,
+        content_bytes,
+        content_tokens_estimate,
+        expand: {
+            route: opts.expandRoute ?? "/search",
+            // `source_table` is carried so the expansion path can fetch the exact
+            // backing row without guessing — critical for `document`/`structured_doc`
+            // tiers, where the same id may exist in both cloud and local tables.
+            //
+            // `content_bytes` and `content_tokens_estimate` are mirrored here so an
+            // agent that copy-pastes the `expand` pointer verbatim sees the size
+            // warning at the moment it chooses to spend context on `detail="full"`.
+            params: {
+                id: ranked.id,
+                tier,
+                source_table: ranked.source_table,
+                detail: "full",
+                content_bytes,
+                content_tokens_estimate,
+                ...evidencePointerParams(ranked.source_table, row),
+            },
+        },
+    };
+    if (opts.includePassage) {
+        hit.match_terms = passageQueryTerms(opts.query);
+        if (best) {
+            hit.passage_start = best.start;
+            hit.passage_end = best.end;
+        }
+    }
+    if (opts.detail === "full") {
+        hit.content = content;
+    }
+    else if (opts.detail === "middle") {
+        const excerpt = buildMiddleExcerpt(content, opts.query);
+        hit.content = excerpt.text;
+        if (excerpt.truncated)
+            hit.content_truncated = true;
+    }
+    if (opts.siblingCount !== undefined && opts.siblingCount > 1) {
+        hit.matched_sections = opts.siblingCount;
+    }
+    if (opts.duplicateCount !== undefined && opts.duplicateCount > 1) {
+        hit.duplicate_count = opts.duplicateCount;
+    }
+    return hit;
+}
+// --- Markdown rendering (MCP text channel) ---
+/** Compact markdown rendering of a set of hits — no raw JSON. */
+export function renderCompactMarkdown(hits, opts = {}) {
+    const lines = [];
+    if (opts.degraded) {
+        lines.push(`> ⚠ degraded${opts.degradedReason ? `: ${opts.degradedReason}` : ""}`, "");
+    }
+    else if (opts.lexical) {
+        // Intentional no-key mode, not a failure — keyword/BM25 results by design.
+        lines.push("> ℹ Keyword search (no embedding key configured — semantic ranking off).", "");
+    }
+    if (hits.length === 0) {
+        lines.push("_No results._");
+        return lines.join("\n");
+    }
+    hits.forEach((h, i) => {
+        const title = escapeMd(h.title ?? "(untitled)");
+        const tier = h.tier ?? "?";
+        const source = h.source === "grok_cli" ? "Grok CLI" : h.source ?? "?";
+        const score = typeof h.score === "number" ? h.score.toFixed(3) : "?";
+        const method = h.retrieval_method ?? "?";
+        const sizeWarning = formatSizeWarning(h.content_tokens_estimate);
+        const sections = typeof h.matched_sections === "number" && h.matched_sections > 1
+            ? ` · +${h.matched_sections - 1} more section${h.matched_sections - 1 === 1 ? "" : "s"}`
+            : "";
+        const duplicates = typeof h.duplicate_count === "number" && h.duplicate_count > 1
+            ? ` · +${h.duplicate_count - 1} duplicate capture${h.duplicate_count - 1 === 1 ? "" : "s"}`
+            : "";
+        lines.push(`${i + 1}. **${title}** — _${tier}_ · ${source} · score ${score} (${method})${sizeWarning ? ` · ${sizeWarning}` : ""}${sections}${duplicates}`);
+        if (h.snippet) {
+            lines.push(`   ${escapeMd(h.snippet)}`);
+        }
+        // Render the expand pointer with the *actual* params object the route
+        // expects, so the printed markdown is round-trippable — an agent can
+        // copy-paste `{ ... }` as the request body for the route.
+        const expandRoute = h.expand?.route ?? "/search";
+        const params = h.expand?.params ?? { id: h.id, detail: "full" };
+        lines.push(`   \`expand: ${expandRoute} ${JSON.stringify(params)}\``);
+    });
+    return lines.join("\n");
+}
+/**
+ * Compact size warning for large hits, e.g. `~82k tokens`. Returns the empty
+ * string for hits below `LARGE_HIT_TOKEN_THRESHOLD` so the markdown stays
+ * concise when expansion would be cheap.
+ */
+export function formatSizeWarning(tokensEstimate) {
+    if (typeof tokensEstimate !== "number" || tokensEstimate < LARGE_HIT_TOKEN_THRESHOLD) {
+        return "";
+    }
+    if (tokensEstimate >= 1000) {
+        return `~${Math.round(tokensEstimate / 1000)}k tokens`;
+    }
+    return `~${tokensEstimate} tokens`;
+}
+// --- Internals ---
+function truncate(text, max) {
+    if (text.length <= max)
+        return text;
+    // Hard cap at `max` to guarantee the locked contract; callers rely on this.
+    return text.slice(0, max);
+}
+function collapseWhitespace(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+function firstNonEmptyLine(text) {
+    for (const line of text.split(/\r?\n/)) {
+        const t = line.trim();
+        if (t) {
+            // Strip common markdown heading hashes and list bullets for titles.
+            return t.replace(/^#+\s*/, "").replace(/^[-*]\s+/, "");
+        }
+    }
+    return "";
+}
+function extractFirstUserTurn(content) {
+    // Best-effort: look for a "User:" / "Human:" marker at the start of a line
+    // followed by text up to the next role marker. Falls back to empty string.
+    const match = content.match(/(?:^|\n)\s*(?:user|human)\s*:\s*([\s\S]*?)(?:\n\s*(?:assistant|ai|model|system)\s*:|\n{2,}#|$)/i);
+    if (match && match[1])
+        return match[1].trim();
+    return "";
+}
+function centerOnQueryMatch(content, query) {
+    const terms = tokenize(query);
+    if (terms.length === 0)
+        return null;
+    const lower = content.toLowerCase();
+    let bestIdx = -1;
+    for (const t of terms) {
+        const idx = lower.indexOf(t.toLowerCase());
+        if (idx >= 0 && (bestIdx === -1 || idx < bestIdx))
+            bestIdx = idx;
+    }
+    if (bestIdx < 0)
+        return null;
+    const halfWindow = Math.floor(SNIPPET_MAX / 2);
+    const start = Math.max(0, bestIdx - halfWindow);
+    const end = Math.min(content.length, start + SNIPPET_MAX);
+    const adjustedStart = Math.max(0, end - SNIPPET_MAX);
+    const slice = content.slice(adjustedStart, end);
+    return collapseWhitespace(slice);
+}
+function readMetadataTitle(row) {
+    const meta = row?.metadata;
+    if (typeof meta !== "string" || !meta)
+        return null;
+    try {
+        const parsed = JSON.parse(meta);
+        const t = parsed["title"];
+        if (typeof t === "string" && t.trim())
+            return t.trim();
+    }
+    catch {
+        // ignore malformed metadata
+    }
+    return null;
+}
+function readMetadataSummary(row) {
+    const meta = row?.metadata;
+    if (typeof meta !== "string" || !meta)
+        return null;
+    try {
+        const parsed = JSON.parse(meta);
+        const s = parsed["summary"];
+        if (typeof s === "string" && s.trim())
+            return s.trim();
+    }
+    catch {
+        // ignore
+    }
+    return null;
+}
+function escapeMd(text) {
+    return text.replace(/\|/g, "\\|").replace(/\n+/g, " ");
+}
+//# sourceMappingURL=compact.js.map