@aitne/daemon 0.1.0

package/dist/core/docs/citation-validator.js

@@ -0,0 +1,195 @@
+/**
+ * Citation post-processor (DOCS_QA_DESIGN.md §9.6).
+ *
+ * The QA model is prompted to embed `[doc:slug#anchor]` tokens for every
+ * claim it makes. The renderer turns those into clickable pills, which
+ * means hallucinated citations would be silently rendered as broken
+ * links if no validation took place.
+ *
+ * This module is the validation layer. It is invoked twice in the daemon:
+ *
+ *   1. **Streaming (`createStreamingValidator`)** — wraps the SSE token
+ *      stream so each text delta passes through `feed()`, which buffers
+ *      only across the byte range of an in-progress `[doc:` token and
+ *      flushes everything else immediately. The output is the validated
+ *      stream the dashboard sees.
+ *
+ *   2. **Final pass (`validateAndRewrite`)** — applied to the full
+ *      assembled reply text before persisting to `messages.content`, so
+ *      what is stored matches what the dashboard rendered.
+ *
+ * Validation outcomes (per §9.6 table):
+ *   - slug + anchor both valid → forward unchanged
+ *   - slug valid, anchor missing → rewrite to `[doc:slug]`, tag pill
+ *     `anchor-not-found` for dashboard rendering
+ *   - slug invalid → strip the token entirely + log to `agent_actions`
+ */
+import { parseCitationTokens } from "@aitne/shared";
+import { createLogger } from "../../logging.js";
+const logger = createLogger("citation-validator");
+/**
+ * Lookup-by-DB factory. `fts_docs.anchors` stores anchors as a
+ * newline-joined string (the indexer flattens the array on insert), so
+ * we split on `\n` and trim. Returns `null` when no row matches the slug.
+ */
+export function makeDbLookup(db) {
+    return {
+        anchorsForSlug(slug) {
+            const row = db
+                .prepare("SELECT anchors FROM fts_docs WHERE slug = ?")
+                .get(slug);
+            if (!row)
+                return null;
+            return (row.anchors ?? "")
+                .split("\n")
+                .map((s) => s.trim())
+                .filter((s) => s.length > 0);
+        },
+    };
+}
+/**
+ * One-shot validator for already-assembled reply text. Returns the
+ * rewritten text plus per-category counts so callers can attach a
+ * `qa_invalid_citation` row to `agent_actions` when slug-missing
+ * tokens are present.
+ */
+export function validateAndRewrite(text, lookup) {
+    const tokens = parseCitationTokens(text);
+    if (tokens.length === 0) {
+        return { text, validCount: 0, anchorMissing: [], slugMissing: [] };
+    }
+    // Walk tokens in order, slicing the input around them.
+    const out = [];
+    const anchorMissing = [];
+    const slugMissing = [];
+    let validCount = 0;
+    let cursor = 0;
+    for (const tok of tokens) {
+        out.push(text.slice(cursor, tok.start));
+        cursor = tok.end;
+        const anchors = lookup.anchorsForSlug(tok.slug);
+        if (anchors === null) {
+            // Unknown slug — strip entirely.
+            slugMissing.push({ slug: tok.slug, anchor: tok.anchor });
+            continue;
+        }
+        if (tok.anchor === null || anchors.includes(tok.anchor)) {
+            out.push(tok.raw);
+            validCount += 1;
+            continue;
+        }
+        // Slug exists but anchor is unknown — rewrite to slug-only.
+        out.push(`[doc:${tok.slug}]`);
+        anchorMissing.push({ slug: tok.slug, anchor: tok.anchor });
+    }
+    out.push(text.slice(cursor));
+    return { text: out.join(""), validCount, anchorMissing, slugMissing };
+}
+const TOKEN_OPEN = "[doc:";
+const MAX_TOKEN_LEN = 256; // beyond this, treat as not a citation and forward
+/**
+ * Returns the length of the longest TOKEN_OPEN prefix that ends `buffer`.
+ * E.g. for `"...x[do"` returns 3 (matching the `[do` prefix); for
+ * `"hello"` returns 0. Used by `feed()` to retain tail bytes that might
+ * complete a `[doc:` opener in the next delta.
+ */
+function countTokenOpenPrefix(buffer) {
+    for (let len = TOKEN_OPEN.length - 1; len > 0; len -= 1) {
+        if (buffer.endsWith(TOKEN_OPEN.slice(0, len)))
+            return len;
+    }
+    return 0;
+}
+export function createStreamingValidator(lookup) {
+    let buffer = "";
+    let validCount = 0;
+    let anchorMissing = 0;
+    let slugMissing = 0;
+    function processChunk(chunk) {
+        if (chunk.length === 0)
+            return "";
+        const result = validateAndRewrite(chunk, lookup);
+        validCount += result.validCount;
+        anchorMissing += result.anchorMissing.length;
+        slugMissing += result.slugMissing.length;
+        return result.text;
+    }
+    function feed(delta) {
+        buffer += delta;
+        let safeUpTo = 0;
+        let i = 0;
+        let runaway = false;
+        while (i < buffer.length) {
+            const openIdx = buffer.indexOf(TOKEN_OPEN, i);
+            if (openIdx === -1) {
+                // No complete `[doc:` open at or after `i`. But the tail of
+                // `buffer` may contain a *prefix* of `[doc:` (e.g. "[", "[d",
+                // "[do", "[doc") that completes in the next delta — flushing
+                // those bytes as plaintext now would lose the citation when
+                // the closing `]` arrives. Retain only the smallest suffix
+                // that matches a TOKEN_OPEN prefix; flush the rest.
+                const retain = countTokenOpenPrefix(buffer);
+                safeUpTo = Math.max(safeUpTo, buffer.length - retain);
+                break;
+            }
+            // Up to `openIdx` is unambiguously safe to flush.
+            safeUpTo = openIdx;
+            const closeIdx = buffer.indexOf("]", openIdx);
+            if (closeIdx === -1) {
+                if (buffer.length - openIdx > MAX_TOKEN_LEN) {
+                    // Runaway — forward as-is; this is not a citation.
+                    safeUpTo = buffer.length;
+                    runaway = true;
+                    break;
+                }
+                // Token is still in flight — wait for more data.
+                break;
+            }
+            // Token spans [openIdx, closeIdx]; everything up to closeIdx+1 is
+            // safe once we process this chunk. Loop continues to look for more
+            // tokens after this one.
+            safeUpTo = closeIdx + 1;
+            i = closeIdx + 1;
+        }
+        void runaway;
+        const safeChunk = buffer.slice(0, safeUpTo);
+        buffer = buffer.slice(safeUpTo);
+        return processChunk(safeChunk);
+    }
+    function flush() {
+        // End-of-stream — anything left in the buffer cannot be a valid
+        // citation (no closing `]`). Forward as-is.
+        const tail = buffer;
+        buffer = "";
+        return processChunk(tail);
+    }
+    function snapshot() {
+        return { validCount, anchorMissing, slugMissing };
+    }
+    return { feed, flush, snapshot };
+}
+/**
+ * Persist `qa_invalid_citation` audit rows. Pure-function form so
+ * callers can decide *when* to flush (per turn, per session, etc.).
+ */
+export function logInvalidCitations(db, result, context = {}) {
+    if (result.anchorMissing.length === 0 && result.slugMissing.length === 0) {
+        return;
+    }
+    try {
+        db.prepare(`INSERT INTO agent_actions (action_type, detail)
+       VALUES (?, ?)`).run("qa_invalid_citation", JSON.stringify({
+            sessionId: context.sessionId ?? null,
+            turnNumber: context.turnNumber ?? null,
+            anchorMissing: result.anchorMissing,
+            slugMissing: result.slugMissing,
+        }));
+    }
+    catch (err) {
+        // Audit logging is best-effort. The agent_actions schema may evolve
+        // (additional NOT NULL columns); a write failure must not break the
+        // QA reply path.
+        logger.debug({ err }, "Failed to log qa_invalid_citation audit row");
+    }
+}
+//# sourceMappingURL=citation-validator.js.map

package/dist/core/docs/citation-validator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"citation-validator.js","sourceRoot":"","sources":["../../../src/core/docs/citation-validator.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,MAAM,MAAM,GAAG,YAAY,CAAC,oBAAoB,CAAC,CAAC;AAOlD;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,EAAqB;IAChD,OAAO;QACL,cAAc,CAAC,IAAY;YACzB,MAAM,GAAG,GAAG,EAAE;iBACX,OAAO,CAAC,6CAA6C,CAAC;iBACtD,GAAG,CAAC,IAAI,CAA2C,CAAC;YACvD,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;iBACvB,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iBACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;KACF,CAAC;AACJ,CAAC;AAaD;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,MAA0B;IAE1B,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IACrE,CAAC;IAED,uDAAuD;IACvD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,aAAa,GAAuC,EAAE,CAAC;IAC7D,MAAM,WAAW,GAA8C,EAAE,CAAC;IAClE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;QAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACrB,iCAAiC;YACjC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACzD,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClB,UAAU,IAAI,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,4DAA4D;QAC5D,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9B,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7B,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;AACxE,CAAC;AA4BD,MAAM,UAAU,GAAG,OAAO,CAAC;AAC3B,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,mDAAmD;AAE9E;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,MAAc;IAC1C,KAAK,IAAI,GAAG,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACxD,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAAE,OAAO,GAAG,CAAC;IAC5D,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,MAA0B;IAE1B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,SAAS,YAAY,CAAC,KAAa;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACjD,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC;QAChC,aAAa,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC;QAC7C,WAAW,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC;QACzC,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,SAAS,IAAI,CAAC,KAAa;QACzB,MAAM,IAAI,KAAK,CAAC;QAChB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC9C,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;gBACnB,4DAA4D;gBAC5D,8DAA8D;gBAC9D,6DAA6D;gBAC7D,4DAA4D;gBAC5D,2DAA2D;gBAC3D,oDAAoD;gBACpD,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC5C,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;gBACtD,MAAM;YACR,CAAC;YACD,kDAAkD;YAClD,QAAQ,GAAG,OAAO,CAAC;YACnB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,IAAI,MAAM,CAAC,MAAM,GAAG,OAAO,GAAG,aAAa,EAAE,CAAC;oBAC5C,mDAAmD;oBACnD,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;oBACzB,OAAO,GAAG,IAAI,CAAC;oBACf,MAAM;gBACR,CAAC;gBACD,iDAAiD;gBACjD,MAAM;YACR,CAAC;YACD,kEAAkE;YAClE,mEAAmE;YACnE,yBAAyB;YACzB,QAAQ,GAAG,QAAQ,GAAG,CAAC,CAAC;YACxB,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,KAAK,OAAO,CAAC;QACb,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC5C,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAChC,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAED,SAAS,KAAK;QACZ,gEAAgE;QAChE,4CAA4C;QAC5C,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,MAAM,GAAG,EAAE,CAAC;QACZ,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,SAAS,QAAQ;QACf,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC;IACpD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,EAAqB,EACrB,MAA+D,EAC/D,UAAqE,EAAE;IAEvE,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzE,OAAO;IACT,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CACR;qBACe,CAChB,CAAC,GAAG,CACH,qBAAqB,EACrB,IAAI,CAAC,SAAS,CAAC;YACb,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACpC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,oEAAoE;QACpE,oEAAoE;QACpE,iBAAiB;QACjB,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,6CAA6C,CAAC,CAAC;IACvE,CAAC;AACH,CAAC"}

package/dist/core/docs/extract-terms.d.ts

@@ -0,0 +1,78 @@
+import { type DocsFrontmatter } from "@aitne/shared";
+/**
+ * Term-level row builder for the docs subindex
+ * (DOCS-QA-SEARCH-PRECISION-PLAN.md §6).
+ *
+ * Pure module: lives outside `indexer.ts` so it lands inside the 100 %
+ * coverage gate (`vitest.config.ts` excludes the indexer's I/O glue).
+ *
+ * Two consumers:
+ *   1. `indexer.ts` calls `extractTerms()` to populate `fts_doc_terms`.
+ *   2. `indexer.ts:extractAnchors` calls `iterateHeadings()` to enumerate
+ *      the doc's H1/H2/H3 anchors. Sharing the helper guarantees the
+ *      anchor set the citation post-processor validates against and the
+ *      anchor set the QA agent sees in `term-search` results cannot drift.
+ *
+ * Code-fence handling: the previous `HEADING_RE` regex matched `#` lines
+ * inside fenced code blocks and produced phantom anchors. The shared
+ * walker tracks fence state so example MD inside ```` ``` ```` /
+ * `~~~` blocks is treated as content.
+ */
+export interface ExtractedTerm {
+    /** Empty string for the doc-level row; slugified heading otherwise. */
+    anchor: string;
+    /** Display label — frontmatter title or heading text. */
+    term: string;
+    /** Newline-joined aliases pool. May be empty. */
+    aliases: string;
+    /**
+     * Section-leading content (see §6.2.1 of the plan), capped at
+     * `LEADING_PARAGRAPH_MAX_LEN`. May be empty.
+     */
+    summary: string;
+}
+export interface HeadingRef {
+    /** Heading depth: 1, 2, or 3. */
+    level: 1 | 2 | 3;
+    /** Raw heading text (after the `# ` markers, before slugify). */
+    text: string;
+    /** Index into the line array where this heading lives. */
+    lineIndex: number;
+}
+/**
+ * Cap on the per-section `summary` column. Picked to keep BM25 ranking
+ * focused on the section's lead-in rather than diluting it with multi-
+ * paragraph prose; `unicode61` tokens average ~5 chars, so 400 chars is
+ * roughly the first 70-80 words.
+ */
+export declare const LEADING_PARAGRAPH_MAX_LEN = 400;
+/**
+ * Walk `body` line by line and yield every H1/H2/H3 heading that lives
+ * **outside** a fenced code block. Lines starting with ```` ``` ```` or
+ * `~~~` toggle a `insideFence` flag; nested or unbalanced fences
+ * default to "remain inside" until the next matching closer or EOF
+ * (FTS impact is bounded — at worst a closing-fence-less doc loses its
+ * post-fence anchors).
+ */
+export declare function iterateHeadings(body: string): HeadingRef[];
+/**
+ * Build term-index rows for a single doc.
+ *
+ * Rows produced:
+ *   1. **Doc-level row** (always emitted). `anchor=""`, `term=fm.title`,
+ *      `aliases = fm.aliases ∪ fm.keywords ∪ fm.ask_examples`
+ *      (joined by `\n`), `summary = fm.summary`.
+ *   2. **One row per H2 / H3 in the body.** `anchor = slugifyAnchor(text)`,
+ *      `term = heading_text`, `aliases = ""`, `summary = section-leading
+ *      content` (see §6.2.1) trimmed and capped at
+ *      `LEADING_PARAGRAPH_MAX_LEN`.
+ *
+ * H1 is intentionally not promoted to its own row — the doc-level row
+ * already covers it (and the H1 anchor exists in `fts_docs.anchors` for
+ * citation validation via `iterateHeadings`).
+ *
+ * Sections with an empty leading paragraph still emit a row (so the
+ * heading text itself is searchable as a `term`).
+ */
+export declare function extractTerms(fm: DocsFrontmatter, body: string): ExtractedTerm[];
+//# sourceMappingURL=extract-terms.d.ts.map

package/dist/core/docs/extract-terms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-terms.d.ts","sourceRoot":"","sources":["../../../src/core/docs/extract-terms.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;;;;;;;GAkBG;AAEH,MAAM,WAAW,aAAa;IAC5B,uEAAuE;IACvE,MAAM,EAAE,MAAM,CAAC;IACf,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,iCAAiC;IACjC,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAK7C;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,EAAE,CAkB1D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,MAAM,GACX,aAAa,EAAE,CAsDjB"}

package/dist/core/docs/extract-terms.js

@@ -0,0 +1,147 @@
+import { slugifyAnchor, } from "@aitne/shared";
+/**
+ * Cap on the per-section `summary` column. Picked to keep BM25 ranking
+ * focused on the section's lead-in rather than diluting it with multi-
+ * paragraph prose; `unicode61` tokens average ~5 chars, so 400 chars is
+ * roughly the first 70-80 words.
+ */
+export const LEADING_PARAGRAPH_MAX_LEN = 400;
+const HEADING_LINE_RE = /^(#{1,3})\s+(.+?)\s*$/;
+const FENCE_LINE_RE = /^\s*(```|~~~)/;
+/**
+ * Walk `body` line by line and yield every H1/H2/H3 heading that lives
+ * **outside** a fenced code block. Lines starting with ```` ``` ```` or
+ * `~~~` toggle a `insideFence` flag; nested or unbalanced fences
+ * default to "remain inside" until the next matching closer or EOF
+ * (FTS impact is bounded — at worst a closing-fence-less doc loses its
+ * post-fence anchors).
+ */
+export function iterateHeadings(body) {
+    const lines = body.split(/\r?\n/);
+    const out = [];
+    let insideFence = false;
+    for (let i = 0; i < lines.length; i += 1) {
+        const line = lines[i];
+        if (FENCE_LINE_RE.test(line)) {
+            insideFence = !insideFence;
+            continue;
+        }
+        if (insideFence)
+            continue;
+        const m = HEADING_LINE_RE.exec(line);
+        if (!m)
+            continue;
+        const level = m[1].length;
+        const text = m[2];
+        out.push({ level, text, lineIndex: i });
+    }
+    return out;
+}
+/**
+ * Build term-index rows for a single doc.
+ *
+ * Rows produced:
+ *   1. **Doc-level row** (always emitted). `anchor=""`, `term=fm.title`,
+ *      `aliases = fm.aliases ∪ fm.keywords ∪ fm.ask_examples`
+ *      (joined by `\n`), `summary = fm.summary`.
+ *   2. **One row per H2 / H3 in the body.** `anchor = slugifyAnchor(text)`,
+ *      `term = heading_text`, `aliases = ""`, `summary = section-leading
+ *      content` (see §6.2.1) trimmed and capped at
+ *      `LEADING_PARAGRAPH_MAX_LEN`.
+ *
+ * H1 is intentionally not promoted to its own row — the doc-level row
+ * already covers it (and the H1 anchor exists in `fts_docs.anchors` for
+ * citation validation via `iterateHeadings`).
+ *
+ * Sections with an empty leading paragraph still emit a row (so the
+ * heading text itself is searchable as a `term`).
+ */
+export function extractTerms(fm, body) {
+    const out = [];
+    const aliasPool = [
+        ...(fm.aliases ?? []),
+        ...(fm.keywords ?? []),
+        ...(fm.ask_examples ?? []),
+    ].join("\n");
+    out.push({
+        anchor: "",
+        term: fm.title,
+        aliases: aliasPool,
+        summary: fm.summary,
+    });
+    const lines = body.split(/\r?\n/);
+    const headings = iterateHeadings(body);
+    // First-wins dedup on slugified anchor. `slugifyAnchor` is lossy
+    // (drops non-ASCII, collapses whitespace), so two H2s with different
+    // raw text can collide ("Why?" / "Why!" → both "why"). Without
+    // dedup, /api/docs/term-search returns multiple rows whose `citation`
+    // string is byte-identical — the operator clicks two distinct cards
+    // that scroll to the same anchor. Keeping the first occurrence is
+    // the standard convention: docs put the definition before examples,
+    // so the earlier section-lead is usually the more useful one.
+    const seenAnchors = new Set();
+    for (let h = 0; h < headings.length; h += 1) {
+        const heading = headings[h];
+        if (heading.level === 1)
+            continue;
+        const anchor = slugifyAnchor(heading.text);
+        if (anchor.length === 0)
+            continue;
+        if (seenAnchors.has(anchor))
+            continue;
+        seenAnchors.add(anchor);
+        const nextHeading = headings[h + 1];
+        const sectionEnd = nextHeading?.lineIndex ?? lines.length;
+        const summary = collectSectionLead(lines, heading.lineIndex + 1, sectionEnd);
+        out.push({
+            anchor,
+            term: heading.text,
+            aliases: "",
+            summary,
+        });
+    }
+    return out;
+}
+/**
+ * Extract the "section-lead" text for a single H2/H3 row.
+ *
+ * Walks `lines[start..end)`, tracking fence state, and joins every
+ * non-blank source line up to (but not including) the first **top-level**
+ * blank line — i.e. a blank line outside any fenced code block. A blank
+ * line *inside* a fence does not terminate the lead-in (a fenced example
+ * directly under the heading stays whole).
+ *
+ * Lines are joined with a single space; the result is trimmed and capped
+ * at `LEADING_PARAGRAPH_MAX_LEN`. List markers, table rows, inline code,
+ * and the **bodies of** fenced code blocks that begin immediately under
+ * the heading with no preceding blank line are all treated as section-
+ * lead content. The fence delimiter lines themselves (` ``` `, `~~~`)
+ * are not included — they are markup, not content, and would otherwise
+ * surface as literal backticks in the API response and any UI rendering
+ * of the term-search summary. This is intentional — the corpus commonly
+ * places a definition list or example block directly under a heading
+ * with no leading prose.
+ *
+ * A heading whose first non-heading line **is** a blank line yields an
+ * empty summary (the blank line terminates the lead-in immediately).
+ */
+function collectSectionLead(lines, start, end) {
+    const collected = [];
+    let insideFence = false;
+    for (let i = start; i < end; i += 1) {
+        const line = lines[i];
+        if (FENCE_LINE_RE.test(line)) {
+            insideFence = !insideFence;
+            continue;
+        }
+        if (!insideFence && line.trim() === "") {
+            break;
+        }
+        collected.push(line);
+    }
+    const joined = collected.join(" ").trim();
+    if (joined.length <= LEADING_PARAGRAPH_MAX_LEN)
+        return joined;
+    return joined.slice(0, LEADING_PARAGRAPH_MAX_LEN);
+}
+//# sourceMappingURL=extract-terms.js.map

package/dist/core/docs/extract-terms.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-terms.js","sourceRoot":"","sources":["../../../src/core/docs/extract-terms.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,aAAa,GAEd,MAAM,eAAe,CAAC;AA6CvB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C,MAAM,eAAe,GAAG,uBAAuB,CAAC;AAChD,MAAM,aAAa,GAAG,eAAe,CAAC;AAEtC;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,WAAW,GAAG,CAAC,WAAW,CAAC;YAC3B,SAAS;QACX,CAAC;QACD,IAAI,WAAW;YAAE,SAAS;QAC1B,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,MAAmB,CAAC;QACxC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACnB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,YAAY,CAC1B,EAAmB,EACnB,IAAY;IAEZ,MAAM,GAAG,GAAoB,EAAE,CAAC;IAEhC,MAAM,SAAS,GAAG;QAChB,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC;QACrB,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC;QACtB,GAAG,CAAC,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC;KAC3B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,GAAG,CAAC,IAAI,CAAC;QACP,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,EAAE,CAAC,KAAK;QACd,OAAO,EAAE,SAAS;QAClB,OAAO,EAAE,EAAE,CAAC,OAAO;KACpB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IACvC,iEAAiE;IACjE,qEAAqE;IACrE,+DAA+D;IAC/D,sEAAsE;IACtE,oEAAoE;IACpE,kEAAkE;IAClE,oEAAoE;IACpE,8DAA8D;IAC9D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;YAAE,SAAS;QAElC,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAClC,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACtC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAExB,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,WAAW,EAAE,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;QAC1D,MAAM,OAAO,GAAG,kBAAkB,CAChC,KAAK,EACL,OAAO,CAAC,SAAS,GAAG,CAAC,EACrB,UAAU,CACX,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC;YACP,MAAM;YACN,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,EAAE;YACX,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,SAAS,kBAAkB,CACzB,KAAe,EACf,KAAa,EACb,GAAW;IAEX,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,WAAW,GAAG,CAAC,WAAW,CAAC;YAC3B,SAAS;QACX,CAAC;QACD,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvC,MAAM;QACR,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1C,IAAI,MAAM,CAAC,MAAM,IAAI,yBAAyB;QAAE,OAAO,MAAM,CAAC;IAC9D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;AACpD,CAAC"}

package/dist/core/docs/indexer.d.ts

@@ -0,0 +1,104 @@
+import type Database from "better-sqlite3";
+/**
+ * Docs corpus indexer (DOCS_QA_DESIGN.md §5, §7, P1).
+ *
+ * Responsibilities:
+ *   1. **Release-aware seed.** Mirror missing files from
+ *      `<workspaceDir>/agent-assets/docs/` to `<workspaceDir>/docs/user/`,
+ *      and refresh files that still match the previous shipped hash.
+ *      User-edited files are preserved and reported via release-assets
+ *      health instead of being overwritten.
+ *   2. **Boot scan.** Walk `docs/user/`, parse frontmatter, validate
+ *      against `docsFrontmatterSchema`, extract H1/H2/H3 anchors, and
+ *      upsert into `fts_docs` + `docs_revisions`.
+ *   3. **Watcher.** A chokidar watch on `docs/user/` with the same
+ *      300ms `awaitWriteFinish.stabilityThreshold` used by
+ *      `management-md.ts` so editor saves debounce cleanly. File events
+ *      re-index that single doc; deletions remove the row.
+ *
+ * Indexer health (`docs/health` route) reads:
+ *   - row count in `fts_docs`
+ *   - `lastErrorCount` accumulated since last successful re-pass
+ *   - `lastIndexedAt` from the most recent `docs_revisions.indexed_at`
+ */
+export interface DocsIndexerHandle {
+    /** Stop the chokidar watcher. */
+    stop(): Promise<void>;
+    /** Manual re-pass over the entire corpus. */
+    rebuild(): Promise<IndexerHealth>;
+    /** Current health snapshot. */
+    health(): IndexerHealth;
+}
+export interface IndexerHealth {
+    status: "ok" | "degraded" | "empty";
+    fileCount: number;
+    errorCount: number;
+    lastIndexedAt: string | null;
+    errors: {
+        slug: string;
+        reason: string;
+    }[];
+}
+export interface DocsIndexerOptions {
+    /**
+     * Repo root. The seed source is `<workspaceDir>/agent-assets/docs/`
+     * and the indexed corpus lives at `<workspaceDir>/docs/user/`.
+     */
+    workspaceDir: string;
+    /** Optional override for the seed source (testing). */
+    seedSourceDir?: string;
+    /** Optional override for the corpus dir (testing). */
+    corpusDir?: string;
+    /** Skip the chokidar watcher. Used in tests + one-shot rebuilds. */
+    watch?: boolean;
+    /**
+     * Optional backup root for release-driven docs refreshes. When omitted,
+     * unedited docs are still refreshed but no extra file-level backup is made.
+     */
+    backupRoot?: string | null;
+}
+interface IndexerErrorRow {
+    slug: string;
+    reason: string;
+}
+/**
+ * Top-level entry point used by the daemon's startup sequence.
+ *
+ * Sequence:
+ *   1. Reconcile `docs/user/` from `agent-assets/docs/` non-destructively.
+ *   2. Run a boot-scan rebuild.
+ *   3. Start the chokidar watcher (unless `watch === false`).
+ *
+ * Returns a handle for graceful shutdown + manual rebuild.
+ */
+export declare function startDocsIndexer(db: Database.Database, options: DocsIndexerOptions): Promise<DocsIndexerHandle>;
+/** Recursive copy from seed → corpus. Used only on first launch. */
+export declare function seedCorpus(seedSource: string, corpusDir: string): void;
+/** Boot-scan rebuild: walk corpusDir, re-upsert every doc, drop orphans. */
+export declare function indexCorpus(db: Database.Database, corpusDir: string): Promise<{
+    fileCount: number;
+    errors: IndexerErrorRow[];
+    lastIndexedAt: string | null;
+}>;
+/** Re-index a single file (chokidar event handler). */
+export declare function reindexSingle(db: Database.Database, corpusDir: string, absPath: string): void;
+/**
+ * Map an absolute file path under `corpusDir` to a slug. Strips the
+ * trailing `.md` and rewrites OS path separators to `/`.
+ */
+export declare function slugFromPath(corpusDir: string, absPath: string): string;
+/**
+ * Extract H1/H2/H3 anchors from a Markdown body. Code-fence aware via the
+ * shared `iterateHeadings` helper in `extract-terms.ts` — both the citation
+ * post-processor (consumes this) and the term subindex (consumes
+ * `iterateHeadings` directly through `extractTerms`) must agree on which
+ * headings exist; sharing the walker is what guarantees that.
+ */
+export declare function extractAnchors(body: string): string[];
+declare function joinTokens(values: readonly string[] | undefined): string;
+export declare const _internal: {
+    sep: "/" | "\\";
+    joinTokens: typeof joinTokens;
+};
+export {};
+//# sourceMappingURL=indexer.d.ts.map

package/dist/core/docs/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../../src/core/docs/indexer.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAe3C;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,MAAM,WAAW,iBAAiB;IAChC,iCAAiC;IACjC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,6CAA6C;IAC7C,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAClC,+BAA+B;IAC/B,MAAM,IAAI,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,IAAI,GAAG,UAAU,GAAG,OAAO,CAAC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AASD,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAID;;;;;;;;;GASG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAmH5B;AAED,oEAAoE;AACpE,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAetE;AAED,4EAA4E;AAC5E,wBAAsB,WAAW,CAC/B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC,CAgDD;AAED,uDAAuD;AACvD,wBAAgB,aAAa,CAC3B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,IAAI,CAIN;AA8DD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAIvE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOrD;AAwFD,iBAAS,UAAU,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,CAGjE;AAsCD,eAAO,MAAM,SAAS;;;CAAsB,CAAC"}