onadit 0.0.1 → 1.0.0

package/dist/src/ingestion/parsers/turns.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Shared rendering of parsed session turns into the stored transcript.
+ *
+ * Every CLI session parser stores a conversation as one flat string
+ * ("User: …\n\nAssistant: …"). Once flattened, a "User:" line can no longer be
+ * trusted: a message may quote or invent dialogue. The parser is the last
+ * place that still knows which text the user really typed, so it records those
+ * spans here, as offsets into the rendered content.
+ */
+import type { TextSpan } from "./types.js";
+export interface SessionTurn {
+    role: string;
+    text: string;
+    /**
+     * A user-role record the user did not write (compaction summary, command
+     * output, hook or harness message). Rendered like any turn, never counted as
+     * the user's own words.
+     */
+    injected?: boolean;
+}
+/**
+ * Claude Code writes harness messages (command output, compaction summaries,
+ * subagent reports, task notifications) as user-role records.
+ */
+export declare function isInjectedUserRecord(entry: {
+    isMeta?: boolean;
+    isCompactSummary?: boolean;
+    origin?: {
+        kind?: string;
+    };
+}): boolean;
+export declare function renderTurns(turns: readonly SessionTurn[]): {
+    content: string;
+    userSpans: TextSpan[];
+};
+//# sourceMappingURL=turns.d.ts.map

package/dist/src/ingestion/parsers/turns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"turns.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/parsers/turns.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5B,GAAG,OAAO,CAMV;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,WAAW,EAAE,GAAG;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,QAAQ,EAAE,CAAC;CACvB,CAaA"}

package/dist/src/ingestion/parsers/turns.js

@@ -0,0 +1,25 @@
+/**
+ * Claude Code writes harness messages (command output, compaction summaries,
+ * subagent reports, task notifications) as user-role records.
+ */
+export function isInjectedUserRecord(entry) {
+    return Boolean(entry.isMeta ||
+        entry.isCompactSummary ||
+        (entry.origin?.kind !== undefined && entry.origin.kind !== "human"));
+}
+export function renderTurns(turns) {
+    const userSpans = [];
+    let content = "";
+    for (const turn of turns) {
+        if (content)
+            content += "\n\n";
+        const isUser = turn.role === "user";
+        content += isUser ? "User: " : "Assistant: ";
+        if (isUser && !turn.injected) {
+            userSpans.push({ start: content.length, end: content.length + turn.text.length });
+        }
+        content += turn.text;
+    }
+    return { content, userSpans };
+}
+//# sourceMappingURL=turns.js.map

package/dist/src/ingestion/parsers/turns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"turns.js","sourceRoot":"","sources":["../../../../src/ingestion/parsers/turns.ts"],"names":[],"mappings":"AAsBA;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAIpC;IACC,OAAO,OAAO,CACZ,KAAK,CAAC,MAAM;QACV,KAAK,CAAC,gBAAgB;QACtB,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAA6B;IAIvD,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO;YAAE,OAAO,IAAI,MAAM,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;QACpC,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;QAC7C,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC"}

package/dist/src/ingestion/parsers/types.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Shared types for web conversation parsers — Slice 13.
+ *
+ * Each source-specific parser returns ParsedConversation[].
+ * Metadata (domain, intent, quality, etc.) is added later by
+ * the MetadataExtractor, not by the parser.
+ */
+export interface ParsedConversation {
+    /** Conversation UUID from the source export. */
+    id: string;
+    /** Conversation title. */
+    title: string;
+    /** ISO 8601 creation timestamp. */
+    createdAt: string;
+    /** Formatted conversation text (all messages, role-prefixed). */
+    content: string;
+    /** Working directory of an agent session (CLI transcripts only). */
+    cwd?: string;
+    /**
+     * Spans of `content` the user really typed (CLI transcripts only), taken
+     * from the source records before flattening. The only trustworthy speaker
+     * attribution: a "User:" line in `content` proves nothing.
+     */
+    userSpans?: TextSpan[];
+}
+/** Character offsets into a string, end exclusive. */
+export interface TextSpan {
+    start: number;
+    end: number;
+}
+/**
+ * A parser takes extracted files (from archive.ts) and returns
+ * normalized conversation entries.
+ */
+export type ConversationParser = (files: Array<{
+    path: string;
+    data: Buffer;
+}>) => ParsedConversation[];
+/** Sources with implemented parsers (backed by real fixtures). */
+export declare const SUPPORTED_INGEST_SOURCES: readonly ["chatgpt_web", "claude_web", "gemini_web", "grok_web"];
+export type SupportedIngestSource = (typeof SUPPORTED_INGEST_SOURCES)[number];
+/** All web sources that POST /ingest accepts in the enum (some deferred). */
+export declare const INGEST_SOURCES: readonly ["chatgpt_web", "claude_web", "grok_web", "gemini_web"];
+export type IngestSource = (typeof INGEST_SOURCES)[number];
+/** Check if a source has an implemented parser. */
+export declare function isSourceSupported(source: string): source is SupportedIngestSource;
+//# sourceMappingURL=types.d.ts.map

package/dist/src/ingestion/parsers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/ingestion/parsers/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,kBAAkB;IACjC,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,mCAAmC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,iEAAiE;IACjE,OAAO,EAAE,MAAM,CAAC;IAChB,oEAAoE;IACpE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;CACxB;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;GAGG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAC/B,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,KACzC,kBAAkB,EAAE,CAAC;AAE1B,kEAAkE;AAClE,eAAO,MAAM,wBAAwB,kEAK3B,CAAC;AACX,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9E,6EAA6E;AAC7E,eAAO,MAAM,cAAc,kEAKjB,CAAC;AACX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D,mDAAmD;AACnD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,qBAAqB,CAEjF"}

package/dist/src/ingestion/parsers/types.js

@@ -0,0 +1,26 @@
+/**
+ * Shared types for web conversation parsers — Slice 13.
+ *
+ * Each source-specific parser returns ParsedConversation[].
+ * Metadata (domain, intent, quality, etc.) is added later by
+ * the MetadataExtractor, not by the parser.
+ */
+/** Sources with implemented parsers (backed by real fixtures). */
+export const SUPPORTED_INGEST_SOURCES = [
+    "chatgpt_web",
+    "claude_web",
+    "gemini_web",
+    "grok_web",
+];
+/** All web sources that POST /ingest accepts in the enum (some deferred). */
+export const INGEST_SOURCES = [
+    "chatgpt_web",
+    "claude_web",
+    "grok_web",
+    "gemini_web",
+];
+/** Check if a source has an implemented parser. */
+export function isSourceSupported(source) {
+    return SUPPORTED_INGEST_SOURCES.includes(source);
+}
+//# sourceMappingURL=types.js.map

package/dist/src/ingestion/parsers/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/ingestion/parsers/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAmCH,kEAAkE;AAClE,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,UAAU;CACF,CAAC;AAGX,6EAA6E;AAC7E,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,aAAa;IACb,YAAY;IACZ,UAAU;IACV,YAAY;CACJ,CAAC;AAGX,mDAAmD;AACnD,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,OAAQ,wBAA8C,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC1E,CAAC"}

package/dist/src/ingestion/report-cards.d.ts

@@ -0,0 +1,75 @@
+export interface ReportCard {
+    /** Stable id derived from the path (filename stem, date-prefix stripped). */
+    slug: string;
+    title: string;
+    /** One line, ≤ SUMMARY_MAX_CHARS. */
+    summary: string;
+    topics: string[];
+    /** YYYY-MM-DD. */
+    date: string;
+    /** Repo-relative path to the real report (click-through + recall target). */
+    path: string;
+}
+export declare const SUMMARY_MAX_CHARS = 200;
+export declare const MAX_TOPICS = 6;
+/** Filename stem with extension(s) and a leading `YYYY-MM-DD-` date stripped. */
+export declare function slugFromReportPath(repoRelPath: string): string;
+interface FrontMatter {
+    body: string;
+    fields: Record<string, string>;
+    /** Raw list values (e.g. tags) collected separately. */
+    lists: Record<string, string[]>;
+}
+/**
+ * Minimal YAML-ish front-matter split. Handles `key: value`, inline
+ * `tags: [a, b]`, and block lists (`tags:` then `  - a`). Not a full YAML
+ * parser — deliberately lazy; only the few keys we read need to work.
+ */
+export declare function parseFrontMatter(content: string): FrontMatter;
+export interface BuildReportCardInput {
+    /** Repo-relative path of the report (the `path` field + recall target). */
+    path: string;
+    /** Raw file contents. */
+    content: string;
+    /** ISO mtime, used only as the date fallback. */
+    mtimeISO: string;
+}
+/** Build a deterministic card for one report. */
+export declare function buildReportCard(input: BuildReportCardInput): ReportCard;
+/**
+ * Make card slugs globally unique across a set. Distinct report paths can share
+ * a filename stem (e.g. the same report name under two dated dirs), which would
+ * collide their `report_card:<slug>` row id — two rows sharing a primary id.
+ * For any slug used by >1 card, disambiguate ALL of them with a short hash of
+ * their repo-relative path. Order-independent and deterministic (depends only on
+ * each card's own path), so a re-run produces identical ids → idempotent.
+ */
+export declare function dedupeCardSlugs(cards: ReportCard[]): ReportCard[];
+/**
+ * The canonical marker for a card row: its LanceDB id is `report_card:<slug>`.
+ * This prefix is the single source of truth used everywhere a card must be
+ * recognized — the SQL retrieval exclusion (flag off) and the reranker's
+ * card-specific guard (flag on) — so there is no parallel `source_type` column
+ * to thread through every candidate-construction site (the id is already
+ * present on every row and every RerankCandidate).
+ */
+export declare const REPORT_CARD_ID_PREFIX = "report_card:";
+/** Stable LanceDB row id for a card. Idempotent backfill deletes+re-adds by this. */
+export declare function reportCardRowId(slug: string): string;
+/** True when a LanceDB row id is a report index card (see REPORT_CARD_ID_PREFIX). */
+export declare function isReportCardRowId(id: string): boolean;
+/**
+ * Retrieval flag for beta.28 index cards. **Default OFF** — with it off, cards
+ * are excluded from vector + lexical retrieval (SQL `id NOT LIKE`), so the
+ * search/result path is byte-identical to pre-beta.28. Backfill/emission may
+ * write cards regardless; they only enter retrieval when this is on.
+ * Enable with `RIFT_INDEX_CARDS=1|true|on|yes`.
+ */
+export declare function indexCardsEnabled(env?: NodeJS.ProcessEnv): boolean;
+/** SQL clause excluding card rows; appended to structured_docs reads when the
+ *  flag is OFF so cards never reach the candidate pool. */
+export declare const REPORT_CARD_SQL_EXCLUSION = "id NOT LIKE 'report_card:%'";
+/** The text that gets embedded — curated fields only, never the report body. */
+export declare function reportCardEmbedText(card: ReportCard): string;
+export {};
+//# sourceMappingURL=report-cards.d.ts.map

package/dist/src/ingestion/report-cards.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-cards.d.ts","sourceRoot":"","sources":["../../../src/ingestion/report-cards.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,UAAU;IACzB,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;CACd;AAED,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,UAAU,IAAI,CAAC;AAY5B,iFAAiF;AACjF,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAK9D;AAED,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACjC;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,CAgD7D;AAiED,MAAM,WAAW,oBAAoB;IACnC,2EAA2E;IAC3E,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,iDAAiD;AACjD,wBAAgB,eAAe,CAAC,KAAK,EAAE,oBAAoB,GAAG,UAAU,CAmBvE;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,UAAU,EAAE,CAQjE;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,iBAAiB,CAAC;AAEpD,qFAAqF;AACrF,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,qFAAqF;AACrF,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAErD;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,OAAO,CAI/E;AAED;2DAC2D;AAC3D,eAAO,MAAM,yBAAyB,gCAA4C,CAAC;AAEnF,gFAAgF;AAChF,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAI5D"}

package/dist/src/ingestion/report-cards.js

@@ -0,0 +1,224 @@
+/**
+ * Write-time index cards for Onadit-owned reports (beta.28 retrieval coverage).
+ *
+ * A card is a small, retrievable pointer to a report: `slug, title, summary,
+ * type, topics, date, path`. It is indexed independently of the report body so
+ * the report becomes findable even though the reports tree is not an indexed
+ * `sources` entry. The card's embedded text is `title + summary + topics` —
+ * curated and short, never the body (precision over recall, and the pollution
+ * guard for the eval gate in docs/10-beta28-retrieval-coverage.md).
+ *
+ * Everything here is PURE and DETERMINISTIC — no AI, no network, no clock reads
+ * beyond the `mtimeISO` the caller passes in. Summaries and topics are derived
+ * from front matter when present, else from the markdown deterministically, so
+ * a backfill is reproducible and idempotent on `slug`.
+ */
+import { createHash } from "node:crypto";
+export const SUMMARY_MAX_CHARS = 200;
+export const MAX_TOPICS = 6;
+/** Common words that never make useful topics. Small, English+a little French. */
+const STOPWORDS = new Set([
+    "the", "a", "an", "and", "or", "but", "of", "to", "in", "on", "for", "with",
+    "is", "are", "was", "were", "be", "been", "being", "it", "its", "this", "that",
+    "these", "those", "as", "at", "by", "from", "into", "over", "under", "out",
+    "we", "i", "you", "they", "he", "she", "our", "your", "their", "not", "no",
+    "report", "summary", "findings", "notes", "doc", "document", "rift", "onadit",
+    "le", "la", "les", "de", "des", "du", "et", "un", "une", "pour", "pas",
+]);
+/** Filename stem with extension(s) and a leading `YYYY-MM-DD-` date stripped. */
+export function slugFromReportPath(repoRelPath) {
+    const base = repoRelPath.split("/").pop() ?? repoRelPath;
+    // Strip ALL trailing extensions (handles `*.slack.md`, `*.md`).
+    const noExt = base.replace(/(\.[a-z0-9]+)+$/i, "");
+    return noExt.replace(/^\d{4}-\d{2}-\d{2}-/, "");
+}
+/**
+ * Minimal YAML-ish front-matter split. Handles `key: value`, inline
+ * `tags: [a, b]`, and block lists (`tags:` then `  - a`). Not a full YAML
+ * parser — deliberately lazy; only the few keys we read need to work.
+ */
+export function parseFrontMatter(content) {
+    const fields = {};
+    const lists = {};
+    if (!content.startsWith("---"))
+        return { body: content, fields, lists };
+    const lines = content.split(/\r?\n/);
+    if (lines[0]?.trim() !== "---")
+        return { body: content, fields, lists };
+    let end = -1;
+    for (let i = 1; i < lines.length; i++) {
+        if (lines[i]?.trim() === "---") {
+            end = i;
+            break;
+        }
+    }
+    if (end === -1)
+        return { body: content, fields, lists };
+    let currentList = null;
+    for (let i = 1; i < end; i++) {
+        const line = lines[i] ?? "";
+        const listItem = /^\s*-\s+(.*)$/.exec(line);
+        if (currentList && listItem) {
+            lists[currentList].push(stripQuotes(listItem[1] ?? ""));
+            continue;
+        }
+        const kv = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line);
+        if (!kv) {
+            currentList = null;
+            continue;
+        }
+        const key = (kv[1] ?? "").toLowerCase();
+        const val = (kv[2] ?? "").trim();
+        if (val === "") {
+            // Block list begins on following `  - item` lines.
+            currentList = key;
+            lists[key] = [];
+            continue;
+        }
+        currentList = null;
+        const inline = /^\[(.*)\]$/.exec(val);
+        if (inline) {
+            lists[key] = inline[1]
+                .split(",")
+                .map((s) => stripQuotes(s.trim()))
+                .filter(Boolean);
+        }
+        else {
+            fields[key] = stripQuotes(val);
+        }
+    }
+    return { body: lines.slice(end + 1).join("\n"), fields, lists };
+}
+function stripQuotes(s) {
+    return s.replace(/^["']|["']$/g, "").trim();
+}
+/** First markdown heading (`# …`) in `body`, else "". */
+function firstHeading(body) {
+    for (const line of body.split(/\r?\n/)) {
+        const m = /^#{1,6}\s+(.*)$/.exec(line.trim());
+        if (m)
+            return (m[1] ?? "").trim();
+    }
+    return "";
+}
+/** First non-empty, non-heading, non-list, non-quote line — the lede. */
+function firstParagraph(body) {
+    for (const raw of body.split(/\r?\n/)) {
+        const line = raw.trim();
+        if (!line)
+            continue;
+        if (/^(#|>|-|\*|\||```|<!--)/.test(line))
+            continue;
+        return line;
+    }
+    return "";
+}
+/** Collapse markdown emphasis/links to plain text, then clamp to one line. */
+function plainOneLine(s, max) {
+    const flat = s
+        .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") // [text](url) -> text
+        .replace(/[*_`#>]/g, "")
+        .replace(/\s+/g, " ")
+        .trim();
+    if (flat.length <= max)
+        return flat;
+    return flat.slice(0, max - 1).trimEnd() + "…";
+}
+/** Top tokens by frequency, stopword- and length-filtered. Deterministic. */
+function keywordTopics(body, max) {
+    const counts = new Map();
+    const order = [];
+    for (const raw of body.toLowerCase().match(/[a-z][a-z0-9+_-]{2,}/g) ?? []) {
+        if (STOPWORDS.has(raw))
+            continue;
+        if (!counts.has(raw))
+            order.push(raw);
+        counts.set(raw, (counts.get(raw) ?? 0) + 1);
+    }
+    // Sort by count desc, then first-seen order (stable, deterministic).
+    return order
+        .sort((a, b) => (counts.get(b) - counts.get(a)) || (order.indexOf(a) - order.indexOf(b)))
+        .slice(0, max);
+}
+const DATE_RX = /(\d{4}-\d{2}-\d{2})/;
+function deriveDate(repoRelPath, fmDate, mtimeISO) {
+    if (fmDate) {
+        const m = DATE_RX.exec(fmDate);
+        if (m)
+            return m[1];
+    }
+    const base = repoRelPath.split("/").pop() ?? repoRelPath;
+    const fromName = DATE_RX.exec(base);
+    if (fromName)
+        return fromName[1];
+    return mtimeISO.slice(0, 10);
+}
+/** Build a deterministic card for one report. */
+export function buildReportCard(input) {
+    const { path: repoRelPath, content, mtimeISO } = input;
+    const { body, fields, lists } = parseFrontMatter(content);
+    const slug = slugFromReportPath(repoRelPath);
+    const title = fields["title"] || firstHeading(body) || slug.replace(/[-_]/g, " ");
+    const summarySource = fields["summary"] || firstParagraph(body) || title;
+    const summary = plainOneLine(summarySource, SUMMARY_MAX_CHARS);
+    const fmTopics = lists["tags"] ?? lists["topics"] ?? [];
+    const topics = (fmTopics.length > 0 ? fmTopics.map((t) => t.toLowerCase()) : keywordTopics(body, MAX_TOPICS)).slice(0, MAX_TOPICS);
+    const date = deriveDate(repoRelPath, fields["date"], mtimeISO);
+    return { slug, title, summary, topics, date, path: repoRelPath };
+}
+/**
+ * Make card slugs globally unique across a set. Distinct report paths can share
+ * a filename stem (e.g. the same report name under two dated dirs), which would
+ * collide their `report_card:<slug>` row id — two rows sharing a primary id.
+ * For any slug used by >1 card, disambiguate ALL of them with a short hash of
+ * their repo-relative path. Order-independent and deterministic (depends only on
+ * each card's own path), so a re-run produces identical ids → idempotent.
+ */
+export function dedupeCardSlugs(cards) {
+    const counts = new Map();
+    for (const c of cards)
+        counts.set(c.slug, (counts.get(c.slug) ?? 0) + 1);
+    return cards.map((c) => (counts.get(c.slug) ?? 0) > 1
+        ? { ...c, slug: `${c.slug}-${createHash("sha1").update(c.path).digest("hex").slice(0, 8)}` }
+        : c);
+}
+/**
+ * The canonical marker for a card row: its LanceDB id is `report_card:<slug>`.
+ * This prefix is the single source of truth used everywhere a card must be
+ * recognized — the SQL retrieval exclusion (flag off) and the reranker's
+ * card-specific guard (flag on) — so there is no parallel `source_type` column
+ * to thread through every candidate-construction site (the id is already
+ * present on every row and every RerankCandidate).
+ */
+export const REPORT_CARD_ID_PREFIX = "report_card:";
+/** Stable LanceDB row id for a card. Idempotent backfill deletes+re-adds by this. */
+export function reportCardRowId(slug) {
+    return `${REPORT_CARD_ID_PREFIX}${slug}`;
+}
+/** True when a LanceDB row id is a report index card (see REPORT_CARD_ID_PREFIX). */
+export function isReportCardRowId(id) {
+    return id.startsWith(REPORT_CARD_ID_PREFIX);
+}
+/**
+ * Retrieval flag for beta.28 index cards. **Default OFF** — with it off, cards
+ * are excluded from vector + lexical retrieval (SQL `id NOT LIKE`), so the
+ * search/result path is byte-identical to pre-beta.28. Backfill/emission may
+ * write cards regardless; they only enter retrieval when this is on.
+ * Enable with `RIFT_INDEX_CARDS=1|true|on|yes`.
+ */
+export function indexCardsEnabled(env = process.env) {
+    const v = env["RIFT_INDEX_CARDS"];
+    if (v == null)
+        return false;
+    return /^(1|true|on|yes)$/i.test(v.trim());
+}
+/** SQL clause excluding card rows; appended to structured_docs reads when the
+ *  flag is OFF so cards never reach the candidate pool. */
+export const REPORT_CARD_SQL_EXCLUSION = `id NOT LIKE '${REPORT_CARD_ID_PREFIX}%'`;
+/** The text that gets embedded — curated fields only, never the report body. */
+export function reportCardEmbedText(card) {
+    return [card.title, card.summary, card.topics.join(", ")]
+        .filter((s) => s.trim().length > 0)
+        .join("\n\n");
+}
+//# sourceMappingURL=report-cards.js.map

package/dist/src/ingestion/report-cards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-cards.js","sourceRoot":"","sources":["../../../src/ingestion/report-cards.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAezC,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AACrC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,kFAAkF;AAClF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM;IAC3E,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC9E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;IAC1E,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI;IAC1E,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ;IAC7E,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;CACvE,CAAC,CAAC;AAEH,iFAAiF;AACjF,MAAM,UAAU,kBAAkB,CAAC,WAAmB;IACpD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,WAAW,CAAC;IACzD,gEAAgE;IAChE,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IACnD,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;AAClD,CAAC;AASD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,KAAK,GAA6B,EAAE,CAAC;IAC3C,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IACxE,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YAC/B,GAAG,GAAG,CAAC,CAAC;YACR,MAAM;QACR,CAAC;IACH,CAAC;IACD,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAExD,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;YAC5B,KAAK,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACzD,SAAS;QACX,CAAC;QACD,MAAM,EAAE,GAAG,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,WAAW,GAAG,IAAI,CAAC;YACnB,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;YACf,mDAAmD;YACnD,WAAW,GAAG,GAAG,CAAC;YAClB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,WAAW,GAAG,IAAI,CAAC;QACnB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,IAAI,MAAM,EAAE,CAAC;YACX,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE;iBACpB,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;iBACjC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED,yDAAyD;AACzD,SAAS,YAAY,CAAC,IAAY;IAChC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,yEAAyE;AACzE,SAAS,cAAc,CAAC,IAAY;IAClC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACnD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,8EAA8E;AAC9E,SAAS,YAAY,CAAC,CAAS,EAAE,GAAW;IAC1C,MAAM,IAAI,GAAG,CAAC;SACX,OAAO,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAC,sBAAsB;SAChE,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IACV,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC;AAChD,CAAC;AAED,6EAA6E;AAC7E,SAAS,aAAa,CAAC,IAAY,EAAE,GAAW;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,uBAAuB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1E,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACjC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,qEAAqE;IACrE,OAAO,KAAK;SACT,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;SAC1F,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,GAAG,qBAAqB,CAAC;AAEtC,SAAS,UAAU,CAAC,WAAmB,EAAE,MAA0B,EAAE,QAAgB;IACnF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAE,CAAC;IACtB,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,WAAW,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,CAAC,CAAE,CAAC;IAClC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/B,CAAC;AAWD,iDAAiD;AACjD,MAAM,UAAU,eAAe,CAAC,KAA2B;IACzD,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC;IACvD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,IAAI,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,KAAK,GACT,MAAM,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEtE,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC;IACzE,MAAM,OAAO,GAAG,YAAY,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxD,MAAM,MAAM,GAAG,CACb,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAC7F,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAEvB,MAAM,IAAI,GAAG,UAAU,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;IAE/D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;AACnE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,KAAmB;IACjD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACrB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;QAC3B,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;QAC5F,CAAC,CAAC,CAAC,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,cAAc,CAAC;AAEpD,qFAAqF;AACrF,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,GAAG,qBAAqB,GAAG,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,iBAAiB,CAAC,EAAU;IAC1C,OAAO,EAAE,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;AAC9C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACpE,MAAM,CAAC,GAAG,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAClC,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED;2DAC2D;AAC3D,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,qBAAqB,IAAI,CAAC;AAEnF,gFAAgF;AAChF,MAAM,UAAU,mBAAmB,CAAC,IAAgB;IAClD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACtD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SAClC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC"}

package/dist/src/ingestion/scanner.d.ts

@@ -0,0 +1,48 @@
+import { type FileEvent } from "./indexer.js";
+export interface ScanResult {
+    events: FileEvent[];
+    filesChecked: number;
+}
+export interface ScanOptions {
+    ignoredPaths?: readonly string[];
+}
+export interface WalkOptions {
+    ignoredPaths?: readonly string[];
+}
+export interface ScanState {
+    last_scan_at: string | null;
+    next_scan_at: string | null;
+}
+export declare class Scanner {
+    private paused;
+    private readonly statePath;
+    private state;
+    constructor(dataDir: string);
+    /**
+     * Scan the given directories for new or modified files.
+     * Returns FileEvents for files that need indexing.
+     * Returns empty result if paused.
+     */
+    scan(dirs: readonly string[], opts?: ScanOptions): Promise<ScanResult>;
+    pause(): void;
+    resume(): void;
+    get isPaused(): boolean;
+    get scanState(): ScanState;
+    setNextScan(nextScanAt: string): void;
+    /**
+     * Whether a scan is due based on the configured interval.
+     * True if we have never scanned or if `last_scan_at + interval` is in the past.
+     */
+    isDue(intervalSeconds: number): boolean;
+    private loadState;
+    private saveState;
+}
+/**
+ * Recursively walk a directory, returning all file paths.
+ * Honors the same ignored-path policy as the live Watcher via
+ * `isWatcherPathIgnored`: always skips .git / node_modules / .DS_Store,
+ * plus any caller-supplied roots (e.g. engine data_dir, dist, logs).
+ * Does not follow symlinks.
+ */
+export declare function walkDirectory(dir: string, opts?: WalkOptions): string[];
+//# sourceMappingURL=scanner.d.ts.map

package/dist/src/ingestion/scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scanner.d.ts","sourceRoot":"","sources":["../../../src/ingestion/scanner.ts"],"names":[],"mappings":"AAYA,OAAO,EAAU,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AAMtD,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,WAAW;IAC1B,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,KAAK,CAAyD;gBAE1D,OAAO,EAAE,MAAM;IAK3B;;;;OAIG;IACG,IAAI,CACR,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,GAAE,WAAgB,GACrB,OAAO,CAAC,UAAU,CAAC;IAyCtB,KAAK,IAAI,IAAI;IAIb,MAAM,IAAI,IAAI;IAId,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAKrC;;;OAGG;IACH,KAAK,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO;IAMvC,OAAO,CAAC,SAAS;YASH,SAAS;CAGxB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,MAAM,EAAE,CAgB3E"}

package/dist/src/ingestion/scanner.js

@@ -0,0 +1,131 @@
+/**
+ * Scheduled scanner for mode: "scheduled_scan" directories.
+ *
+ * Walks directories, compares file mtimes against indexed state
+ * in structured_docs, and returns FileEvents for new/modified files.
+ * Reuses the Indexer pipeline from Slice 5.
+ *
+ * Exposes pause()/resume() for Live-Indexer / Repair-Job Coordination.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { isSupported } from "./extractor.js";
+import { fileId } from "./indexer.js";
+import { getTable } from "../storage/tables.js";
+import { atomicWrite } from "../storage/atomic.js";
+import { isWatcherPathIgnored } from "./watcher.js";
+export class Scanner {
+    paused = false;
+    statePath;
+    state = { last_scan_at: null, next_scan_at: null };
+    constructor(dataDir) {
+        this.statePath = path.join(dataDir, "scan-state.json");
+        this.loadState();
+    }
+    /**
+     * Scan the given directories for new or modified files.
+     * Returns FileEvents for files that need indexing.
+     * Returns empty result if paused.
+     */
+    async scan(dirs, opts = {}) {
+        if (this.paused)
+            return { events: [], filesChecked: 0 };
+        let filesChecked = 0;
+        const events = [];
+        const table = getTable("structured_docs");
+        const walkOpts = opts.ignoredPaths
+            ? { ignoredPaths: opts.ignoredPaths }
+            : {};
+        for (const dir of dirs) {
+            const files = walkDirectory(dir, walkOpts);
+            for (const filePath of files) {
+                if (!isSupported(filePath))
+                    continue;
+                filesChecked++;
+                const id = fileId(filePath);
+                const existing = (await table
+                    .query()
+                    .where(`id = '${id}'`)
+                    .toArray());
+                if (existing.length === 0) {
+                    events.push({ type: "add", path: filePath });
+                }
+                else {
+                    const stats = fs.statSync(filePath);
+                    const indexedAt = new Date(existing[0].indexed_at);
+                    if (stats.mtime > indexedAt) {
+                        events.push({ type: "change", path: filePath });
+                    }
+                }
+            }
+        }
+        this.state.last_scan_at = new Date().toISOString();
+        await this.saveState();
+        return { events, filesChecked };
+    }
+    pause() {
+        this.paused = true;
+    }
+    resume() {
+        this.paused = false;
+    }
+    get isPaused() {
+        return this.paused;
+    }
+    get scanState() {
+        return { ...this.state };
+    }
+    setNextScan(nextScanAt) {
+        this.state.next_scan_at = nextScanAt;
+        this.saveState().catch(() => { });
+    }
+    /**
+     * Whether a scan is due based on the configured interval.
+     * True if we have never scanned or if `last_scan_at + interval` is in the past.
+     */
+    isDue(intervalSeconds) {
+        if (!this.state.last_scan_at)
+            return true;
+        const lastScan = new Date(this.state.last_scan_at).getTime();
+        return Date.now() >= lastScan + intervalSeconds * 1000;
+    }
+    loadState() {
+        try {
+            const raw = fs.readFileSync(this.statePath, "utf-8");
+            this.state = JSON.parse(raw);
+        }
+        catch {
+            // No state file yet — use defaults
+        }
+    }
+    async saveState() {
+        await atomicWrite(this.statePath, JSON.stringify(this.state, null, 2));
+    }
+}
+/**
+ * Recursively walk a directory, returning all file paths.
+ * Honors the same ignored-path policy as the live Watcher via
+ * `isWatcherPathIgnored`: always skips .git / node_modules / .DS_Store,
+ * plus any caller-supplied roots (e.g. engine data_dir, dist, logs).
+ * Does not follow symlinks.
+ */
+export function walkDirectory(dir, opts = {}) {
+    const results = [];
+    if (!fs.existsSync(dir))
+        return results;
+    const ignoredPaths = opts.ignoredPaths ?? [];
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (isWatcherPathIgnored(fullPath, ignoredPaths))
+            continue;
+        if (entry.isDirectory() && !entry.isSymbolicLink()) {
+            results.push(...walkDirectory(fullPath, opts));
+        }
+        else if (entry.isFile()) {
+            results.push(fullPath);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=scanner.js.map