@isaacriehm/cairn-core 0.6.0 → 0.7.0

package/dist/init/ingest-docs.d.ts

@@ -1,18 +1,44 @@
 /**
- * Phase 6 — docs ingestion (v0.5.0 SoT model).
+ * Phase 6 — staged docs ingestion (PHASE_6_REDESIGN §4.1).
  *
- * Reads the topic-index built by phase 5b, filters to entries whose SoT
- * source lives under `docs/*`, and emits verbatim DEC files under
- * `.cairn/ground/decisions/`. Auto-promoted to `status: accepted`. No
- * draft inbox, no LLM paraphrase — the doc paragraph itself IS the
- * canonical body, recorded with `sot_kind: path` so the lens renders
- * the live source on every read.
+ * Replaces the v0.6 bulk-classifier path. Cuts wall from ~15 min →
+ * ~75 s on gcb-platform-scale repos AND collapses the noisy ledger
+ * (7000 DECs) to a curated draft inbox (30-80 drafts).
  *
- * Per-entry Haiku call decides `kind` only (decision / domain-rule /
- * voice-guidelines / api-docs / other). The first two emit a DEC; the
- * rest are skipped at this layer (voice + canonical-topic flows are
- * handled by other tooling now — they were file-level concerns under
- * the v0.4.x model and have no clean paragraph-level analogue).
+ * Pipeline:
+ *
+ *   Stage 3 (deterministic, 0 Haiku) — marker scan
+ *     Topic-index entries with `marker_kind` in {"decision","rule"} go
+ *     straight to emit. The walker stamped them at parse time when it
+ *     saw frontmatter `cairn.kind` or `<!-- cairn:decision -->` /
+ *     `<!-- cairn:rule -->` within 3 lines of the heading.
+ *
+ *   Stage 1 — file-purpose binary filter (batch=30, concurrency=5)
+ *     Per file: filepath + frontmatter + first 800 chars + every
+ *     H1/H2/H3 line (capped at 100). Locked rigid prompt: a file is
+ *     authoritative ONLY if it's a canonical rulebook, formal ADR,
+ *     or list of binding domain invariants. Plans / scratchpads /
+ *     UAT logs / API docs are NOT authoritative even if they
+ *     contain proposed or historical decisions.
+ *
+ *   Stage 2 — section-level batch classifier (batch=30, concurrency=5)
+ *     Same shape as the v0.6 classifier, but scoped to sections
+ *     belonging to Stage-1-authoritative files AND not already
+ *     handled by a marker. This is where Haiku still adds signal —
+ *     the file passed the rigid filter; now decide WHICH sections
+ *     of it are decisions vs context.
+ *
+ *   Stage 4 — emit
+ *     Stage 2 + Stage 3 outputs → `.cairn/ground/decisions/_inbox/<id>.draft.md`.
+ *     `status: draft`, `capture_source: init-docs-ingest`,
+ *     `decided_by: cairn-init`. Body is verbatim via
+ *     `readSotBody` — no Haiku paraphrasing. Operator triages via
+ *     the existing `cairn-attention` skill.
+ *
+ * Skipped entries (everything else) stay in the topic-index as
+ * unpromoted candidates. The PR 2 `cairn_search_candidates` /
+ * `cairn_propose_decision` MCP tools surface them to AI agents as
+ * the project lives.
  */
 import { type TopicIndexEntry } from "../ground/index.js";
 export interface DocCandidate {
@@ -32,33 +58,67 @@ export interface ClassifiedDoc {
     errorMessage?: string;
 }
 export interface IngestionResult {
-    /** Verbatim DEC files written under `.cairn/ground/decisions/`. */
+    /** Verbatim DEC drafts written under `_inbox/`. */
     decsWritten: {
         id: string;
         path: string;
         sourceFile: string;
         slug: string;
     }[];
-    /** Topic-index entries that were not emitted, with reasons. */
+    /** Topic-index entries skipped at Stage 4 (with reason). */
     skipped: {
         slug: string;
         reason: string;
     }[];
-    /** Number of topic-index entries considered (sot_source under docs/). */
+    /** Total topic-index entries considered (pre-rejection-filter). */
     scannedEntries: number;
+    /** Stage 3 — entries emitted from operator markers (0 Haiku). */
+    markerEmits: number;
+    /** Stage 2 — entries emitted from authoritative-file section classifications. */
+    sectionEmits: number;
+    /**
+     * Stage 1 — files the binary filter accepted as authoritative
+     * (i.e. eligible to feed Stage 2).
+     */
+    authoritativeFiles: number;
+    /** Total distinct files Stage 1 evaluated. */
+    filesEvaluated: number;
+    /**
+     * Topic-index entries that remain unpromoted (`dec_id IS NULL`)
+     * after this run. Surface this in the cold-start summary as
+     * "K unpromoted candidates indexed".
+     */
+    unpromotedCandidates: number;
+}
+export interface ChunkProgressRow {
+    chunksDone: number;
+    totalChunks: number;
+    entriesDone: number;
+    totalEntries: number;
+    /** Which staged phase the chunk belongs to ("file-filter" or "section-classify"). */
+    stage: "file-filter" | "section-classify";
 }
 export interface RunDocsIngestionArgs {
     repoRoot: string;
-    /** Smoke override — feed canned classifications keyed by slug. */
+    /**
+     * Smoke override — feed canned classifications keyed by entry. When
+     * set, Stages 1+2 are bypassed entirely; every non-marker candidate
+     * runs through this synchronously. Stage 3 still fires.
+     */
     mockClassify?: (entry: TopicIndexEntry, body: string) => DocClassification;
     /** Caller-supplied id collision Set (parallel pipeline). */
     existingDecIds?: Set<string>;
-    /** Progress callback fired once per emitted entry. */
-    onEntryProgress?: (row: {
-        slug: string;
-        emitted: boolean;
-        total: number;
-    }) => void;
+    /** Progress callback fired once per completed batch. */
+    onChunkProgress?: (row: ChunkProgressRow) => void;
 }
 export declare function discoverDocs(repoRoot: string): DocCandidate[];
+export interface FileFilterVerdict {
+    is_authoritative: boolean;
+    reason: string;
+}
 export declare function runDocsIngestion(args: RunDocsIngestionArgs): Promise<IngestionResult>;
+export declare function runStage1FileFilter(args: {
+    repoRoot: string;
+    files: string[];
+    onChunkProgress?: (row: ChunkProgressRow) => void;
+}): Promise<Map<string, FileFilterVerdict>>;