leedab 0.7.2 → 0.9.0

package/dist/brain/cli.d.ts

@@ -0,0 +1,26 @@
+export interface AddFolderOpts {
+    clientId?: string;
+    rootId?: string;
+}
+export declare function cmdAddFolder(path: string, opts?: AddFolderOpts): Promise<void>;
+export interface ScanOpts {
+    clientId?: string;
+    json?: boolean;
+}
+export declare function cmdScan(opts?: ScanOpts): Promise<void>;
+export interface SearchOpts {
+    clientId?: string;
+    limit?: number;
+    json?: boolean;
+}
+export declare function cmdSearch(query: string, opts?: SearchOpts): void;
+export interface StatusOpts {
+    clientId?: string;
+    json?: boolean;
+}
+export declare function cmdStatus(opts?: StatusOpts): void;
+export interface ForgetOpts {
+    clientId?: string;
+    confirm?: string;
+}
+export declare function cmdForget(opts?: ForgetOpts): void;

package/dist/brain/cli.js

@@ -0,0 +1,124 @@
+import chalk from "chalk";
+import { resolve } from "node:path";
+import { rmSync } from "node:fs";
+import { openDb, closeDb } from "./db/connection.js";
+import { registerRoot, scanAndIndex } from "./ingest/indexer.js";
+import { search } from "./search/bm25.js";
+import { brainEntitlement } from "../license/entitlements.js";
+import { loadLicense } from "../license.js";
+import { brainIndexDir } from "../paths.js";
+/**
+ * Resolve the active client id. v1: read from the license email or
+ * leedab.config.json. For now we accept an explicit --client flag and
+ * fall back to "default" — the single-tenant local install case.
+ */
+function resolveClientId(explicit) {
+    if (explicit)
+        return explicit;
+    return process.env.LEEDAB_CLIENT_ID ?? "default";
+}
+async function resolveTier() {
+    const license = await loadLicense().catch(() => null);
+    return license?.tier ?? "starter";
+}
+export async function cmdAddFolder(path, opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const absolute = resolve(path);
+    const rootId = opts.rootId ?? `external-${Date.now().toString(36)}`;
+    const root = {
+        id: rootId,
+        clientId,
+        kind: "external",
+        uri: absolute,
+        addedAt: Date.now(),
+    };
+    await registerRoot(root);
+    console.log(chalk.green(`  ✓ registered root ${chalk.bold(rootId)} → ${absolute}`));
+    console.log(chalk.dim(`  Run ${chalk.cyan("leedab brain scan")} to ingest.`));
+}
+export async function cmdScan(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const tier = await resolveTier();
+    const entitlement = brainEntitlement(tier);
+    if (!opts.json) {
+        console.log(chalk.bold(`  Scanning brain for ${chalk.cyan(clientId)} (tier: ${tier})...`));
+    }
+    const stats = await scanAndIndex(clientId, entitlement);
+    if (opts.json) {
+        process.stdout.write(JSON.stringify({ clientId, tier, stats }) + "\n");
+        return;
+    }
+    console.log(chalk.green("  ✓ scan complete"));
+    console.log(chalk.dim(`    roots: ${stats.rootsScanned} · seen: ${stats.filesSeen} · indexed: ${stats.filesIndexed} ` +
+        `· unchanged: ${stats.filesUnchanged} · skipped: ${stats.filesSkipped} ` +
+        `· deleted: ${stats.filesDeleted} · errors: ${stats.errors} · ${stats.durationMs}ms`));
+}
+export function cmdSearch(query, opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const hits = search(clientId, query, { limit: opts.limit ?? 10 });
+    if (opts.json) {
+        // Pure machine-readable output. Stdout is parseable by JSON.parse;
+        // no chalk, no extra lines. The clientId echoes back so the caller
+        // can confirm tenant scoping.
+        process.stdout.write(JSON.stringify({ clientId, query, hits }) + "\n");
+        return;
+    }
+    if (hits.length === 0) {
+        console.log(chalk.dim(`  No hits for ${chalk.bold(query)}.`));
+        console.log(chalk.dim(`  Index may be empty — run ${chalk.cyan("leedab brain scan")} first.`));
+        return;
+    }
+    console.log(chalk.bold(`  ${hits.length} hit(s) for ${chalk.cyan(query)}:\n`));
+    for (const hit of hits) {
+        console.log(`  ${chalk.bold(hit.sourceUri)} ${chalk.dim(`(${hit.rootId} · chunk ${hit.chunkOrd} · score ${hit.score.toFixed(2)})`)}`);
+        console.log(`    ${hit.snippet}\n`);
+    }
+}
+export function cmdStatus(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const db = openDb(clientId);
+    const roots = db.prepare("SELECT id, kind, uri FROM roots WHERE client_id = ? ORDER BY added_at").all(clientId);
+    const docCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ?").get(clientId).n;
+    const chunkCount = db.prepare(`SELECT COUNT(*) AS n FROM chunks
+       JOIN documents d ON d.id = chunks.doc_id
+       WHERE d.client_id = ?`).get(clientId).n;
+    const errorCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ? AND parse_error IS NOT NULL").get(clientId).n;
+    const skippedCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ? AND skipped_reason IS NOT NULL").get(clientId).n;
+    const lastScan = db.prepare("SELECT value FROM _meta WHERE key = 'last_full_scan_at'").get()?.value;
+    if (opts.json) {
+        process.stdout.write(JSON.stringify({
+            clientId,
+            indexDb: brainIndexDir(clientId) + "/index.db",
+            roots,
+            documents: docCount,
+            chunks: chunkCount,
+            skipped: skippedCount,
+            parseErrors: errorCount,
+            lastFullScanAt: lastScan ? parseInt(lastScan, 10) : null,
+        }) + "\n");
+        return;
+    }
+    console.log(chalk.bold(`  Brain status — ${chalk.cyan(clientId)}\n`));
+    console.log(`  Index DB:       ${chalk.dim(brainIndexDir(clientId) + "/index.db")}`);
+    console.log(`  Roots:          ${roots.length}`);
+    for (const r of roots) {
+        console.log(chalk.dim(`    ${r.id}  ${r.kind}  ${r.uri}`));
+    }
+    console.log(`  Documents:      ${docCount}`);
+    console.log(`  Chunks:         ${chunkCount}`);
+    console.log(`  Skipped:        ${skippedCount}`);
+    console.log(`  Parse errors:   ${errorCount}`);
+    console.log(`  Last full scan: ${lastScan ? new Date(parseInt(lastScan, 10)).toISOString() : chalk.dim("never")}`);
+}
+export function cmdForget(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    if (opts.confirm !== clientId) {
+        console.log(chalk.yellow(`  Refusing to wipe brain index. Re-run with ${chalk.bold(`--confirm ${clientId}`)} to confirm.`));
+        console.log(chalk.dim("  This only deletes the search-index cache; the markdown vault is untouched."));
+        return;
+    }
+    closeDb(clientId);
+    const dir = brainIndexDir(clientId);
+    rmSync(dir, { recursive: true, force: true });
+    console.log(chalk.green(`  ✓ wiped ${dir}`));
+}

package/dist/brain/db/connection.d.ts

@@ -0,0 +1,4 @@
+import Database from "better-sqlite3";
+export declare function openDb(clientId: string): Database.Database;
+export declare function closeDb(clientId: string): void;
+export declare function closeAll(): void;

package/dist/brain/db/connection.js

@@ -0,0 +1,38 @@
+import Database from "better-sqlite3";
+import { mkdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { brainIndexDir } from "../../paths.js";
+import { applyMigrations } from "./migrate.js";
+const SAFE_CLIENT_ID = /^[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?$/;
+const cache = new Map();
+export function openDb(clientId) {
+    if (!SAFE_CLIENT_ID.test(clientId)) {
+        throw new Error(`Invalid clientId: ${clientId}`);
+    }
+    const existing = cache.get(clientId);
+    if (existing && existing.open)
+        return existing;
+    const dir = brainIndexDir(clientId);
+    mkdirSync(dir, { recursive: true });
+    const dbPath = resolve(dir, "index.db");
+    const db = new Database(dbPath);
+    db.pragma("journal_mode = WAL");
+    db.pragma("synchronous = NORMAL");
+    db.pragma("foreign_keys = ON");
+    applyMigrations(db);
+    cache.set(clientId, db);
+    return db;
+}
+export function closeDb(clientId) {
+    const db = cache.get(clientId);
+    if (db?.open)
+        db.close();
+    cache.delete(clientId);
+}
+export function closeAll() {
+    for (const db of cache.values()) {
+        if (db.open)
+            db.close();
+    }
+    cache.clear();
+}

package/dist/brain/db/migrate.d.ts

@@ -0,0 +1,5 @@
+import type Database from "better-sqlite3";
+export declare function applyMigrations(db: Database.Database): {
+    applied: string[];
+    current: number;
+};

package/dist/brain/db/migrate.js

@@ -0,0 +1,38 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+const MIGRATIONS_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "migrations");
+export function applyMigrations(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS _migrations (
+      version INTEGER PRIMARY KEY,
+      name    TEXT NOT NULL,
+      applied_at INTEGER NOT NULL
+    )
+  `);
+    const appliedRows = db.prepare("SELECT version FROM _migrations ORDER BY version").all();
+    const applied = new Set(appliedRows.map((r) => r.version));
+    const files = readdirSync(MIGRATIONS_DIR)
+        .filter((f) => /^\d{4}_.*\.sql$/.test(f))
+        .sort();
+    const newlyApplied = [];
+    for (const file of files) {
+        const version = parseInt(file.slice(0, 4), 10);
+        if (applied.has(version))
+            continue;
+        const sql = readFileSync(resolve(MIGRATIONS_DIR, file), "utf-8");
+        db.exec("BEGIN");
+        try {
+            db.exec(sql);
+            db.prepare("INSERT INTO _migrations (version, name, applied_at) VALUES (?, ?, ?)").run(version, file, Date.now());
+            db.exec("COMMIT");
+            newlyApplied.push(file);
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw new Error(`Migration ${file} failed: ${err.message}`);
+        }
+    }
+    const current = Math.max(0, ...Array.from(applied), ...newlyApplied.map((f) => parseInt(f.slice(0, 4), 10)));
+    return { applied: newlyApplied, current };
+}

package/dist/brain/db/migrations/0001_init.sql

@@ -0,0 +1,134 @@
+-- Initial schema for the brain search-index cache.
+-- See docs/plans/brain-bootstrap.md §4.2.
+-- Per-customer DB; client_id present on every row so we never mix
+-- customers across queries even within a single DB.
+
+CREATE TABLE roots (
+  id              TEXT PRIMARY KEY,
+  client_id       TEXT NOT NULL,
+  kind            TEXT NOT NULL,                 -- 'vault' | 'external'
+  uri             TEXT NOT NULL,
+  added_at        INTEGER NOT NULL,
+  excludes_file   TEXT,
+  UNIQUE(client_id, uri)
+);
+
+CREATE TABLE documents (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  root_id         TEXT NOT NULL REFERENCES roots(id) ON DELETE CASCADE,
+  source_uri      TEXT NOT NULL,
+  mime            TEXT,
+  size            INTEGER,
+  mtime           INTEGER,
+  content_hash    TEXT,
+  parser_id       TEXT,
+  parser_version  INTEGER,
+  last_ingested   INTEGER NOT NULL,
+  parse_error     TEXT,
+  skipped_reason  TEXT,
+  UNIQUE(client_id, root_id, source_uri),
+  CHECK ((content_hash IS NOT NULL AND skipped_reason IS NULL)
+      OR (content_hash IS NULL AND skipped_reason IS NOT NULL))
+);
+
+CREATE TABLE chunks (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  ord             INTEGER NOT NULL,
+  text            TEXT NOT NULL,
+  embedding       BLOB,
+  embedding_model TEXT,
+  UNIQUE(doc_id, ord)
+);
+
+CREATE VIRTUAL TABLE chunks_fts USING fts5(
+  text, content='chunks', content_rowid='id', tokenize='porter unicode61'
+);
+
+CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
+  INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+
+CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text);
+END;
+
+CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text);
+  INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+
+CREATE TABLE entities (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  kind            TEXT NOT NULL,
+  name            TEXT NOT NULL,
+  norm            TEXT NOT NULL,
+  mentions        INTEGER NOT NULL DEFAULT 0,
+  UNIQUE(client_id, kind, norm)
+);
+
+CREATE TABLE entity_mentions (
+  entity_id       INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  chunk_ord       INTEGER,
+  UNIQUE(entity_id, doc_id, chunk_ord)
+);
+
+CREATE TABLE app_signals (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  app             TEXT NOT NULL,
+  rule_id         TEXT NOT NULL,
+  rule_version    INTEGER NOT NULL,
+  evidence_doc    INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  confidence      REAL NOT NULL,
+  detected_at     INTEGER NOT NULL,
+  UNIQUE(client_id, app, evidence_doc, rule_id, rule_version)
+);
+
+CREATE TABLE doc_tags (
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  tag             TEXT NOT NULL,
+  source          TEXT NOT NULL,
+  UNIQUE(doc_id, tag)
+);
+
+CREATE TABLE workflow_suggestions (
+  id               INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id        TEXT NOT NULL,
+  fingerprint      TEXT NOT NULL,
+  kind             TEXT NOT NULL,
+  workflow_id      TEXT,
+  title            TEXT NOT NULL,
+  rationale        TEXT NOT NULL,
+  confidence       REAL NOT NULL,
+  candidate_agent  TEXT,
+  status           TEXT NOT NULL DEFAULT 'pending',
+  promoted_to      TEXT,
+  dismissed_reason TEXT,
+  snoozed_until    INTEGER,
+  created_at       INTEGER NOT NULL,
+  updated_at       INTEGER NOT NULL,
+  UNIQUE(client_id, fingerprint)
+);
+
+CREATE TABLE workflow_suggestion_evidence (
+  suggestion_id   INTEGER NOT NULL REFERENCES workflow_suggestions(id) ON DELETE CASCADE,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  weight          REAL NOT NULL,
+  UNIQUE(suggestion_id, doc_id)
+);
+
+CREATE TABLE _meta (
+  key             TEXT PRIMARY KEY,
+  value           TEXT NOT NULL
+);
+
+CREATE INDEX idx_documents_root ON documents(client_id, root_id);
+CREATE INDEX idx_documents_hash ON documents(content_hash);
+CREATE INDEX idx_chunks_doc ON chunks(doc_id);
+CREATE INDEX idx_signals_app ON app_signals(client_id, app);
+CREATE INDEX idx_entities_norm ON entities(client_id, kind, norm);
+CREATE INDEX idx_suggestions_status ON workflow_suggestions(client_id, status);

package/dist/brain/ingest/classifier.d.ts

@@ -0,0 +1,6 @@
+export type ParserId = "text" | "spreadsheet" | "document" | "email";
+export interface Classification {
+    parser: ParserId;
+    mime: string;
+}
+export declare function classify(path: string): Classification | null;

package/dist/brain/ingest/classifier.js

@@ -0,0 +1,21 @@
+import { extname } from "node:path";
+const EXT_TO_PARSER = {
+    ".md": { parser: "text", mime: "text/markdown" },
+    ".markdown": { parser: "text", mime: "text/markdown" },
+    ".txt": { parser: "text", mime: "text/plain" },
+    ".json": { parser: "text", mime: "application/json" },
+    ".yaml": { parser: "text", mime: "text/yaml" },
+    ".yml": { parser: "text", mime: "text/yaml" },
+    ".csv": { parser: "text", mime: "text/csv" },
+    ".tsv": { parser: "text", mime: "text/tab-separated-values" },
+    ".xlsx": { parser: "spreadsheet", mime: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" },
+    ".ods": { parser: "spreadsheet", mime: "application/vnd.oasis.opendocument.spreadsheet" },
+    ".pdf": { parser: "document", mime: "application/pdf" },
+    ".docx": { parser: "document", mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" },
+    ".eml": { parser: "email", mime: "message/rfc822" },
+    ".mbox": { parser: "email", mime: "application/mbox" },
+};
+export function classify(path) {
+    const ext = extname(path).toLowerCase();
+    return EXT_TO_PARSER[ext] ?? null;
+}

package/dist/brain/ingest/indexer.d.ts

@@ -0,0 +1,4 @@
+import type { Root, ScanStats } from "../types.js";
+import type { BrainEntitlement } from "../../license/entitlements.js";
+export declare function registerRoot(root: Root): Promise<void>;
+export declare function scanAndIndex(clientId: string, entitlement: BrainEntitlement): Promise<ScanStats>;

package/dist/brain/ingest/indexer.js

@@ -0,0 +1,175 @@
+import { readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import { openDb } from "../db/connection.js";
+import { scanRoot } from "./scanner.js";
+import { classify } from "./classifier.js";
+import { parseText, PARSER_VERSION as TEXT_PARSER_VERSION } from "./parsers/text.js";
+export async function registerRoot(root) {
+    const db = openDb(root.clientId);
+    db.prepare(`INSERT INTO roots (id, client_id, kind, uri, added_at, excludes_file)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT(id) DO UPDATE SET
+       kind = excluded.kind,
+       uri = excluded.uri,
+       excludes_file = excluded.excludes_file`).run(root.id, root.clientId, root.kind, root.uri, root.addedAt, root.excludesFile ?? null);
+}
+export async function scanAndIndex(clientId, entitlement) {
+    const t0 = Date.now();
+    const db = openDb(clientId);
+    const ctx = { db, clientId, entitlement };
+    const stats = {
+        rootsScanned: 0,
+        filesSeen: 0,
+        filesIndexed: 0,
+        filesSkipped: 0,
+        filesUnchanged: 0,
+        filesDeleted: 0,
+        errors: 0,
+        durationMs: 0,
+    };
+    const roots = db.prepare("SELECT id, client_id, kind, uri, added_at, excludes_file FROM roots WHERE client_id = ?").all(clientId);
+    for (const row of roots) {
+        stats.rootsScanned++;
+        const seenUris = new Set();
+        const root = {
+            id: row.id,
+            clientId: row.client_id,
+            kind: row.kind,
+            uri: row.uri,
+            addedAt: row.added_at,
+            excludesFile: row.excludes_file ?? undefined,
+        };
+        for await (const scanned of scanRoot(root)) {
+            stats.filesSeen++;
+            seenUris.add(scanned.sourceUri);
+            try {
+                const outcome = await ingestOne(ctx, scanned);
+                if (outcome === "indexed")
+                    stats.filesIndexed++;
+                else if (outcome === "unchanged")
+                    stats.filesUnchanged++;
+                else
+                    stats.filesSkipped++;
+            }
+            catch (err) {
+                stats.errors++;
+                recordParseError(ctx, scanned, err.message);
+            }
+        }
+        stats.filesDeleted += cleanupVanished(ctx, row.id, seenUris);
+    }
+    updateMeta(db, "last_full_scan_at", Date.now().toString());
+    stats.durationMs = Date.now() - t0;
+    return stats;
+}
+async function ingestOne(ctx, scanned) {
+    const { db, clientId, entitlement } = ctx;
+    if (scanned.skippedReason) {
+        upsertSkipped(db, clientId, scanned, scanned.skippedReason);
+        return "skipped";
+    }
+    const cls = classify(scanned.sourceUri);
+    if (!cls) {
+        upsertSkipped(db, clientId, scanned, "unsupported");
+        return "skipped";
+    }
+    if (!entitlement.parsers.includes(cls.parser)) {
+        upsertSkipped(db, clientId, scanned, "unsupported");
+        return "skipped";
+    }
+    if (cls.parser !== "text") {
+        // Phase 1 ships text parser only. Other parsers land in Phase 5+.
+        upsertSkipped(db, clientId, scanned, "unsupported");
+        return "skipped";
+    }
+    const bytes = await readFile(scanned.sourceUri);
+    const contentHash = sha256(bytes);
+    const existing = db.prepare(`SELECT id, content_hash, parser_version
+       FROM documents
+      WHERE client_id = ? AND root_id = ? AND source_uri = ?`).get(clientId, scanned.rootId, scanned.sourceUri);
+    if (existing &&
+        existing.content_hash === contentHash &&
+        existing.parser_version === TEXT_PARSER_VERSION) {
+        return "unchanged";
+    }
+    const parsed = await parseText(scanned.sourceUri);
+    const tx = db.transaction(() => {
+        let docId;
+        if (existing) {
+            db.prepare(`UPDATE documents
+            SET mime = ?, size = ?, mtime = ?, content_hash = ?,
+                parser_id = ?, parser_version = ?, last_ingested = ?,
+                parse_error = NULL, skipped_reason = NULL
+          WHERE id = ?`).run(cls.mime, scanned.size, scanned.mtime, contentHash, parsed.parserId, parsed.parserVersion, Date.now(), existing.id);
+            docId = existing.id;
+            db.prepare("DELETE FROM chunks WHERE doc_id = ?").run(docId);
+        }
+        else {
+            const info = db.prepare(`INSERT INTO documents
+           (client_id, root_id, source_uri, mime, size, mtime,
+            content_hash, parser_id, parser_version, last_ingested)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(clientId, scanned.rootId, scanned.sourceUri, cls.mime, scanned.size, scanned.mtime, contentHash, parsed.parserId, parsed.parserVersion, Date.now());
+            docId = Number(info.lastInsertRowid);
+        }
+        if (parsed.chunks.length > 0) {
+            const insertChunk = db.prepare("INSERT INTO chunks (doc_id, ord, text) VALUES (?, ?, ?)");
+            for (const chunk of parsed.chunks) {
+                insertChunk.run(docId, chunk.ord, chunk.text);
+            }
+        }
+    });
+    tx();
+    return "indexed";
+}
+function upsertSkipped(db, clientId, scanned, reason) {
+    const existing = db.prepare(`SELECT id FROM documents WHERE client_id = ? AND root_id = ? AND source_uri = ?`).get(clientId, scanned.rootId, scanned.sourceUri);
+    if (existing) {
+        db.prepare(`UPDATE documents
+          SET mime = NULL, size = ?, mtime = ?, content_hash = NULL,
+              parser_id = NULL, parser_version = NULL, last_ingested = ?,
+              parse_error = NULL, skipped_reason = ?
+        WHERE id = ?`).run(scanned.size, scanned.mtime, Date.now(), reason, existing.id);
+        db.prepare("DELETE FROM chunks WHERE doc_id = ?").run(existing.id);
+        return;
+    }
+    db.prepare(`INSERT INTO documents
+       (client_id, root_id, source_uri, size, mtime, last_ingested, skipped_reason)
+     VALUES (?, ?, ?, ?, ?, ?, ?)`).run(clientId, scanned.rootId, scanned.sourceUri, scanned.size, scanned.mtime, Date.now(), reason);
+}
+function recordParseError(ctx, scanned, message) {
+    const { db, clientId } = ctx;
+    const existing = db.prepare(`SELECT id FROM documents WHERE client_id = ? AND root_id = ? AND source_uri = ?`).get(clientId, scanned.rootId, scanned.sourceUri);
+    if (existing) {
+        db.prepare(`UPDATE documents
+          SET parse_error = ?, skipped_reason = ?, last_ingested = ?,
+              content_hash = NULL, parser_id = NULL, parser_version = NULL
+        WHERE id = ?`).run(message.slice(0, 500), "parse-error", Date.now(), existing.id);
+        db.prepare("DELETE FROM chunks WHERE doc_id = ?").run(existing.id);
+    }
+    else {
+        db.prepare(`INSERT INTO documents
+         (client_id, root_id, source_uri, size, mtime, last_ingested,
+          parse_error, skipped_reason)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(clientId, scanned.rootId, scanned.sourceUri, scanned.size, scanned.mtime, Date.now(), message.slice(0, 500), "parse-error");
+    }
+}
+function cleanupVanished(ctx, rootId, seen) {
+    const { db, clientId } = ctx;
+    const known = db.prepare("SELECT id, source_uri FROM documents WHERE client_id = ? AND root_id = ?").all(clientId, rootId);
+    let removed = 0;
+    const del = db.prepare("DELETE FROM documents WHERE id = ?");
+    for (const row of known) {
+        if (!seen.has(row.source_uri)) {
+            del.run(row.id);
+            removed++;
+        }
+    }
+    return removed;
+}
+function updateMeta(db, key, value) {
+    db.prepare(`INSERT INTO _meta (key, value) VALUES (?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(key, value);
+}
+function sha256(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}

package/dist/brain/ingest/parsers/text.d.ts

@@ -0,0 +1,4 @@
+import type { ParsedDoc } from "../../types.js";
+export declare const PARSER_ID = "text";
+export declare const PARSER_VERSION = 1;
+export declare function parseText(path: string): Promise<ParsedDoc>;

package/dist/brain/ingest/parsers/text.js

@@ -0,0 +1,38 @@
+import { readFile } from "node:fs/promises";
+export const PARSER_ID = "text";
+export const PARSER_VERSION = 1;
+const TARGET_CHARS = 2048;
+const OVERLAP_CHARS = 256;
+export async function parseText(path) {
+    const buffer = await readFile(path);
+    const text = stripBom(buffer.toString("utf-8"));
+    const chunks = chunkText(text);
+    return { chunks, parserId: PARSER_ID, parserVersion: PARSER_VERSION };
+}
+function stripBom(s) {
+    return s.charCodeAt(0) === 0xfeff ? s.slice(1) : s;
+}
+function chunkText(text) {
+    if (text.trim().length === 0)
+        return [];
+    if (text.length <= TARGET_CHARS) {
+        return [{ ord: 0, text: text.trim() }];
+    }
+    const out = [];
+    let start = 0;
+    let ord = 0;
+    while (start < text.length) {
+        const end = Math.min(start + TARGET_CHARS, text.length);
+        const slice = text.slice(start, end).trim();
+        if (slice.length > 0) {
+            out.push({ ord, text: slice });
+            ord++;
+        }
+        if (end >= text.length)
+            break;
+        start = end - OVERLAP_CHARS;
+        if (start < 0)
+            start = 0;
+    }
+    return out;
+}

package/dist/brain/ingest/scanner.d.ts

@@ -0,0 +1,19 @@
+import { relative } from "node:path";
+import type { Root, RootKind } from "../types.js";
+export interface ScannedFile {
+    rootId: string;
+    rootKind: RootKind;
+    sourceUri: string;
+    size: number;
+    mtime: number;
+    skippedReason?: "oversize" | "denied" | "binary";
+}
+export declare function scanRoot(root: Root): AsyncGenerator<ScannedFile>;
+declare function isHardDenied(absPath: string): boolean;
+export declare const _internal: {
+    isHardDenied: typeof isHardDenied;
+    VAULT_ALLOWED_PREFIXES: string[];
+    DEFAULT_SKIP_EXTENSIONS: Set<string>;
+    MAX_FILE_SIZE: number;
+};
+export { relative as _rel };