leedab 0.8.0 → 0.9.1

package/bin/leedab.js

@@ -1304,6 +1304,91 @@ gtm
     }
   });
 
+// ---------------------------------------------------------------------------
+// `leedab brain` — search-index over the customer's markdown vault plus any
+// registered external folders. See docs/plans/brain-bootstrap.md for the
+// full design. Phase 1 ships scan / search / status / forget / add-folder.
+// ---------------------------------------------------------------------------
+const brain = program
+  .command("brain")
+  .description("Search-index over the customer's vault + registered external folders");
+
+brain
+  .command("add-folder <path>")
+  .description("Register an external folder as a brain source")
+  .option("-c, --client <id>", "Client id (default: $LEEDAB_CLIENT_ID or 'default')")
+  .option("--root-id <id>", "Explicit root id (default: auto-generated)")
+  .action(async (path, opts) => {
+    const { cmdAddFolder } = await import("../dist/brain/cli.js");
+    try {
+      await cmdAddFolder(path, { clientId: opts.client, rootId: opts.rootId });
+    } catch (err) {
+      console.error(chalk.red(`Add-folder failed: ${err.message}`));
+      process.exit(1);
+    }
+  });
+
+brain
+  .command("scan")
+  .description("Scan every registered root and index changed files")
+  .option("-c, --client <id>", "Client id (default: $LEEDAB_CLIENT_ID or 'default')")
+  .option("--json", "Emit machine-readable JSON instead of formatted output")
+  .action(async (opts) => {
+    const { cmdScan } = await import("../dist/brain/cli.js");
+    try {
+      await cmdScan({ clientId: opts.client, json: opts.json });
+    } catch (err) {
+      console.error(chalk.red(`Scan failed: ${err.message}`));
+      process.exit(1);
+    }
+  });
+
+brain
+  .command("search <query...>")
+  .description("BM25 search across indexed chunks")
+  .option("-c, --client <id>", "Client id")
+  .option("-n, --limit <n>", "Max hits", (v) => parseInt(v, 10), 10)
+  .option("--json", "Emit machine-readable JSON instead of formatted output")
+  .action(async (query, opts) => {
+    const { cmdSearch } = await import("../dist/brain/cli.js");
+    try {
+      cmdSearch(query.join(" "), { clientId: opts.client, limit: opts.limit, json: opts.json });
+    } catch (err) {
+      console.error(chalk.red(`Search failed: ${err.message}`));
+      process.exit(1);
+    }
+  });
+
+brain
+  .command("status")
+  .description("Brain index status — doc count, errors, last scan")
+  .option("-c, --client <id>", "Client id")
+  .option("--json", "Emit machine-readable JSON instead of formatted output")
+  .action(async (opts) => {
+    const { cmdStatus } = await import("../dist/brain/cli.js");
+    try {
+      cmdStatus({ clientId: opts.client, json: opts.json });
+    } catch (err) {
+      console.error(chalk.red(`Status failed: ${err.message}`));
+      process.exit(1);
+    }
+  });
+
+brain
+  .command("forget")
+  .description("Wipe the brain search-index cache (markdown vault is untouched)")
+  .option("-c, --client <id>", "Client id")
+  .requiredOption("--confirm <clientId>", "Pass the client id to confirm")
+  .action(async (opts) => {
+    const { cmdForget } = await import("../dist/brain/cli.js");
+    try {
+      cmdForget({ clientId: opts.client, confirm: opts.confirm });
+    } catch (err) {
+      console.error(chalk.red(`Forget failed: ${err.message}`));
+      process.exit(1);
+    }
+  });
+
 // Default action: show status if in a project, nudge to onboard if not
 if (process.argv.length <= 2) {
   if (!existsSync(STATE_DIR)) {

package/clients/default/manifest.json

@@ -190,7 +190,13 @@
       "The Champion is asking the same routing question for the third time — surface a pattern + adjust pacing."
     ]
   },
-  "hidden_agents": [],
+  "hidden_agents": [
+    "video",
+    "campaigns",
+    "automation",
+    "concierge",
+    "content"
+  ],
   "custom_agents": [
     {
       "id": "x-gtm-engineer-wgby",

package/dist/brain/cli.d.ts

@@ -0,0 +1,26 @@
+export interface AddFolderOpts {
+    clientId?: string;
+    rootId?: string;
+}
+export declare function cmdAddFolder(path: string, opts?: AddFolderOpts): Promise<void>;
+export interface ScanOpts {
+    clientId?: string;
+    json?: boolean;
+}
+export declare function cmdScan(opts?: ScanOpts): Promise<void>;
+export interface SearchOpts {
+    clientId?: string;
+    limit?: number;
+    json?: boolean;
+}
+export declare function cmdSearch(query: string, opts?: SearchOpts): void;
+export interface StatusOpts {
+    clientId?: string;
+    json?: boolean;
+}
+export declare function cmdStatus(opts?: StatusOpts): void;
+export interface ForgetOpts {
+    clientId?: string;
+    confirm?: string;
+}
+export declare function cmdForget(opts?: ForgetOpts): void;

package/dist/brain/cli.js

@@ -0,0 +1,124 @@
+import chalk from "chalk";
+import { resolve } from "node:path";
+import { rmSync } from "node:fs";
+import { openDb, closeDb } from "./db/connection.js";
+import { registerRoot, scanAndIndex } from "./ingest/indexer.js";
+import { search } from "./search/bm25.js";
+import { brainEntitlement } from "../license/entitlements.js";
+import { loadLicense } from "../license.js";
+import { brainIndexDir } from "../paths.js";
+/**
+ * Resolve the active client id. v1: read from the license email or
+ * leedab.config.json. For now we accept an explicit --client flag and
+ * fall back to "default" — the single-tenant local install case.
+ */
+function resolveClientId(explicit) {
+    if (explicit)
+        return explicit;
+    return process.env.LEEDAB_CLIENT_ID ?? "default";
+}
+async function resolveTier() {
+    const license = await loadLicense().catch(() => null);
+    return license?.tier ?? "starter";
+}
+export async function cmdAddFolder(path, opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const absolute = resolve(path);
+    const rootId = opts.rootId ?? `external-${Date.now().toString(36)}`;
+    const root = {
+        id: rootId,
+        clientId,
+        kind: "external",
+        uri: absolute,
+        addedAt: Date.now(),
+    };
+    await registerRoot(root);
+    console.log(chalk.green(`  ✓ registered root ${chalk.bold(rootId)} → ${absolute}`));
+    console.log(chalk.dim(`  Run ${chalk.cyan("leedab brain scan")} to ingest.`));
+}
+export async function cmdScan(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const tier = await resolveTier();
+    const entitlement = brainEntitlement(tier);
+    if (!opts.json) {
+        console.log(chalk.bold(`  Scanning brain for ${chalk.cyan(clientId)} (tier: ${tier})...`));
+    }
+    const stats = await scanAndIndex(clientId, entitlement);
+    if (opts.json) {
+        process.stdout.write(JSON.stringify({ clientId, tier, stats }) + "\n");
+        return;
+    }
+    console.log(chalk.green("  ✓ scan complete"));
+    console.log(chalk.dim(`    roots: ${stats.rootsScanned} · seen: ${stats.filesSeen} · indexed: ${stats.filesIndexed} ` +
+        `· unchanged: ${stats.filesUnchanged} · skipped: ${stats.filesSkipped} ` +
+        `· deleted: ${stats.filesDeleted} · errors: ${stats.errors} · ${stats.durationMs}ms`));
+}
+export function cmdSearch(query, opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const hits = search(clientId, query, { limit: opts.limit ?? 10 });
+    if (opts.json) {
+        // Pure machine-readable output. Stdout is parseable by JSON.parse;
+        // no chalk, no extra lines. The clientId echoes back so the caller
+        // can confirm tenant scoping.
+        process.stdout.write(JSON.stringify({ clientId, query, hits }) + "\n");
+        return;
+    }
+    if (hits.length === 0) {
+        console.log(chalk.dim(`  No hits for ${chalk.bold(query)}.`));
+        console.log(chalk.dim(`  Index may be empty — run ${chalk.cyan("leedab brain scan")} first.`));
+        return;
+    }
+    console.log(chalk.bold(`  ${hits.length} hit(s) for ${chalk.cyan(query)}:\n`));
+    for (const hit of hits) {
+        console.log(`  ${chalk.bold(hit.sourceUri)} ${chalk.dim(`(${hit.rootId} · chunk ${hit.chunkOrd} · score ${hit.score.toFixed(2)})`)}`);
+        console.log(`    ${hit.snippet}\n`);
+    }
+}
+export function cmdStatus(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    const db = openDb(clientId);
+    const roots = db.prepare("SELECT id, kind, uri FROM roots WHERE client_id = ? ORDER BY added_at").all(clientId);
+    const docCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ?").get(clientId).n;
+    const chunkCount = db.prepare(`SELECT COUNT(*) AS n FROM chunks
+       JOIN documents d ON d.id = chunks.doc_id
+       WHERE d.client_id = ?`).get(clientId).n;
+    const errorCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ? AND parse_error IS NOT NULL").get(clientId).n;
+    const skippedCount = db.prepare("SELECT COUNT(*) AS n FROM documents WHERE client_id = ? AND skipped_reason IS NOT NULL").get(clientId).n;
+    const lastScan = db.prepare("SELECT value FROM _meta WHERE key = 'last_full_scan_at'").get()?.value;
+    if (opts.json) {
+        process.stdout.write(JSON.stringify({
+            clientId,
+            indexDb: brainIndexDir(clientId) + "/index.db",
+            roots,
+            documents: docCount,
+            chunks: chunkCount,
+            skipped: skippedCount,
+            parseErrors: errorCount,
+            lastFullScanAt: lastScan ? parseInt(lastScan, 10) : null,
+        }) + "\n");
+        return;
+    }
+    console.log(chalk.bold(`  Brain status — ${chalk.cyan(clientId)}\n`));
+    console.log(`  Index DB:       ${chalk.dim(brainIndexDir(clientId) + "/index.db")}`);
+    console.log(`  Roots:          ${roots.length}`);
+    for (const r of roots) {
+        console.log(chalk.dim(`    ${r.id}  ${r.kind}  ${r.uri}`));
+    }
+    console.log(`  Documents:      ${docCount}`);
+    console.log(`  Chunks:         ${chunkCount}`);
+    console.log(`  Skipped:        ${skippedCount}`);
+    console.log(`  Parse errors:   ${errorCount}`);
+    console.log(`  Last full scan: ${lastScan ? new Date(parseInt(lastScan, 10)).toISOString() : chalk.dim("never")}`);
+}
+export function cmdForget(opts = {}) {
+    const clientId = resolveClientId(opts.clientId);
+    if (opts.confirm !== clientId) {
+        console.log(chalk.yellow(`  Refusing to wipe brain index. Re-run with ${chalk.bold(`--confirm ${clientId}`)} to confirm.`));
+        console.log(chalk.dim("  This only deletes the search-index cache; the markdown vault is untouched."));
+        return;
+    }
+    closeDb(clientId);
+    const dir = brainIndexDir(clientId);
+    rmSync(dir, { recursive: true, force: true });
+    console.log(chalk.green(`  ✓ wiped ${dir}`));
+}

package/dist/brain/db/connection.d.ts

@@ -0,0 +1,4 @@
+import Database from "better-sqlite3";
+export declare function openDb(clientId: string): Database.Database;
+export declare function closeDb(clientId: string): void;
+export declare function closeAll(): void;

package/dist/brain/db/connection.js

@@ -0,0 +1,38 @@
+import Database from "better-sqlite3";
+import { mkdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { brainIndexDir } from "../../paths.js";
+import { applyMigrations } from "./migrate.js";
+const SAFE_CLIENT_ID = /^[a-z0-9](?:[a-z0-9-]{0,58}[a-z0-9])?$/;
+const cache = new Map();
+export function openDb(clientId) {
+    if (!SAFE_CLIENT_ID.test(clientId)) {
+        throw new Error(`Invalid clientId: ${clientId}`);
+    }
+    const existing = cache.get(clientId);
+    if (existing && existing.open)
+        return existing;
+    const dir = brainIndexDir(clientId);
+    mkdirSync(dir, { recursive: true });
+    const dbPath = resolve(dir, "index.db");
+    const db = new Database(dbPath);
+    db.pragma("journal_mode = WAL");
+    db.pragma("synchronous = NORMAL");
+    db.pragma("foreign_keys = ON");
+    applyMigrations(db);
+    cache.set(clientId, db);
+    return db;
+}
+export function closeDb(clientId) {
+    const db = cache.get(clientId);
+    if (db?.open)
+        db.close();
+    cache.delete(clientId);
+}
+export function closeAll() {
+    for (const db of cache.values()) {
+        if (db.open)
+            db.close();
+    }
+    cache.clear();
+}

package/dist/brain/db/migrate.d.ts

@@ -0,0 +1,5 @@
+import type Database from "better-sqlite3";
+export declare function applyMigrations(db: Database.Database): {
+    applied: string[];
+    current: number;
+};

package/dist/brain/db/migrate.js

@@ -0,0 +1,38 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+const MIGRATIONS_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "migrations");
+export function applyMigrations(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS _migrations (
+      version INTEGER PRIMARY KEY,
+      name    TEXT NOT NULL,
+      applied_at INTEGER NOT NULL
+    )
+  `);
+    const appliedRows = db.prepare("SELECT version FROM _migrations ORDER BY version").all();
+    const applied = new Set(appliedRows.map((r) => r.version));
+    const files = readdirSync(MIGRATIONS_DIR)
+        .filter((f) => /^\d{4}_.*\.sql$/.test(f))
+        .sort();
+    const newlyApplied = [];
+    for (const file of files) {
+        const version = parseInt(file.slice(0, 4), 10);
+        if (applied.has(version))
+            continue;
+        const sql = readFileSync(resolve(MIGRATIONS_DIR, file), "utf-8");
+        db.exec("BEGIN");
+        try {
+            db.exec(sql);
+            db.prepare("INSERT INTO _migrations (version, name, applied_at) VALUES (?, ?, ?)").run(version, file, Date.now());
+            db.exec("COMMIT");
+            newlyApplied.push(file);
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw new Error(`Migration ${file} failed: ${err.message}`);
+        }
+    }
+    const current = Math.max(0, ...Array.from(applied), ...newlyApplied.map((f) => parseInt(f.slice(0, 4), 10)));
+    return { applied: newlyApplied, current };
+}

package/dist/brain/db/migrations/0001_init.sql

@@ -0,0 +1,134 @@
+-- Initial schema for the brain search-index cache.
+-- See docs/plans/brain-bootstrap.md §4.2.
+-- Per-customer DB; client_id present on every row so we never mix
+-- customers across queries even within a single DB.
+
+CREATE TABLE roots (
+  id              TEXT PRIMARY KEY,
+  client_id       TEXT NOT NULL,
+  kind            TEXT NOT NULL,                 -- 'vault' | 'external'
+  uri             TEXT NOT NULL,
+  added_at        INTEGER NOT NULL,
+  excludes_file   TEXT,
+  UNIQUE(client_id, uri)
+);
+
+CREATE TABLE documents (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  root_id         TEXT NOT NULL REFERENCES roots(id) ON DELETE CASCADE,
+  source_uri      TEXT NOT NULL,
+  mime            TEXT,
+  size            INTEGER,
+  mtime           INTEGER,
+  content_hash    TEXT,
+  parser_id       TEXT,
+  parser_version  INTEGER,
+  last_ingested   INTEGER NOT NULL,
+  parse_error     TEXT,
+  skipped_reason  TEXT,
+  UNIQUE(client_id, root_id, source_uri),
+  CHECK ((content_hash IS NOT NULL AND skipped_reason IS NULL)
+      OR (content_hash IS NULL AND skipped_reason IS NOT NULL))
+);
+
+CREATE TABLE chunks (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  ord             INTEGER NOT NULL,
+  text            TEXT NOT NULL,
+  embedding       BLOB,
+  embedding_model TEXT,
+  UNIQUE(doc_id, ord)
+);
+
+CREATE VIRTUAL TABLE chunks_fts USING fts5(
+  text, content='chunks', content_rowid='id', tokenize='porter unicode61'
+);
+
+CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
+  INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+
+CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text);
+END;
+
+CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
+  INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text);
+  INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text);
+END;
+
+CREATE TABLE entities (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  kind            TEXT NOT NULL,
+  name            TEXT NOT NULL,
+  norm            TEXT NOT NULL,
+  mentions        INTEGER NOT NULL DEFAULT 0,
+  UNIQUE(client_id, kind, norm)
+);
+
+CREATE TABLE entity_mentions (
+  entity_id       INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  chunk_ord       INTEGER,
+  UNIQUE(entity_id, doc_id, chunk_ord)
+);
+
+CREATE TABLE app_signals (
+  id              INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id       TEXT NOT NULL,
+  app             TEXT NOT NULL,
+  rule_id         TEXT NOT NULL,
+  rule_version    INTEGER NOT NULL,
+  evidence_doc    INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  confidence      REAL NOT NULL,
+  detected_at     INTEGER NOT NULL,
+  UNIQUE(client_id, app, evidence_doc, rule_id, rule_version)
+);
+
+CREATE TABLE doc_tags (
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  tag             TEXT NOT NULL,
+  source          TEXT NOT NULL,
+  UNIQUE(doc_id, tag)
+);
+
+CREATE TABLE workflow_suggestions (
+  id               INTEGER PRIMARY KEY AUTOINCREMENT,
+  client_id        TEXT NOT NULL,
+  fingerprint      TEXT NOT NULL,
+  kind             TEXT NOT NULL,
+  workflow_id      TEXT,
+  title            TEXT NOT NULL,
+  rationale        TEXT NOT NULL,
+  confidence       REAL NOT NULL,
+  candidate_agent  TEXT,
+  status           TEXT NOT NULL DEFAULT 'pending',
+  promoted_to      TEXT,
+  dismissed_reason TEXT,
+  snoozed_until    INTEGER,
+  created_at       INTEGER NOT NULL,
+  updated_at       INTEGER NOT NULL,
+  UNIQUE(client_id, fingerprint)
+);
+
+CREATE TABLE workflow_suggestion_evidence (
+  suggestion_id   INTEGER NOT NULL REFERENCES workflow_suggestions(id) ON DELETE CASCADE,
+  doc_id          INTEGER NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+  weight          REAL NOT NULL,
+  UNIQUE(suggestion_id, doc_id)
+);
+
+CREATE TABLE _meta (
+  key             TEXT PRIMARY KEY,
+  value           TEXT NOT NULL
+);
+
+CREATE INDEX idx_documents_root ON documents(client_id, root_id);
+CREATE INDEX idx_documents_hash ON documents(content_hash);
+CREATE INDEX idx_chunks_doc ON chunks(doc_id);
+CREATE INDEX idx_signals_app ON app_signals(client_id, app);
+CREATE INDEX idx_entities_norm ON entities(client_id, kind, norm);
+CREATE INDEX idx_suggestions_status ON workflow_suggestions(client_id, status);

package/dist/brain/ingest/classifier.d.ts

@@ -0,0 +1,6 @@
+export type ParserId = "text" | "spreadsheet" | "document" | "email";
+export interface Classification {
+    parser: ParserId;
+    mime: string;
+}
+export declare function classify(path: string): Classification | null;

package/dist/brain/ingest/classifier.js

@@ -0,0 +1,21 @@
+import { extname } from "node:path";
+const EXT_TO_PARSER = {
+    ".md": { parser: "text", mime: "text/markdown" },
+    ".markdown": { parser: "text", mime: "text/markdown" },
+    ".txt": { parser: "text", mime: "text/plain" },
+    ".json": { parser: "text", mime: "application/json" },
+    ".yaml": { parser: "text", mime: "text/yaml" },
+    ".yml": { parser: "text", mime: "text/yaml" },
+    ".csv": { parser: "text", mime: "text/csv" },
+    ".tsv": { parser: "text", mime: "text/tab-separated-values" },
+    ".xlsx": { parser: "spreadsheet", mime: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" },
+    ".ods": { parser: "spreadsheet", mime: "application/vnd.oasis.opendocument.spreadsheet" },
+    ".pdf": { parser: "document", mime: "application/pdf" },
+    ".docx": { parser: "document", mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document" },
+    ".eml": { parser: "email", mime: "message/rfc822" },
+    ".mbox": { parser: "email", mime: "application/mbox" },
+};
+export function classify(path) {
+    const ext = extname(path).toLowerCase();
+    return EXT_TO_PARSER[ext] ?? null;
+}

package/dist/brain/ingest/indexer.d.ts

@@ -0,0 +1,4 @@
+import type { Root, ScanStats } from "../types.js";
+import type { BrainEntitlement } from "../../license/entitlements.js";
+export declare function registerRoot(root: Root): Promise<void>;
+export declare function scanAndIndex(clientId: string, entitlement: BrainEntitlement): Promise<ScanStats>;