@toolbaux/guardian 0.1.22 → 0.1.23

package/dist/db/file-specs-store.js

@@ -0,0 +1,174 @@
+/**
+ * FileSpecsStore — filesystem implementation of SpecsStore.
+ *
+ * This is a drop-in wrapper around the existing file-based behavior.
+ * It maps each SpecsStore call to the exact same read/write the codebase
+ * did before the interface existed, so no existing behavior changes.
+ *
+ * File layout (unchanged):
+ *   <machineDir>/
+ *     architecture.snapshot.yaml
+ *     ux.snapshot.yaml
+ *     codebase-intelligence.json
+ *     structural-intelligence.json
+ *     function-intelligence.json
+ *     mcp-metrics.jsonl
+ *   <humanDir>/
+ *     overview.md
+ *     modules/
+ *       src-extract.md
+ *       ...
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+/** Maps spec name → filename on disk. */
+const SPEC_FILENAMES = {
+    "architecture.snapshot": "architecture.snapshot.yaml",
+    "ux.snapshot": "ux.snapshot.yaml",
+    "codebase-intelligence": "codebase-intelligence.json",
+    "structural-intelligence": "structural-intelligence.json",
+    "function-intelligence": "function-intelligence.json",
+    "mcp-metrics": "mcp-metrics.jsonl",
+};
+function nameToFilename(name) {
+    return SPEC_FILENAMES[name] ?? `${name}.json`;
+}
+function filenameToFormat(filename) {
+    if (filename.endsWith(".yaml"))
+        return "yaml";
+    if (filename.endsWith(".jsonl"))
+        return "jsonl";
+    if (filename.endsWith(".json"))
+        return "json";
+    return "text";
+}
+export class FileSpecsStore {
+    machineDir;
+    humanDir;
+    constructor(machineDir, humanDir) {
+        this.machineDir = machineDir;
+        this.humanDir = humanDir;
+    }
+    async init() {
+        await fs.mkdir(this.machineDir, { recursive: true });
+        await fs.mkdir(this.humanDir, { recursive: true });
+    }
+    async close() {
+        // nothing to close for file IO
+    }
+    // ── Spec blobs ─────────────────────────────────────────────────────────────
+    async readSpec(name) {
+        const filename = nameToFilename(name);
+        const filePath = path.join(this.machineDir, filename);
+        try {
+            const content = await fs.readFile(filePath, "utf8");
+            const stat = await fs.stat(filePath);
+            return {
+                name,
+                format: filenameToFormat(filename),
+                content,
+                tier: "free",
+                updatedAt: stat.mtimeMs,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    async writeSpec(name, content, format, tier = "free") {
+        const filename = nameToFilename(name);
+        await fs.writeFile(path.join(this.machineDir, filename), content, "utf8");
+    }
+    async listSpecs() {
+        try {
+            const entries = await fs.readdir(this.machineDir);
+            return entries
+                .filter(e => e.endsWith(".json") || e.endsWith(".yaml") || e.endsWith(".jsonl"))
+                .map(e => {
+                const found = Object.entries(SPEC_FILENAMES).find(([, v]) => v === e);
+                return found ? found[0] : e;
+            });
+        }
+        catch {
+            return [];
+        }
+    }
+    async hasSpec(name) {
+        const filename = nameToFilename(name);
+        try {
+            await fs.stat(path.join(this.machineDir, filename));
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // ── Human docs ─────────────────────────────────────────────────────────────
+    async readDoc(id) {
+        const filePath = this._docPath(id);
+        try {
+            const body = await fs.readFile(filePath, "utf8");
+            const stat = await fs.stat(filePath);
+            const title = body.match(/^#\s+(.+)$/m)?.[1] ?? id;
+            return { id, section: id.split(":")[0], title, body, tier: "free", updatedAt: stat.mtimeMs };
+        }
+        catch {
+            return null;
+        }
+    }
+    async writeDoc(entry) {
+        const filePath = this._docPath(entry.id);
+        await fs.mkdir(path.dirname(filePath), { recursive: true });
+        await fs.writeFile(filePath, entry.body, "utf8");
+    }
+    async listDocs(section) {
+        const results = [];
+        await this._walkDocs(this.humanDir, results, section);
+        return results;
+    }
+    async _walkDocs(dir, acc, section) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir);
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            const full = path.join(dir, e);
+            const stat = await fs.stat(full);
+            if (stat.isDirectory()) {
+                await this._walkDocs(full, acc, section);
+            }
+            else if (e.endsWith(".md")) {
+                const id = path.relative(this.humanDir, full).replace(/\.md$/, "").replace(/\\/g, "/");
+                if (section && !id.startsWith(section))
+                    continue;
+                const body = await fs.readFile(full, "utf8");
+                const title = body.match(/^#\s+(.+)$/m)?.[1] ?? id;
+                acc.push({ id, section: id.split("/")[0], title, body, tier: "free", updatedAt: stat.mtimeMs });
+            }
+        }
+    }
+    _docPath(id) {
+        return path.join(this.humanDir, `${id.replace(/:/g, "/")}.md`);
+    }
+    // ── Metrics log ────────────────────────────────────────────────────────────
+    async appendMetric(event, payload) {
+        const line = JSON.stringify({ ts: Date.now(), event, payload }) + "\n";
+        await fs.appendFile(path.join(this.machineDir, "mcp-metrics.jsonl"), line, "utf8");
+    }
+    async readMetrics(limit = 1000) {
+        try {
+            const raw = await fs.readFile(path.join(this.machineDir, "mcp-metrics.jsonl"), "utf8");
+            return raw
+                .split("\n")
+                .filter(Boolean)
+                .slice(-limit)
+                .map(l => JSON.parse(l));
+        }
+        catch {
+            return [];
+        }
+    }
+}

package/dist/db/fts-builder.js

@@ -0,0 +1,305 @@
+/**
+ * FTS index builder — converts codebase-intelligence.json into FTS5 rows.
+ *
+ * One row per file: aggregates all endpoints, symbols, and descriptions
+ * associated with that file so BM25 can rank files, not individual records.
+ *
+ * This is what fixes cases like csharp-auth-001 where the current linear
+ * scorer misses Users/Login.cs because "auth" doesn't appear literally in
+ * the file path — BM25 + porter stemmer ranks it via "login" + "user" + module.
+ */
+import path from "node:path";
+import { normPath } from "./sqlite-specs-store.js";
+const SOURCE_EXTS = new Set([".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".java", ".cs", ".rb", ".rs", ".cpp", ".c", ".h", ".php", ".swift", ".kt"]);
+// Only filter true noise — dependency trees and test-only dirs.
+// examples/, docs/, fixtures/ may contain real source files.
+const NOISE_RE = /(?:^|\/)(?:test|tests|spec|specs|__pycache__|node_modules|vendor|\.git|\.tox|\.venv|venv)\//i;
+function isSourceFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return SOURCE_EXTS.has(ext) && !NOISE_RE.test(filePath);
+}
+/**
+ * Build FTS rows from a raw codebase-intelligence object.
+ * The intel object is the parsed JSON from codebase-intelligence.json —
+ * no schema changes, same structure as today.
+ */
+export function buildFTSRows(intel) {
+    // Per-file accumulators
+    const files = new Map();
+    function getRow(filePath, module = "") {
+        if (!isSourceFile(filePath))
+            return null;
+        const normalised = normPath(filePath);
+        if (!files.has(normalised)) {
+            files.set(normalised, { file_path: normalised, symbol_name: "", endpoint: "", body: "", module });
+        }
+        return files.get(normalised);
+    }
+    function appendToken(row, field, token) {
+        if (!token)
+            return;
+        row[field] += (row[field] ? " " : "") + token;
+    }
+    // ── API registry: endpoints → files ──────────────────────────────────────
+    for (const [route, entry] of Object.entries(intel.api_registry ?? {})) {
+        if (!entry.file)
+            continue;
+        const row = getRow(entry.file, entry.module ?? "");
+        if (!row)
+            continue;
+        appendToken(row, "endpoint", route);
+        appendToken(row, "symbol_name", entry.handler ?? "");
+        if (entry.request_schema)
+            appendToken(row, "body", entry.request_schema);
+        if (entry.response_schema)
+            appendToken(row, "body", entry.response_schema);
+        for (const sc of entry.service_calls ?? [])
+            appendToken(row, "body", sc);
+    }
+    // ── Model registry: ORM models → files ───────────────────────────────────
+    for (const [name, entry] of Object.entries(intel.model_registry ?? {})) {
+        if (!entry.file)
+            continue;
+        const row = getRow(entry.file, entry.module ?? "");
+        if (!row)
+            continue;
+        appendToken(row, "symbol_name", name);
+        for (const f of entry.fields ?? [])
+            appendToken(row, "body", f);
+        for (const r of entry.relationships ?? [])
+            appendToken(row, "body", r);
+    }
+    // ── Enum registry ─────────────────────────────────────────────────────────
+    for (const [name, entry] of Object.entries(intel.enum_registry ?? {})) {
+        if (!entry.file)
+            continue;
+        const row = getRow(entry.file, "");
+        if (!row)
+            continue;
+        appendToken(row, "symbol_name", name);
+        for (const v of entry.values ?? [])
+            appendToken(row, "body", v);
+    }
+    // ── Service map: module files ─────────────────────────────────────────────
+    for (const svc of intel.service_map ?? []) {
+        for (const filePath of svc.files ?? []) {
+            const row = getRow(filePath, svc.name ?? "");
+            if (!row)
+                continue;
+            appendToken(row, "module", svc.name ?? "");
+            for (const dep of svc.dependencies ?? [])
+                appendToken(row, "body", dep);
+        }
+    }
+    // ── Frontend pages ────────────────────────────────────────────────────────
+    for (const page of intel.frontend_pages ?? []) {
+        const filePath = page.file ?? page.component ?? page.path;
+        if (!filePath)
+            continue;
+        const row = getRow(filePath, "frontend");
+        if (!row)
+            continue;
+        appendToken(row, "endpoint", page.path ?? "");
+        appendToken(row, "symbol_name", page.component ?? "");
+        for (const api of page.api_calls ?? [])
+            appendToken(row, "body", api);
+        for (const c of page.components ?? [])
+            appendToken(row, "body", c);
+    }
+    // ── Background tasks ──────────────────────────────────────────────────────
+    for (const task of intel.background_tasks ?? []) {
+        if (!task.file)
+            continue;
+        const row = getRow(task.file, task.module ?? "");
+        if (!row)
+            continue;
+        appendToken(row, "symbol_name", task.name ?? "");
+        appendToken(row, "body", task.queue ?? "");
+    }
+    return Array.from(files.values());
+}
+/**
+ * Merge all data from architecture.snapshot into the FTS row map.
+ * Covers: endpoints, data_models, enums, tasks, module files + exports.
+ * This is the main enrichment for library repos with few intel entries.
+ */
+export function mergeArchitectureRows(rows, arch) {
+    function upsert(filePath, module = "") {
+        if (!filePath)
+            return null;
+        const ext = path.extname(filePath).toLowerCase();
+        if (!SOURCE_EXTS.has(ext) || NOISE_RE.test(filePath))
+            return null;
+        const norm = normPath(filePath);
+        if (!rows.has(norm))
+            rows.set(norm, { file_path: norm, symbol_name: "", endpoint: "", body: "", module });
+        return rows.get(norm);
+    }
+    function add(row, field, token) {
+        if (!token)
+            return;
+        row[field] += (row[field] ? " " : "") + token;
+    }
+    // ── arch.endpoints[] ─────────────────────────────────────────────────────
+    for (const ep of arch.endpoints ?? []) {
+        const row = upsert(ep.file, ep.module ?? "");
+        if (!row)
+            continue;
+        add(row, "endpoint", ep.path ?? "");
+        add(row, "endpoint", ep.method ?? "");
+        add(row, "symbol_name", ep.handler ?? "");
+        for (const sc of ep.service_calls ?? [])
+            add(row, "body", sc);
+    }
+    // ── arch.data_models[] ───────────────────────────────────────────────────
+    for (const m of arch.data_models ?? []) {
+        const row = upsert(m.file, m.module ?? "");
+        if (!row)
+            continue;
+        add(row, "symbol_name", m.name ?? "");
+        for (const f of m.fields ?? [])
+            add(row, "body", f);
+        for (const r of m.relationships ?? [])
+            add(row, "body", r);
+    }
+    // ── arch.enums[] ─────────────────────────────────────────────────────────
+    for (const e of arch.enums ?? []) {
+        const row = upsert(e.file, "");
+        if (!row)
+            continue;
+        add(row, "symbol_name", e.name ?? "");
+        for (const v of e.values ?? [])
+            add(row, "body", v);
+    }
+    // ── arch.tasks[] (background tasks / celery / etc.) ──────────────────────
+    for (const t of arch.tasks ?? []) {
+        const row = upsert(t.file, t.module ?? "");
+        if (!row)
+            continue;
+        add(row, "symbol_name", t.name ?? "");
+        add(row, "body", t.queue ?? "");
+    }
+    // ── arch.modules[].files + exports ────────────────────────────────────────
+    // mod.exports is [{file, symbols: string[], exports: [...]}], not a flat string array
+    for (const mod of arch.modules ?? []) {
+        for (const filePath of mod.files ?? []) {
+            const row = upsert(filePath, mod.id ?? mod.name ?? "");
+            if (!row)
+                continue;
+            if (mod.id && !row.module)
+                row.module = mod.id;
+        }
+        for (const expEntry of mod.exports ?? []) {
+            // expEntry may be a string (old format) or {file, symbols} (new format)
+            if (typeof expEntry === "string") {
+                const row = upsert(expEntry, mod.id ?? "");
+                if (row)
+                    add(row, "symbol_name", expEntry);
+            }
+            else if (expEntry && typeof expEntry === "object") {
+                const row = upsert(expEntry.file ?? "", mod.id ?? "");
+                if (!row)
+                    continue;
+                for (const sym of expEntry.symbols ?? []) {
+                    if (typeof sym === "string")
+                        add(row, "symbol_name", sym);
+                }
+            }
+        }
+    }
+    // ── arch.frontend_files[] ────────────────────────────────────────────────
+    for (const ff of arch.frontend_files ?? []) {
+        const filePath = ff.file ?? ff;
+        upsert(typeof filePath === "string" ? filePath : "", "frontend");
+    }
+}
+/**
+ * Merge function-intelligence.json entries into the FTS row map.
+ * Each function becomes a symbol_name token on its file's row.
+ */
+export function mergeFunctionIntelRows(rows, funcIntel) {
+    function upsert(filePath) {
+        if (!filePath)
+            return null;
+        const ext = path.extname(filePath).toLowerCase();
+        if (!SOURCE_EXTS.has(ext) || NOISE_RE.test(filePath))
+            return null;
+        const norm = normPath(filePath);
+        if (!rows.has(norm))
+            rows.set(norm, { file_path: norm, symbol_name: "", endpoint: "", body: "", module: "" });
+        return rows.get(norm);
+    }
+    for (const fn of funcIntel.functions ?? []) {
+        const row = upsert(fn.file ?? "");
+        if (!row)
+            continue;
+        if (fn.name)
+            row.symbol_name += (row.symbol_name ? " " : "") + fn.name;
+        if (fn.docstring)
+            row.body += (row.body ? " " : "") + fn.docstring;
+        for (const p of fn.params ?? [])
+            row.body += " " + p;
+        for (const c of fn.calls ?? [])
+            row.body += " " + c;
+    }
+}
+/**
+ * Build import edges from arch.dependencies.file_graph.
+ * Returns normalized {file, imports} pairs for all source-to-source edges.
+ */
+export function buildDepEdges(arch) {
+    const edges = [];
+    const graph = arch?.dependencies?.file_graph;
+    if (!graph)
+        return edges;
+    // file_graph may be a list of {from, to} edges (new format)
+    // or a dict of {file: {imports: []}} (old format)
+    if (Array.isArray(graph)) {
+        for (const edge of graph) {
+            const from = edge.from ?? edge.file;
+            const to = edge.to ?? edge.imports;
+            if (typeof from !== "string" || typeof to !== "string")
+                continue;
+            if (!isSourceFile(from) || !isSourceFile(to))
+                continue;
+            const normFrom = normPath(from);
+            const normTo = normPath(to);
+            edges.push({ file: normFrom, imports: normTo });
+        }
+    }
+    else {
+        for (const [file, info] of Object.entries(graph)) {
+            if (!isSourceFile(file))
+                continue;
+            const normFile = normPath(file);
+            const deps = info.imports ?? info.dependencies ?? [];
+            for (const imp of deps) {
+                if (typeof imp !== "string" || !isSourceFile(imp))
+                    continue;
+                edges.push({ file: normFile, imports: normPath(imp) });
+            }
+        }
+    }
+    return edges;
+}
+/**
+ * Populate the FTS5 search_fts table + file_deps graph from all extract output.
+ *   intel      — parsed codebase-intelligence.json
+ *   arch       — parsed architecture.snapshot.yaml (optional)
+ *   funcIntel  — parsed function-intelligence.json (optional)
+ */
+export function populateFTSIndex(store, intel, arch, funcIntel) {
+    const rowMap = new Map();
+    for (const row of buildFTSRows(intel))
+        rowMap.set(row.file_path, row);
+    if (arch)
+        mergeArchitectureRows(rowMap, arch);
+    if (funcIntel)
+        mergeFunctionIntelRows(rowMap, funcIntel);
+    store.rebuildSearchIndex(Array.from(rowMap.values()));
+    // Build dependency graph
+    if (arch) {
+        const edges = buildDepEdges(arch);
+        store.rebuildDeps(edges);
+    }
+}

package/dist/db/index.js

@@ -0,0 +1,55 @@
+/**
+ * Public API for the specs store module.
+ *
+ * Usage:
+ *   import { openSpecsStore } from "./db/index.js";
+ *   const store = await openSpecsStore(layout);
+ *   const entry = await store.readSpec("codebase-intelligence");
+ *   await store.close();
+ *
+ * The adapter is chosen automatically:
+ *   - If guardian.db exists in the specs root → SqliteSpecsStore
+ *   - Otherwise → FileSpecsStore (current behavior, fully backward-compatible)
+ *
+ * To force SQLite (e.g. during `guardian extract --backend sqlite`):
+ *   const store = await openSpecsStore(layout, { backend: "sqlite" });
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { FileSpecsStore } from "./file-specs-store.js";
+import { SqliteSpecsStore, DB_FILENAME } from "./sqlite-specs-store.js";
+export { FileSpecsStore } from "./file-specs-store.js";
+export { SqliteSpecsStore, DB_FILENAME } from "./sqlite-specs-store.js";
+/**
+ * Open the appropriate SpecsStore for the given output layout.
+ * Always calls store.init() before returning.
+ *
+ * backend "auto" (default): use SQLite if guardian.db exists, else file
+ * backend "sqlite":         require SQLite (caller must handle missing db)
+ * backend "file":           always use file store
+ */
+export async function openSpecsStore(layout, options = {}) {
+    // "auto" and undefined both probe for guardian.db
+    const requested = options.backend ?? "auto";
+    const resolved = requested === "auto" || requested === undefined
+        ? await detectBackend(layout.rootDir)
+        : requested;
+    let store;
+    if (resolved === "sqlite") {
+        store = new SqliteSpecsStore(layout.rootDir);
+    }
+    else {
+        store = new FileSpecsStore(layout.machineDir, layout.humanDir);
+    }
+    await store.init();
+    return store;
+}
+async function detectBackend(rootDir) {
+    try {
+        await fs.stat(path.join(rootDir, DB_FILENAME));
+        return "sqlite";
+    }
+    catch {
+        return "file";
+    }
+}

package/dist/db/specs-store.js

@@ -0,0 +1,13 @@
+/**
+ * SpecsStore — IO adapter interface for guardian spec files.
+ *
+ * All reads and writes of .specs content flow through this interface.
+ * Two implementations exist:
+ *   - FileSpecsStore  (default today) — reads/writes individual files on disk
+ *   - SqliteSpecsStore               — reads/writes a single guardian.db
+ *
+ * Callers never reference the filesystem or SQLite directly; they use
+ * this interface. Switching storage backends requires zero changes outside
+ * this module.
+ */
+export {};