@mxalbert/context-mode 2.0.0

package/build/store-directory.js

@@ -0,0 +1,254 @@
+/**
+ * walkDirectory — bounded recursive directory walker for ctx_index (#687).
+ *
+ * Issue: ctx_index refused directory paths via the security gate at
+ * src/store.ts:845 ("refusing to index <path>: not a regular file"). The gate
+ * is a TOCTOU defense from #442 round-3 and MUST be preserved — directory
+ * support is layered as a separate concern here. Each file produced by
+ * walkDirectory is then read via the existing per-file
+ * `openSync + fstatSync.isFile()` invariant in `ContentStore.index()`.
+ *
+ * Reported by @matiasduartee across 4 clients × Windows 11.
+ * https://github.com/anthropic-experimental/context-mode/issues/687
+ *
+ * Design constraints:
+ *   - No new dependencies (avoid the `ignore` package — issue #687 Diagnose).
+ *   - Cross-OS: path.sep / path.join everywhere, never raw "/" string ops.
+ *   - Symlink cycle detection via a resolved-path Set.
+ *   - Symlink-escape rejection: refuse to follow symlinks that resolve outside
+ *     the rootPath (defense-in-depth alongside per-file checkFilePathDenyPolicy).
+ *   - FTS5-blowup guard: hard cap maxFiles (default 200, per Architect).
+ */
+import { readdirSync, statSync, lstatSync, realpathSync, existsSync, readFileSync, } from "node:fs";
+import { join, extname, relative, sep, resolve } from "node:path";
+const DEFAULT_EXCLUDES = [
+    "node_modules",
+    ".git",
+    "dist",
+    "build",
+    ".next",
+    "coverage",
+    ".venv",
+    "__pycache__",
+    ".DS_Store",
+];
+const DEFAULT_EXTENSIONS = [
+    ".md",
+    ".mdx",
+    ".txt",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".py",
+    ".rs",
+    ".go",
+    ".sh",
+];
+const DEFAULT_MAX_DEPTH = 5;
+const DEFAULT_MAX_FILES = 200;
+/**
+ * Convert a simple glob pattern (`*`, `**`, `?`) to a RegExp. Anchors at
+ * boundaries so `node_modules` matches `node_modules` AND `node_modules/pkg`.
+ * Patterns are matched against POSIX-style relative paths (forward slashes)
+ * to give consistent behavior across macOS / Windows.
+ */
+function globToRegExp(pattern) {
+    // Escape regex metachars except glob ones.
+    let re = "";
+    for (let i = 0; i < pattern.length; i++) {
+        const c = pattern[i];
+        if (c === "*") {
+            if (pattern[i + 1] === "*") {
+                re += ".*";
+                i++;
+            }
+            else {
+                re += "[^/]*";
+            }
+        }
+        else if (c === "?") {
+            re += "[^/]";
+        }
+        else if ("\\^$.|+()[]{}".includes(c)) {
+            re += "\\" + c;
+        }
+        else {
+            re += c;
+        }
+    }
+    return new RegExp(`^${re}$`);
+}
+/** Match a posix-style relative path against any of the patterns. */
+function matchesAny(relPosix, patterns) {
+    if (patterns.length === 0)
+        return false;
+    const basename = relPosix.split("/").pop() ?? relPosix;
+    for (const p of patterns) {
+        // Bare names match basename OR any path segment.
+        if (!p.includes("/") && !p.includes("*")) {
+            if (basename === p)
+                return true;
+            if (relPosix.split("/").includes(p))
+                return true;
+            continue;
+        }
+        const re = globToRegExp(p);
+        if (re.test(relPosix))
+            return true;
+        if (re.test(basename))
+            return true;
+    }
+    return false;
+}
+/**
+ * Parse a .gitignore file into a list of patterns. Comments and blank lines
+ * are stripped. Negation (`!`) is not supported — kept conservative.
+ */
+function parseGitignore(rootPath) {
+    const giPath = join(rootPath, ".gitignore");
+    if (!existsSync(giPath))
+        return [];
+    try {
+        const text = readFileSync(giPath, "utf-8");
+        return text
+            .split(/\r?\n/)
+            .map(l => l.trim())
+            .filter(l => l.length > 0 && !l.startsWith("#") && !l.startsWith("!"))
+            .map(l => l.replace(/^\//, "").replace(/\/$/, ""));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Convert an absolute path under rootPath to a POSIX-style relative path
+ * so glob matching is identical across macOS/Linux/Windows.
+ */
+function toPosixRel(rootPath, absPath) {
+    const rel = relative(rootPath, absPath);
+    return rel.split(sep).join("/");
+}
+/**
+ * Walk `rootPath` recursively under the given bounds and return absolute file
+ * paths matching the filters. Pure synchronous traversal — no allocations
+ * beyond the result array. Symlink cycles are detected via a resolved-path
+ * Set; symlink escapes (resolving outside rootPath) are silently skipped.
+ */
+export function walkDirectory(rootPath, opts = {}) {
+    return walkDirectoryDetailed(rootPath, opts).files;
+}
+/**
+ * Same as walkDirectory but returns capped + totalSeen so callers can surface
+ * a "capped at N files" notice in their response.
+ */
+export function walkDirectoryDetailed(rootPath, opts = {}) {
+    const { include, exclude, maxDepth = DEFAULT_MAX_DEPTH, maxFiles = DEFAULT_MAX_FILES, extensions, respectGitignore = true, followSymlinks = false, } = opts;
+    // Normalize rootPath to its real path so symlink-escape detection is sound.
+    let rootReal;
+    try {
+        rootReal = realpathSync(rootPath);
+    }
+    catch {
+        return { files: [], capped: false, totalSeen: 0 };
+    }
+    const exts = (extensions && extensions.length > 0 ? extensions : DEFAULT_EXTENSIONS)
+        .map(e => (e.startsWith(".") ? e : "." + e).toLowerCase());
+    const excludes = [
+        ...DEFAULT_EXCLUDES,
+        ...(exclude ?? []),
+        ...(respectGitignore ? parseGitignore(rootReal) : []),
+    ];
+    const includes = include ?? [];
+    const out = [];
+    const visited = new Set([rootReal]);
+    let totalSeen = 0;
+    let capped = false;
+    function walk(absDir, depth) {
+        if (capped)
+            return;
+        if (depth > maxDepth)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return; // unreadable directory — skip silently
+        }
+        for (const ent of entries) {
+            if (capped)
+                return;
+            const absChild = join(absDir, ent.name);
+            const relPosix = toPosixRel(rootReal, absChild);
+            // Exclude check applies to both files and dirs — early prune.
+            if (matchesAny(relPosix, excludes))
+                continue;
+            // Include filter applies to files only — see below.
+            // Resolve symlinks once; reject escapes; track for cycle detection.
+            let isDirChild = ent.isDirectory();
+            let isFileChild = ent.isFile();
+            let isSymlink = false;
+            try {
+                const lst = lstatSync(absChild);
+                isSymlink = lst.isSymbolicLink();
+            }
+            catch {
+                continue;
+            }
+            if (isSymlink) {
+                if (!followSymlinks)
+                    continue;
+                let resolved;
+                try {
+                    resolved = realpathSync(absChild);
+                }
+                catch {
+                    continue; // dangling
+                }
+                // Symlink-escape: refuse to follow if the resolved target leaves rootReal.
+                const escapeRel = relative(rootReal, resolved);
+                if (escapeRel.startsWith("..") || resolve(escapeRel) === resolved) {
+                    // resolve(absolute) === absolute → target is absolute outside root
+                    if (escapeRel.startsWith(".."))
+                        continue;
+                }
+                if (visited.has(resolved))
+                    continue;
+                visited.add(resolved);
+                try {
+                    const st = statSync(resolved);
+                    isDirChild = st.isDirectory();
+                    isFileChild = st.isFile();
+                }
+                catch {
+                    continue;
+                }
+            }
+            if (isDirChild) {
+                walk(absChild, depth + 1);
+                continue;
+            }
+            if (!isFileChild)
+                continue;
+            // Extension filter.
+            const ext = extname(absChild).toLowerCase();
+            if (!exts.includes(ext))
+                continue;
+            // Include filter (if any): file must match at least one include pattern.
+            if (includes.length > 0 && !matchesAny(relPosix, includes))
+                continue;
+            totalSeen++;
+            if (out.length >= maxFiles) {
+                capped = true;
+                return;
+            }
+            out.push(absChild);
+        }
+    }
+    walk(rootReal, 0);
+    return { files: out, capped, totalSeen };
+}

package/build/store.d.ts

@@ -0,0 +1,147 @@
+/**
+ * ContentStore — FTS5 BM25-based knowledge base for context-mode.
+ *
+ * Chunks markdown content by headings (keeping code blocks intact),
+ * stores in SQLite FTS5, and retrieves via BM25-ranked search.
+ *
+ * Use for documentation, API references, and any content where
+ * you need EXACT text later — not summaries.
+ */
+import { type WalkOptions } from "./store-directory.js";
+type SourceMatchMode = "like" | "exact";
+import type { IndexResult, SearchResult, StoreStats } from "./types.js";
+export type { IndexResult, SearchResult, StoreStats } from "./types.js";
+export declare function sanitizeQuery(query: string, mode?: "AND" | "OR"): string;
+export declare function sanitizeTrigramQuery(query: string, mode?: "AND" | "OR"): string;
+/**
+ * Remove stale DB files from previous sessions whose processes no longer exist.
+ */
+export declare function cleanupStaleDBs(): number;
+/**
+ * Clean up stale per-project content store DBs older than maxAgeDays.
+ * Scans the given directory for *.db files and checks mtime.
+ * Also detects zombie processes holding WAL locks — if a WAL file exists
+ * but the owning PID is dead, the DB files are cleaned up regardless of age.
+ */
+export declare function cleanupStaleContentDBs(contentDir: string, maxAgeDays: number): number;
+export declare class ContentStore {
+    #private;
+    static readonly OPTIMIZE_EVERY = 50;
+    static readonly FUZZY_CACHE_SIZE = 256;
+    constructor(dbPath?: string);
+    /** Delete this session's DB files. Call on process exit. */
+    cleanup(): void;
+    /**
+     * Register a deny-policy checker. When set, #refreshStaleSources
+     * calls it before re-reading any file_path during auto-refresh.
+     * Returning `true` causes the source to be skipped (kept in cache,
+     * not re-indexed). server.ts wires this to the Read deny patterns.
+     */
+    setDenyChecker(fn: ((filePath: string) => boolean) | undefined): void;
+    index(options: {
+        content?: string;
+        path?: string;
+        source?: string;
+        /**
+         * Optional FK metadata recorded on each indexed chunk so per-session
+         * honest-savings stats can join chunks → session_events. When omitted,
+         * chunks fall back to empty-string columns (legacy behaviour).
+         */
+        attribution?: {
+            sessionId?: string;
+            eventId?: string;
+        };
+    }): IndexResult;
+    /**
+     * Index every file under a directory by walking it with `walkDirectory` and
+     * delegating each discovered file to `this.index({ path })`. The per-file
+     * `openSync + fstatSync.isFile()` security gate at line ~845 stays active
+     * for every file — directory support never bypasses the TOCTOU defense
+     * from #442 round-3.
+     *
+     * Reported by @matiasduartee in #687.
+     */
+    indexDirectory(opts: {
+        path: string;
+        source?: string;
+        attribution?: {
+            sessionId?: string;
+            eventId?: string;
+        };
+        /** Optional per-file deny check — runs INSIDE the walk loop so a denied
+         *  file does not even open a fd. Returns true to deny. */
+        perFileDeny?: (absPath: string) => boolean;
+    } & WalkOptions): {
+        filesIndexed: number;
+        totalChunks: number;
+        capped: boolean;
+        totalSeen: number;
+        denied: number;
+        failed: number;
+        label: string;
+    };
+    /**
+     * Index plain-text output (logs, build output, test results) by splitting
+     * into fixed-size line groups. Unlike markdown indexing, this does not
+     * look for headings — it chunks by line count with overlap.
+     */
+    indexPlainText(content: string, source: string, linesPerChunk?: number, attribution?: {
+        sessionId?: string;
+        eventId?: string;
+    }, maxChunkBytes?: number): IndexResult;
+    /**
+     * Index JSON content by walking the object tree and using key paths
+     * as chunk titles (analogous to heading hierarchy in markdown). Objects
+     * recurse by key; arrays batch items by size.
+     *
+     * Falls back to `indexPlainText` if the content is not valid JSON.
+     */
+    indexJSON(content: string, source: string, maxChunkBytes?: number, attribution?: {
+        sessionId?: string;
+        eventId?: string;
+    }): IndexResult;
+    search(query: string, limit?: number, source?: string, mode?: "AND" | "OR", contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode): SearchResult[];
+    searchTrigram(query: string, limit?: number, source?: string, mode?: "AND" | "OR", contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode): SearchResult[];
+    fuzzyCorrect(query: string): string | null;
+    searchWithFallback(query: string, limit?: number, source?: string, contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode, sessionIdAllowSet?: Set<string>): SearchResult[];
+    /** Number of sources auto-refreshed in the last searchWithFallback call. */
+    lastRefreshCount: number;
+    getSourceMeta(label: string): {
+        label: string;
+        chunkCount: number;
+        codeChunkCount: number;
+        indexedAt: string;
+        filePath: string | null;
+        contentHash: string | null;
+    } | null;
+    listSources(): Array<{
+        label: string;
+        chunkCount: number;
+    }>;
+    /**
+     * Aggregate snapshot of the persistent content store. Returns total
+     * chunk count, source count, and the most recent indexed_at timestamp.
+     * Used by ctx_stats so callers can see observability state in the same
+     * round trip instead of inferring it from snapshot diffs.
+     */
+    getIndexState(): {
+        totalChunks: number;
+        totalSources: number;
+        lastIndexedAt?: string;
+    };
+    /**
+     * Get all chunks for a given source by ID — bypasses FTS5 MATCH entirely.
+     * Use this for inventory/listing where you need all sections, not search.
+     */
+    getChunksBySource(sourceId: number): SearchResult[];
+    getDistinctiveTerms(sourceId: number, maxTerms?: number): string[];
+    getStats(): StoreStats;
+    /**
+     * Delete sources (and their chunks) older than maxAgeDays.
+     * Returns count of deleted sources.
+     */
+    cleanupStaleSources(maxAgeDays: number): number;
+    /** Get DB file size in bytes. */
+    getDBSizeBytes(): number;
+    close(): void;
+}