context-mode 1.0.168 → 2.0.0-next.5

package/build/store-directory.js

@@ -1,254 +0,0 @@
-/**
- * walkDirectory — bounded recursive directory walker for ctx_index (#687).
- *
- * Issue: ctx_index refused directory paths via the security gate at
- * src/store.ts:845 ("refusing to index <path>: not a regular file"). The gate
- * is a TOCTOU defense from #442 round-3 and MUST be preserved — directory
- * support is layered as a separate concern here. Each file produced by
- * walkDirectory is then read via the existing per-file
- * `openSync + fstatSync.isFile()` invariant in `ContentStore.index()`.
- *
- * Reported by @matiasduartee across 4 clients × Windows 11.
- * https://github.com/anthropic-experimental/context-mode/issues/687
- *
- * Design constraints:
- *   - No new dependencies (avoid the `ignore` package — issue #687 Diagnose).
- *   - Cross-OS: path.sep / path.join everywhere, never raw "/" string ops.
- *   - Symlink cycle detection via a resolved-path Set.
- *   - Symlink-escape rejection: refuse to follow symlinks that resolve outside
- *     the rootPath (defense-in-depth alongside per-file checkFilePathDenyPolicy).
- *   - FTS5-blowup guard: hard cap maxFiles (default 200, per Architect).
- */
-import { readdirSync, statSync, lstatSync, realpathSync, existsSync, readFileSync, } from "node:fs";
-import { join, extname, relative, sep, resolve } from "node:path";
-const DEFAULT_EXCLUDES = [
-    "node_modules",
-    ".git",
-    "dist",
-    "build",
-    ".next",
-    "coverage",
-    ".venv",
-    "__pycache__",
-    ".DS_Store",
-];
-const DEFAULT_EXTENSIONS = [
-    ".md",
-    ".mdx",
-    ".txt",
-    ".json",
-    ".yaml",
-    ".yml",
-    ".ts",
-    ".tsx",
-    ".js",
-    ".jsx",
-    ".py",
-    ".rs",
-    ".go",
-    ".sh",
-];
-const DEFAULT_MAX_DEPTH = 5;
-const DEFAULT_MAX_FILES = 200;
-/**
- * Convert a simple glob pattern (`*`, `**`, `?`) to a RegExp. Anchors at
- * boundaries so `node_modules` matches `node_modules` AND `node_modules/pkg`.
- * Patterns are matched against POSIX-style relative paths (forward slashes)
- * to give consistent behavior across macOS / Windows.
- */
-function globToRegExp(pattern) {
-    // Escape regex metachars except glob ones.
-    let re = "";
-    for (let i = 0; i < pattern.length; i++) {
-        const c = pattern[i];
-        if (c === "*") {
-            if (pattern[i + 1] === "*") {
-                re += ".*";
-                i++;
-            }
-            else {
-                re += "[^/]*";
-            }
-        }
-        else if (c === "?") {
-            re += "[^/]";
-        }
-        else if ("\\^$.|+()[]{}".includes(c)) {
-            re += "\\" + c;
-        }
-        else {
-            re += c;
-        }
-    }
-    return new RegExp(`^${re}$`);
-}
-/** Match a posix-style relative path against any of the patterns. */
-function matchesAny(relPosix, patterns) {
-    if (patterns.length === 0)
-        return false;
-    const basename = relPosix.split("/").pop() ?? relPosix;
-    for (const p of patterns) {
-        // Bare names match basename OR any path segment.
-        if (!p.includes("/") && !p.includes("*")) {
-            if (basename === p)
-                return true;
-            if (relPosix.split("/").includes(p))
-                return true;
-            continue;
-        }
-        const re = globToRegExp(p);
-        if (re.test(relPosix))
-            return true;
-        if (re.test(basename))
-            return true;
-    }
-    return false;
-}
-/**
- * Parse a .gitignore file into a list of patterns. Comments and blank lines
- * are stripped. Negation (`!`) is not supported — kept conservative.
- */
-function parseGitignore(rootPath) {
-    const giPath = join(rootPath, ".gitignore");
-    if (!existsSync(giPath))
-        return [];
-    try {
-        const text = readFileSync(giPath, "utf-8");
-        return text
-            .split(/\r?\n/)
-            .map(l => l.trim())
-            .filter(l => l.length > 0 && !l.startsWith("#") && !l.startsWith("!"))
-            .map(l => l.replace(/^\//, "").replace(/\/$/, ""));
-    }
-    catch {
-        return [];
-    }
-}
-/**
- * Convert an absolute path under rootPath to a POSIX-style relative path
- * so glob matching is identical across macOS/Linux/Windows.
- */
-function toPosixRel(rootPath, absPath) {
-    const rel = relative(rootPath, absPath);
-    return rel.split(sep).join("/");
-}
-/**
- * Walk `rootPath` recursively under the given bounds and return absolute file
- * paths matching the filters. Pure synchronous traversal — no allocations
- * beyond the result array. Symlink cycles are detected via a resolved-path
- * Set; symlink escapes (resolving outside rootPath) are silently skipped.
- */
-export function walkDirectory(rootPath, opts = {}) {
-    return walkDirectoryDetailed(rootPath, opts).files;
-}
-/**
- * Same as walkDirectory but returns capped + totalSeen so callers can surface
- * a "capped at N files" notice in their response.
- */
-export function walkDirectoryDetailed(rootPath, opts = {}) {
-    const { include, exclude, maxDepth = DEFAULT_MAX_DEPTH, maxFiles = DEFAULT_MAX_FILES, extensions, respectGitignore = true, followSymlinks = false, } = opts;
-    // Normalize rootPath to its real path so symlink-escape detection is sound.
-    let rootReal;
-    try {
-        rootReal = realpathSync(rootPath);
-    }
-    catch {
-        return { files: [], capped: false, totalSeen: 0 };
-    }
-    const exts = (extensions && extensions.length > 0 ? extensions : DEFAULT_EXTENSIONS)
-        .map(e => (e.startsWith(".") ? e : "." + e).toLowerCase());
-    const excludes = [
-        ...DEFAULT_EXCLUDES,
-        ...(exclude ?? []),
-        ...(respectGitignore ? parseGitignore(rootReal) : []),
-    ];
-    const includes = include ?? [];
-    const out = [];
-    const visited = new Set([rootReal]);
-    let totalSeen = 0;
-    let capped = false;
-    function walk(absDir, depth) {
-        if (capped)
-            return;
-        if (depth > maxDepth)
-            return;
-        let entries;
-        try {
-            entries = readdirSync(absDir, { withFileTypes: true });
-        }
-        catch {
-            return; // unreadable directory — skip silently
-        }
-        for (const ent of entries) {
-            if (capped)
-                return;
-            const absChild = join(absDir, ent.name);
-            const relPosix = toPosixRel(rootReal, absChild);
-            // Exclude check applies to both files and dirs — early prune.
-            if (matchesAny(relPosix, excludes))
-                continue;
-            // Include filter applies to files only — see below.
-            // Resolve symlinks once; reject escapes; track for cycle detection.
-            let isDirChild = ent.isDirectory();
-            let isFileChild = ent.isFile();
-            let isSymlink = false;
-            try {
-                const lst = lstatSync(absChild);
-                isSymlink = lst.isSymbolicLink();
-            }
-            catch {
-                continue;
-            }
-            if (isSymlink) {
-                if (!followSymlinks)
-                    continue;
-                let resolved;
-                try {
-                    resolved = realpathSync(absChild);
-                }
-                catch {
-                    continue; // dangling
-                }
-                // Symlink-escape: refuse to follow if the resolved target leaves rootReal.
-                const escapeRel = relative(rootReal, resolved);
-                if (escapeRel.startsWith("..") || resolve(escapeRel) === resolved) {
-                    // resolve(absolute) === absolute → target is absolute outside root
-                    if (escapeRel.startsWith(".."))
-                        continue;
-                }
-                if (visited.has(resolved))
-                    continue;
-                visited.add(resolved);
-                try {
-                    const st = statSync(resolved);
-                    isDirChild = st.isDirectory();
-                    isFileChild = st.isFile();
-                }
-                catch {
-                    continue;
-                }
-            }
-            if (isDirChild) {
-                walk(absChild, depth + 1);
-                continue;
-            }
-            if (!isFileChild)
-                continue;
-            // Extension filter.
-            const ext = extname(absChild).toLowerCase();
-            if (!exts.includes(ext))
-                continue;
-            // Include filter (if any): file must match at least one include pattern.
-            if (includes.length > 0 && !matchesAny(relPosix, includes))
-                continue;
-            totalSeen++;
-            if (out.length >= maxFiles) {
-                capped = true;
-                return;
-            }
-            out.push(absChild);
-        }
-    }
-    walk(rootReal, 0);
-    return { files: out, capped, totalSeen };
-}

package/build/store.d.ts

@@ -1,147 +0,0 @@
-/**
- * ContentStore — FTS5 BM25-based knowledge base for context-mode.
- *
- * Chunks markdown content by headings (keeping code blocks intact),
- * stores in SQLite FTS5, and retrieves via BM25-ranked search.
- *
- * Use for documentation, API references, and any content where
- * you need EXACT text later — not summaries.
- */
-import { type WalkOptions } from "./store-directory.js";
-type SourceMatchMode = "like" | "exact";
-import type { IndexResult, SearchResult, StoreStats } from "./types.js";
-export type { IndexResult, SearchResult, StoreStats } from "./types.js";
-export declare function sanitizeQuery(query: string, mode?: "AND" | "OR"): string;
-export declare function sanitizeTrigramQuery(query: string, mode?: "AND" | "OR"): string;
-/**
- * Remove stale DB files from previous sessions whose processes no longer exist.
- */
-export declare function cleanupStaleDBs(): number;
-/**
- * Clean up stale per-project content store DBs older than maxAgeDays.
- * Scans the given directory for *.db files and checks mtime.
- * Also detects zombie processes holding WAL locks — if a WAL file exists
- * but the owning PID is dead, the DB files are cleaned up regardless of age.
- */
-export declare function cleanupStaleContentDBs(contentDir: string, maxAgeDays: number): number;
-export declare class ContentStore {
-    #private;
-    static readonly OPTIMIZE_EVERY = 50;
-    static readonly FUZZY_CACHE_SIZE = 256;
-    constructor(dbPath?: string);
-    /** Delete this session's DB files. Call on process exit. */
-    cleanup(): void;
-    /**
-     * Register a deny-policy checker. When set, #refreshStaleSources
-     * calls it before re-reading any file_path during auto-refresh.
-     * Returning `true` causes the source to be skipped (kept in cache,
-     * not re-indexed). server.ts wires this to the Read deny patterns.
-     */
-    setDenyChecker(fn: ((filePath: string) => boolean) | undefined): void;
-    index(options: {
-        content?: string;
-        path?: string;
-        source?: string;
-        /**
-         * Optional FK metadata recorded on each indexed chunk so per-session
-         * honest-savings stats can join chunks → session_events. When omitted,
-         * chunks fall back to empty-string columns (legacy behaviour).
-         */
-        attribution?: {
-            sessionId?: string;
-            eventId?: string;
-        };
-    }): IndexResult;
-    /**
-     * Index every file under a directory by walking it with `walkDirectory` and
-     * delegating each discovered file to `this.index({ path })`. The per-file
-     * `openSync + fstatSync.isFile()` security gate at line ~845 stays active
-     * for every file — directory support never bypasses the TOCTOU defense
-     * from #442 round-3.
-     *
-     * Reported by @matiasduartee in #687.
-     */
-    indexDirectory(opts: {
-        path: string;
-        source?: string;
-        attribution?: {
-            sessionId?: string;
-            eventId?: string;
-        };
-        /** Optional per-file deny check — runs INSIDE the walk loop so a denied
-         *  file does not even open a fd. Returns true to deny. */
-        perFileDeny?: (absPath: string) => boolean;
-    } & WalkOptions): {
-        filesIndexed: number;
-        totalChunks: number;
-        capped: boolean;
-        totalSeen: number;
-        denied: number;
-        failed: number;
-        label: string;
-    };
-    /**
-     * Index plain-text output (logs, build output, test results) by splitting
-     * into fixed-size line groups. Unlike markdown indexing, this does not
-     * look for headings — it chunks by line count with overlap.
-     */
-    indexPlainText(content: string, source: string, linesPerChunk?: number, attribution?: {
-        sessionId?: string;
-        eventId?: string;
-    }, maxChunkBytes?: number): IndexResult;
-    /**
-     * Index JSON content by walking the object tree and using key paths
-     * as chunk titles (analogous to heading hierarchy in markdown). Objects
-     * recurse by key; arrays batch items by size.
-     *
-     * Falls back to `indexPlainText` if the content is not valid JSON.
-     */
-    indexJSON(content: string, source: string, maxChunkBytes?: number, attribution?: {
-        sessionId?: string;
-        eventId?: string;
-    }): IndexResult;
-    search(query: string, limit?: number, source?: string, mode?: "AND" | "OR", contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode): SearchResult[];
-    searchTrigram(query: string, limit?: number, source?: string, mode?: "AND" | "OR", contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode): SearchResult[];
-    fuzzyCorrect(query: string): string | null;
-    searchWithFallback(query: string, limit?: number, source?: string, contentType?: "code" | "prose", sourceMatchMode?: SourceMatchMode, sessionIdAllowSet?: Set<string>): SearchResult[];
-    /** Number of sources auto-refreshed in the last searchWithFallback call. */
-    lastRefreshCount: number;
-    getSourceMeta(label: string): {
-        label: string;
-        chunkCount: number;
-        codeChunkCount: number;
-        indexedAt: string;
-        filePath: string | null;
-        contentHash: string | null;
-    } | null;
-    listSources(): Array<{
-        label: string;
-        chunkCount: number;
-    }>;
-    /**
-     * Aggregate snapshot of the persistent content store. Returns total
-     * chunk count, source count, and the most recent indexed_at timestamp.
-     * Used by ctx_stats so callers can see observability state in the same
-     * round trip instead of inferring it from snapshot diffs.
-     */
-    getIndexState(): {
-        totalChunks: number;
-        totalSources: number;
-        lastIndexedAt?: string;
-    };
-    /**
-     * Get all chunks for a given source by ID — bypasses FTS5 MATCH entirely.
-     * Use this for inventory/listing where you need all sections, not search.
-     */
-    getChunksBySource(sourceId: number): SearchResult[];
-    getDistinctiveTerms(sourceId: number, maxTerms?: number): string[];
-    getStats(): StoreStats;
-    /**
-     * Delete sources (and their chunks) older than maxAgeDays.
-     * Returns count of deleted sources.
-     */
-    cleanupStaleSources(maxAgeDays: number): number;
-    /** Get DB file size in bytes. */
-    getDBSizeBytes(): number;
-    close(): void;
-}