@cruxy/cli 0.1.0 → 0.2.0

package/dist/indexing/walker.js

@@ -0,0 +1,166 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { globToRegexBody, isBinary } from "./util.js";
+/** Directories never descended into, regardless of ignore files. */
+const ALWAYS_IGNORE_DIRS = new Set([".git", "node_modules", ".cruxy"]);
+const DEFAULT_IGNORE_FILES = [".gitignore", ".cruxyignore"];
+/**
+ * Hard denylist for secret-bearing files. Applied independently of ignore files
+ * (and of negation rules), so secrets are never indexed even when untracked.
+ * Matched against the project-relative POSIX path.
+ */
+const SECRET_PATTERNS = [
+    /(^|\/)\.env($|\.|rc)/i, // .env, .env.local, .env.production, .envrc
+    /(^|\/)id_(rsa|dsa|ecdsa|ed25519)(\.|$)/, // ssh private keys
+    /\.(pem|key|pfx|p12|p8|keystore|jks|asc|gpg)$/i, // keys / certs / keystores
+    /(^|\/)\.(npmrc|netrc|pgpass)$/i, // credential dotfiles
+    /(^|\/)\.aws\/credentials$/i,
+];
+function isSecretPath(relPath) {
+    return SECRET_PATTERNS.some((re) => re.test(relPath));
+}
+/**
+ * A compiled set of gitignore-style patterns, matched against paths relative to
+ * the file's own directory. Supports comments, blank lines, `!` negation,
+ * leading-`/` anchoring, trailing-`/` directory-only rules, and `*`/`**`/`?`.
+ */
+export class IgnoreMatcher {
+    rules = [];
+    constructor(patterns) {
+        for (const line of patterns)
+            this.add(line);
+    }
+    add(raw) {
+        // Strip a trailing CR and unescaped trailing whitespace.
+        let line = raw.replace(/\r$/, "").replace(/(?<!\\)\s+$/, "");
+        if (line === "" || line.startsWith("#"))
+            return;
+        let negate = false;
+        if (line.startsWith("!")) {
+            negate = true;
+            line = line.slice(1);
+        }
+        // Unescape a leading "\#" / "\!".
+        line = line.replace(/^\\([#!])/, "$1");
+        let dirOnly = false;
+        if (line.endsWith("/")) {
+            dirOnly = true;
+            line = line.slice(0, -1);
+        }
+        const anchored = line.startsWith("/");
+        if (anchored)
+            line = line.slice(1);
+        // A pattern with an internal separator is anchored to this directory;
+        // otherwise it may match at any depth.
+        const hasInternalSlash = line.includes("/");
+        const prefix = anchored || hasInternalSlash ? "" : "(?:.*/)?";
+        // Trailing "(?:/.*)?" lets a directory match also cover its contents.
+        const re = new RegExp(`^${prefix}${globToRegexBody(line)}(?:/.*)?$`);
+        this.rules.push({ re, negate, dirOnly });
+    }
+    /**
+     * Decide whether `relPath` is ignored: `true`/`false` when a rule matches
+     * (last match wins), or `undefined` when no rule applies.
+     */
+    decide(relPath, isDir) {
+        let decision;
+        for (const rule of this.rules) {
+            if (rule.dirOnly && !isDir)
+                continue;
+            if (rule.re.test(relPath))
+                decision = !rule.negate;
+        }
+        return decision;
+    }
+}
+/**
+ * Resolve the ignore decision for a path against the full stack (shallow→deep);
+ * the deepest scope that makes a decision wins, mirroring git's precedence.
+ */
+function isIgnored(stack, absPath, isDir) {
+    let ignored = false;
+    for (const { baseDir, matcher } of stack) {
+        const rel = path.relative(baseDir, absPath).split(path.sep).join("/");
+        const decision = matcher.decide(rel, isDir);
+        if (decision !== undefined)
+            ignored = decision;
+    }
+    return ignored;
+}
+async function loadIgnoreScope(dir, ignoreFileNames) {
+    const patterns = [];
+    for (const name of ignoreFileNames) {
+        try {
+            const text = await fs.readFile(path.join(dir, name), "utf8");
+            patterns.push(...text.split("\n"));
+        }
+        catch {
+            /* no such ignore file in this directory */
+        }
+    }
+    return patterns.length > 0
+        ? { baseDir: dir, matcher: new IgnoreMatcher(patterns) }
+        : null;
+}
+async function isBinaryFile(absPath) {
+    const fh = await fs.open(absPath, "r");
+    try {
+        const buf = Buffer.alloc(4096);
+        const { bytesRead } = await fh.read(buf, 0, buf.length, 0);
+        return isBinary(buf.subarray(0, bytesRead));
+    }
+    finally {
+        await fh.close();
+    }
+}
+/**
+ * Walk `root` and yield every indexable text file. Directories in the
+ * always-ignore set are skipped wholesale; ignore files accumulate down the
+ * tree; secret-bearing, oversized, and binary files are filtered out.
+ */
+export async function* walkRepo(root, opts) {
+    const absRoot = path.resolve(root);
+    const ignoreFileNames = opts.ignoreFileNames ?? DEFAULT_IGNORE_FILES;
+    yield* walkDir(absRoot, absRoot, [], ignoreFileNames, opts.maxFileBytes);
+}
+async function* walkDir(dir, root, parentStack, ignoreFileNames, maxFileBytes) {
+    const scope = await loadIgnoreScope(dir, ignoreFileNames);
+    const stack = scope ? [...parentStack, scope] : parentStack;
+    let entries;
+    try {
+        entries = await fs.readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return; // unreadable directory — skip
+    }
+    entries.sort((a, b) => a.name.localeCompare(b.name));
+    for (const entry of entries) {
+        const absPath = path.join(dir, entry.name);
+        const relPath = path.relative(root, absPath).split(path.sep).join("/");
+        const isDir = entry.isDirectory();
+        if (isDir && ALWAYS_IGNORE_DIRS.has(entry.name))
+            continue;
+        if (!isDir && !entry.isFile())
+            continue; // symlinks, sockets, fifos, etc.
+        if (isSecretPath(relPath))
+            continue;
+        if (isIgnored(stack, absPath, isDir))
+            continue;
+        if (isDir) {
+            yield* walkDir(absPath, root, stack, ignoreFileNames, maxFileBytes);
+            continue;
+        }
+        let size;
+        try {
+            size = (await fs.stat(absPath)).size;
+        }
+        catch {
+            continue;
+        }
+        if (size > maxFileBytes)
+            continue;
+        if (await isBinaryFile(absPath))
+            continue;
+        yield { relPath, absPath, size };
+    }
+}

package/dist/tools/index.d.ts

@@ -2,4 +2,5 @@ export * from "./types.js";
 export * from "./registry.js";
 export * from "./list-files.js";
 export * from "./git-status.js";
+export * from "./search-codebase.js";
 export * from "./file/index.js";

package/dist/tools/index.js

@@ -2,4 +2,5 @@ export * from "./types.js";
 export * from "./registry.js";
 export * from "./list-files.js";
 export * from "./git-status.js";
+export * from "./search-codebase.js";
 export * from "./file/index.js";

package/dist/tools/registry.js

@@ -3,6 +3,7 @@ import { listFilesTool } from "./list-files.js";
 import { gitStatusTool } from "./git-status.js";
 import { readFileTool, writeFileTool, editFileTool, applyPatchTool, globTool, grepFilesTool, } from "./file/index.js";
 import { runCommandTool } from "./shell/index.js";
+import { searchCodebaseTool } from "./search-codebase.js";
 /**
  * In-memory catalogue of the tools available to the agent. Names are unique;
  * `toToolSpecs()` projects the catalogue into the `@cruxy/sdk` wire format.
@@ -59,5 +60,6 @@ export function buildDefaultRegistry() {
     registry.register(grepFilesTool);
     registry.register(gitStatusTool);
     registry.register(runCommandTool);
+    registry.register(searchCodebaseTool);
     return registry;
 }

package/dist/tools/search-codebase.d.ts

@@ -0,0 +1,25 @@
+import { z } from "zod";
+import type { Tool } from "./types.js";
+declare const parameters: z.ZodObject<{
+    query: z.ZodString;
+    k: z.ZodOptional<z.ZodNumber>;
+    pathGlob: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    query: string;
+    k?: number | undefined;
+    pathGlob?: string | undefined;
+}, {
+    query: string;
+    k?: number | undefined;
+    pathGlob?: string | undefined;
+}>;
+/**
+ * Semantic search over the project's local code index (C.17). Read-only — no
+ * approval — like read_file and grep_files. The index is built/refreshed lazily
+ * on first use; results are ranked by cosine similarity and token-budgeted.
+ *
+ * Complements `grep_files`: prefer this for conceptual "where / how does X work"
+ * questions, and grep for exact strings or symbols.
+ */
+export declare const searchCodebaseTool: Tool<typeof parameters>;
+export {};

package/dist/tools/search-codebase.js

@@ -0,0 +1,70 @@
+import { z } from "zod";
+import { getIndexService } from "../indexing/index.js";
+/** Hard cap on `k`, mirroring the retriever. */
+const MAX_K = 50;
+const parameters = z.object({
+    query: z
+        .string()
+        .min(1)
+        .describe("A natural-language description of the code you're looking for (e.g. 'where are tool results fed back to the model', 'JWT verification'). Concepts work better than exact tokens."),
+    k: z
+        .number()
+        .int()
+        .positive()
+        .max(MAX_K)
+        .optional()
+        .describe("Number of results to return (default 8)."),
+    pathGlob: z
+        .string()
+        .optional()
+        .describe("Optional glob to restrict results by path, e.g. 'src/**/*.ts' or 'packages/cli/**'."),
+});
+/**
+ * Semantic search over the project's local code index (C.17). Read-only — no
+ * approval — like read_file and grep_files. The index is built/refreshed lazily
+ * on first use; results are ranked by cosine similarity and token-budgeted.
+ *
+ * Complements `grep_files`: prefer this for conceptual "where / how does X work"
+ * questions, and grep for exact strings or symbols.
+ */
+export const searchCodebaseTool = {
+    name: "search_codebase",
+    description: "Semantically search the project's code index for the snippets most relevant to a natural-language query. Returns ranked matches as 'path:startLine-endLine (score)' with a code snippet. Read-only, no approval. Prefer this for conceptual 'where is / how does X work' questions; use grep_files for exact strings or symbol names.",
+    parameters,
+    async execute(input, ctx) {
+        if (!ctx.config.index.enabled) {
+            return {
+                ok: false,
+                error: "codebase indexing is disabled (set index.enabled = true to use search_codebase)",
+            };
+        }
+        try {
+            const service = await getIndexService(ctx.cwd, ctx.config, ctx.logger);
+            const hits = await service.search({
+                query: input.query,
+                k: input.k,
+                pathGlob: input.pathGlob,
+            });
+            if (hits.length === 0) {
+                return { ok: true, output: "(no matches in the codebase index)" };
+            }
+            return { ok: true, output: formatHits(hits) };
+        }
+        catch (err) {
+            return { ok: false, error: err.message };
+        }
+    },
+};
+/** Render hits as a compact, model-readable block. */
+function formatHits(hits) {
+    return hits
+        .map((hit) => {
+        const header = `${hit.path}:${hit.startLine}-${hit.endLine}  (score ${hit.score.toFixed(3)})`;
+        const body = hit.snippet
+            .split("\n")
+            .map((line) => `    ${line}`)
+            .join("\n");
+        return `${header}\n${body}`;
+    })
+        .join("\n\n");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cruxy/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "an agentic coding CLI",
   "type": "module",
   "bin": {
@@ -28,7 +28,9 @@
     "directory": "packages/cli"
   },
   "dependencies": {
+    "better-sqlite3": "^12.11.1",
     "commander": "^12.1.0",
+    "fastembed": "^2.1.0",
     "picocolors": "^1.1.1",
     "tinyglobby": "^0.2.10",
     "zod": "^3.23.8",
@@ -36,6 +38,7 @@
     "@cruxy/sdk": "0.1.0"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^22.10.0",
     "tsx": "^4.19.2",
     "typescript": "^5.7.2",