@functional-systems/srcx 0.1.0-dev.4 → 0.1.0-dev.40

package/src/cli.ts

@@ -1,12 +1,23 @@
 #!/usr/bin/env node
 import { readFileSync } from "node:fs";
-import { Command, InvalidArgumentError } from "commander";
+import { Command, InvalidArgumentError, Option } from "commander";
 import { mkdir } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import { configure, loadSettings, stateRoot } from "./settings.js";
-import { identity } from "./git.js";
-import { loadBuild, materialize, validateBuild, type Build } from "./build.js";
+import { identity, resolveCommit } from "./git.js";
+import {
+  loadBuild,
+  materialize,
+  validateBuild,
+  type Build,
+  type Preset,
+  PRESET,
+  presetFor,
+  normalizeAnalyzers,
+  ANALYZERS,
+} from "./build.js";
+import { normalizeFilePolicy } from "./file-policy.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
 import {
@@ -16,9 +27,23 @@ import {
   register,
   selectRepository,
 } from "./repository.js";
-import { exclusive, publish, published, type Published } from "./publish.js";
+import {
+  exclusive,
+  lastBuildPath,
+  publish,
+  published,
+  type Published,
+} from "./publish.js";
 import { resolveVersion, syncTags } from "./releases.js";
-import { directHandle, loadHandle, readHandle, search } from "./search.js";
+import {
+  directHandle,
+  loadHandle,
+  readHandle,
+  search,
+  lookup,
+} from "./search.js";
+import { candidateLimit } from "./rerank.js";
+import { manageSkill, SKILL_AGENTS, SKILL_SCOPES } from "./skills.js";
 const cli = new Command()
   .name("srcx")
   .description("Version-aware code search on LambdaDB")
@@ -39,10 +64,48 @@ const integer = (value: string) => {
     throw new InvalidArgumentError("Integer is too large.");
   return n;
 };
+const analyzerOption = (description: string) =>
+  new Option(
+    "--analyzers <names>",
+    description + ` (${ANALYZERS.join(", ")}; default: standard)`,
+  ).argParser((value) =>
+    normalizeAnalyzers(value.split(",").map((name) => name.trim())),
+  );
+const filePolicyOption = () =>
+  new Option(
+    "--file-policy <json>",
+    "Path to file policy JSON; pinned at registration or local dry run",
+  ).argParser((path) =>
+    normalizeFilePolicy(JSON.parse(readFileSync(path, "utf8"))),
+  );
 async function connected() {
   const settings = await loadSettings();
   return { settings, remote: new LambdaRemote(settings) };
 }
+const skills = cli
+  .command("skills")
+  .description(
+    "Install and manage the bundled agent skill (no credentials required)",
+  );
+for (const operation of ["install", "update", "remove", "status"] as const) {
+  skills
+    .command(operation)
+    .addOption(
+      new Option("--agent <name>", "Agent to configure")
+        .choices(SKILL_AGENTS)
+        .makeOptionMandatory(),
+    )
+    .addOption(
+      new Option("--scope <scope>", "Installation scope")
+        .choices(SKILL_SCOPES)
+        .default("user"),
+    )
+    .option(
+      "--path <directory>",
+      "Project directory (project scope only; default: current directory)",
+    )
+    .action(async (options) => output(await manageSkill(operation, options)));
+}
 cli
   .command("configure")
   .requiredOption("--endpoint <origin>")
@@ -81,6 +144,16 @@ repo
   .option("--remote <name>")
   .option("--description <text>")
   .option("--tag <key=value>", "One optional Collection metadata tag")
+  .addOption(analyzerOption("Comma-separated text analyzers"))
+  .addOption(filePolicyOption())
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Managed model (source text is sent to the provider on import)",
+    )
+      .choices(["none", "text-embedding-3-small", "text-embedding-3-large"])
+      .default("none"),
+  )
   .action(async (o) => {
     const { remote } = await connected();
     let labels: Record<string, string> | undefined;
@@ -95,6 +168,7 @@ repo
         remote: o.remote,
         description: o.description,
         labels,
+        preset: presetFor(o.embedding, o.analyzers, o.filePolicy),
       }),
     );
   });
@@ -122,7 +196,27 @@ cli
   .option("--repo <name>", "Registered remote repository")
   .option("--ref <ref>", "Local branch, Git tag, or commit OID")
   .option("--dry-run", "Only materialize and validate a local build")
+  .addOption(filePolicyOption())
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Preset for --dry-run --path; connected imports use the repository preset",
+    ).choices(["none", "text-embedding-3-small", "text-embedding-3-large"]),
+  )
+  .addOption(
+    analyzerOption(
+      "Text analyzers for --dry-run --path; connected imports use the repository preset",
+    ),
+  )
   .option("--output <directory>", "New artifact directory")
+  .addOption(
+    new Option(
+      "--artifact-storage <storage>",
+      "Experimental local artifact storage",
+    )
+      .choices(["jsonl", "segments"])
+      .default("jsonl"),
+  )
   .option(
     "--previous <directory>",
     "Previous build artifact for a dry-run diff",
@@ -133,7 +227,19 @@ cli
     "Resume the same pending artifact after inspecting its journal",
   )
   .option("--timeout <seconds>", "Validation wait deadline", integer, 120)
-  .action(async (o) => {
+  .action(async (o, command) => {
+    invariant(
+      !o.embedding || (o.dryRun && o.path),
+      "--embedding is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
+    invariant(
+      !o.analyzers || (o.dryRun && o.path),
+      "--analyzers is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
+    invariant(
+      !o.filePolicy || (o.dryRun && o.path),
+      "--file-policy is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
     invariant(!(o.path && o.repo), "Choose --path or --repo.");
     invariant(
       !o.previous || o.dryRun,
@@ -144,19 +250,41 @@ cli
       !(o.artifact && (o.ref || o.path || o.output || o.previous)),
       "--artifact cannot be combined with --ref/--path/--output/--previous.",
     );
+    invariant(
+      !o.artifact || command.getOptionValueSource("artifactStorage") !== "cli",
+      "--artifact-storage cannot be combined with --artifact; the artifact records its storage format.",
+    );
     const buildLocal = async (
       source: Awaited<ReturnType<typeof identity>>,
       previous?: Build,
+      state?: string,
+      preset: Preset = PRESET,
     ) => {
       invariant(o.ref, "--ref is required to build a commit.");
+      const commit = await resolveCommit(source.path, o.ref);
+      if (state) {
+        const last = await optionalJson<{ artifact: string }>(
+          lastBuildPath(state, commit.branch),
+        );
+        if (last) {
+          try {
+            previous = await loadBuild(last.artifact);
+          } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+          }
+        }
+      }
       const parent = join(stateRoot(), "builds");
       await mkdir(parent, { recursive: true, mode: 0o700 });
       const path = o.output ? resolve(o.output) : join(parent, randomUUID());
       return materialize({
         identity: source,
         ref: o.ref,
+        resolvedCommit: commit,
         output: path,
         previous,
+        storage: o.artifactStorage,
+        preset,
       });
     };
     if (o.dryRun) {
@@ -166,23 +294,26 @@ cli
         await validateBuild(b);
       } else {
         let source;
+        let preset = presetFor(o.embedding, o.analyzers, o.filePolicy);
         if (o.path) source = await identity(o.path, o.remote);
         else {
           invariant(o.repo, "Use --path for a credential-free dry run.");
           const { remote, settings } = await connected();
-          source = await attachment(
-            settings,
-            await selectRepository(remote, o.repo),
-          );
+          const selected = await selectRepository(remote, o.repo);
+          preset = selected.preset ?? PRESET;
+          source = await attachment(settings, selected);
         }
         b = await buildLocal(
           source,
           o.previous ? await loadBuild(o.previous) : undefined,
+          undefined,
+          preset,
         );
       }
       output({
         status: "locally-validated",
         artifact: b.directory,
+        artifactFormat: b.format,
         commitOid: b.commitOid,
         configHash: b.configHash,
         counts: b.counts,
@@ -192,6 +323,8 @@ cli
           path: e.path,
           status: e.status,
           reason: e.reason,
+          retrieval: e.retrieval,
+          policyReason: e.policyReason,
           parseStatus: e.parseStatus,
         })),
         uploaded: false,
@@ -212,20 +345,34 @@ cli
         // Pending attempts reconcile against the journal's immutable remote
         // baseline; they do not need the previous build's local files.
         const pending = await optionalJson(join(state, "pending.json"));
+        const b = o.artifact
+          ? await loadBuild(o.artifact)
+          : await buildLocal(
+              await attachment(settings, r),
+              undefined,
+              pending ? undefined : state,
+              r.preset ?? PRESET,
+            );
         let baseline: { version: Published; build: Build } | undefined;
-        if (!pending) {
+        // Tracked branches discover their own immutable baseline remotely.
+        // Manual SHA/tag imports retain the previous frozen workspace optimization.
+        if (!pending && !b.branchRef) {
           const last = await optionalJson<{
             artifact: string;
             version: Published;
-          }>(join(state, "last-build.json"));
+          }>(lastBuildPath(state));
           if (last) {
-            const b = await loadBuild(last.artifact);
-            baseline = { version: last.version, build: b };
+            try {
+              baseline = {
+                version: last.version,
+                build: await loadBuild(last.artifact),
+              };
+            } catch (error) {
+              if ((error as NodeJS.ErrnoException).code !== "ENOENT")
+                throw error;
+            }
           }
         }
-        const b = o.artifact
-          ? await loadBuild(o.artifact)
-          : await buildLocal(await attachment(settings, r), baseline?.build);
         return publish({
           store,
           binding: r,
@@ -249,7 +396,7 @@ cli
 cli
   .command("resolve")
   .requiredOption("--repo <name>")
-  .requiredOption("--ref <commit-or-release>")
+  .requiredOption("--ref <commit-branch-or-release>")
   .action(async (o) => {
     const { remote } = await connected();
     const r = await selectRepository(remote, o.repo);
@@ -258,18 +405,102 @@ cli
 cli
   .command("search")
   .requiredOption("--repo <name>")
-  .requiredOption("--version <commit-or-release>")
-  .requiredOption("--query <text>")
+  .requiredOption("--version <commit-branch-or-release>")
+  .requiredOption(
+    "--query <text>",
+    "Search text; use the target source language",
+  )
+  .addHelpText(
+    "after",
+    "\nQuery language: any language is accepted; queries are not automatically translated.\n" +
+      "For English code/comments, prefer English terms and preserve identifiers, API names and error messages.\n" +
+      "For Korean documentation, use Korean queries with the korean analyzer.\n" +
+      "Agents can translate the search query while answering in the user's language.\n",
+  )
   .option("--limit <count>", "Maximum hits", integer, 10)
+  .addOption(
+    new Option(
+      "--mode <mode>",
+      "Retrieval method; semantic/hybrid incur managed query embedding usage",
+    )
+      .choices(["lexical", "semantic", "hybrid"])
+      .default("lexical"),
+  )
+  .addOption(
+    new Option(
+      "--rerank <model>",
+      "Opt-in local reranker (requires separate Python/model setup)",
+    ).choices(["qwen"]),
+  )
+  .option(
+    "--candidates <count>",
+    "Rerank pool size (default: max(50, limit); max: 100)",
+    integer,
+  )
   .option("--path <path>", "Exact path filter")
-  .option("--language <name>")
+  .option("--language <name>", "Exact programming-language filter")
   .action(async (o) => {
+    const started = performance.now();
+    candidateLimit(o.limit, o);
     const { remote, settings } = await connected();
     const r = await selectRepository(remote, o.repo),
       store = remote.store(r.collection),
       v = await resolveVersion(store, r, o.version);
     output(
-      await search(store, settings, r, v, o.query, o.limit, {
+      await search(
+        store,
+        settings,
+        r,
+        v,
+        o.query,
+        o.limit,
+        {
+          path: o.path,
+          language: o.language,
+        },
+        o.mode,
+        {
+          rerank: o.rerank,
+          candidates: o.candidates,
+          onTiming: (timing) =>
+            process.stderr.write(
+              JSON.stringify({
+                event: "rerank-timing",
+                ...timing,
+                commandMs: performance.now() - started,
+              }) + "\n",
+            ),
+        },
+      ),
+    );
+  });
+cli
+  .command("lookup")
+  .description(
+    "Find exact indexed symbol definitions and return their source chunks",
+  )
+  .requiredOption("--repo <name>")
+  .requiredOption("--version <commit-branch-or-release>")
+  .requiredOption("--symbol <name>", "Exact case-sensitive indexed symbol name")
+  .option("--path <path>", "Exact path filter for ambiguous names")
+  .option("--language <name>", "Exact programming-language filter")
+  .option(
+    "--limit <count>",
+    "Maximum source chunks (1–20; results may be partial)",
+    integer,
+    5,
+  )
+  .action(async (o) => {
+    invariant(
+      o.limit >= 1 && o.limit <= 20,
+      "Lookup limit must be between 1 and 20.",
+    );
+    const { remote, settings } = await connected();
+    const r = await selectRepository(remote, o.repo),
+      store = remote.store(r.collection);
+    const v = await resolveVersion(store, r, o.version);
+    output(
+      await lookup(store, settings, r, v, o.symbol, o.limit, {
         path: o.path,
         language: o.language,
       }),
@@ -279,12 +510,20 @@ cli
   .command("read")
   .option("--result <id>")
   .option("--repo <name>")
-  .option("--version <commit-or-release>")
+  .option("--version <commit-branch-or-release>")
   .option("--path <path>")
   .option("--lines <start:end>")
   .option("--context <lines>", "Expand an evidence span", integer, 0)
   .option("--full-file")
+  .option(
+    "--implementation",
+    "Read a linked Python/TypeScript overload implementation",
+  )
   .action(async (o) => {
+    invariant(
+      !o.implementation || (o.result && !o.fullFile && !o.lines && !o.context),
+      "--implementation requires --result and cannot be combined with range options.",
+    );
     invariant(
       !(o.fullFile && (o.lines || o.context)),
       "--full-file cannot be combined with --lines or --context.",
@@ -331,6 +570,7 @@ cli
           context: o.context,
           fullFile: o.fullFile || (!o.result && !o.lines),
           lines,
+          implementation: o.implementation,
         },
       ),
     );

package/src/file-policy.ts

@@ -0,0 +1,166 @@
+import { invariant } from "./common.js";
+
+export type FileAction = "exclude" | "lexical" | "semantic";
+export type FileRule = { pattern: string; action: FileAction };
+export type FilePolicy = { version: 1; rules: FileRule[] };
+export const FILE_POLICY: FilePolicy = { version: 1, rules: [] };
+export type FileDecision = { action: FileAction; reason: string };
+
+/** Deliberately small glob syntax: repo-relative paths, *, ?, and whole-segment **. */
+function glob(pattern: string): RegExp {
+  let expression = "^";
+  const parts = pattern.split("/");
+  for (const [index, part] of parts.entries()) {
+    if (part === "**")
+      expression += index === parts.length - 1 ? "[\\s\\S]*" : "(?:[^/]+/)*";
+    else {
+      for (const char of part)
+        expression +=
+          char === "*"
+            ? "[^/]*"
+            : char === "?"
+              ? "[^/]"
+              : char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      if (index !== parts.length - 1) expression += "/";
+    }
+  }
+  return new RegExp(expression + "$", "u");
+}
+
+export function normalizeFilePolicy(value: unknown): FilePolicy {
+  invariant(
+    value !== null && typeof value === "object" && !Array.isArray(value),
+    "File policy must be an object.",
+  );
+  const policy = value as FilePolicy;
+  invariant(
+    Object.keys(policy).sort().join(",") === "rules,version" &&
+      policy.version === 1 &&
+      Array.isArray(policy.rules) &&
+      policy.rules.length <= 100,
+    "File policy requires version: 1 and at most 100 rules.",
+  );
+  const rules = policy.rules.map((rule) => {
+    invariant(
+      rule &&
+        typeof rule === "object" &&
+        Object.keys(rule).sort().join(",") === "action,pattern" &&
+        ["exclude", "lexical", "semantic"].includes(rule.action),
+      "Each file rule requires pattern and action (exclude, lexical, semantic).",
+    );
+    invariant(
+      typeof rule.pattern === "string" &&
+        rule.pattern.length > 0 &&
+        rule.pattern.length <= 512 &&
+        !/[\\\x00-\x1f\x7f\[\]{}!]/.test(rule.pattern) &&
+        rule.pattern
+          .split("/")
+          .every(
+            (part) =>
+              part &&
+              part !== "." &&
+              part !== ".." &&
+              (!part.includes("**") || part === "**"),
+          ),
+      "File policy patterns must be relative paths using only *, ?, and whole-segment ** wildcards.",
+    );
+    glob(rule.pattern);
+    return { pattern: rule.pattern, action: rule.action };
+  });
+  return { version: 1, rules };
+}
+
+/** These bodies cannot be re-enabled by repository rules. This is not a secret scanner. */
+export function hardExclusion(path: string): string | undefined {
+  const name = path.split("/").at(-1)!;
+  if (
+    (/^\.env(?:\.|$)/i.test(name) &&
+      !/\.(example|sample|template)$/i.test(name)) ||
+    /^(\.npmrc|\.pypirc|\.netrc|_netrc|id_rsa|id_dsa|id_ecdsa|id_ed25519)$/i.test(
+      name,
+    ) ||
+    /(^|\/)\.(aws|ssh)(\/|$)/i.test(path)
+  )
+    return "credential-file";
+  if (
+    /\.(pem|key|crt|cer|csr|der|p12|pfx|p7b|p7c|jks|keystore|srl)$/i.test(path)
+  )
+    return "crypto-material";
+  if (
+    /\.(svgz?|png|apng|jpe?g|jfif|gif|webp|avif|bmp|dib|tiff?|ico|icns|heic|heif|jxl|psd|ai|eps|pnm|pbm|pgm|ppm|xbm|xpm)$/i.test(
+      path,
+    )
+  )
+    return "image-extension";
+  if (
+    /\.(pdf|zip|gz|bz2|xz|7z|tar|rar|jar|class|woff2?|ttf|otf|mp[34]|wav|ogg|mov|exe|dll|so|dylib|wasm|pyc|pyo|o|a|sqlite3?|db)$/i.test(
+      path,
+    )
+  )
+    return "binary-extension";
+  return undefined;
+}
+
+export function privateKeyContent(source: string): boolean {
+  return /(?:^|\r?\n)-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----\r?\n/.test(
+    source,
+  );
+}
+
+function excludedDirectory(path: string): boolean {
+  // Under a Java source-set root, build/dist/target/coverage can be package
+  // names. Output directories above that root still exclude the whole tree.
+  const javaRoot = /(^|\/)src\/[^/]+\/java\//i.exec(path);
+  const outputPath =
+    javaRoot && /\.java$/i.test(path) ? path.slice(0, javaRoot.index) : path;
+  return (
+    /(^|\/)(node_modules|vendor|vendored|third_party|third-party|\.git|\.next|\.nuxt|\.venv|venv|__pycache__|\.cache|\.pytest_cache|\.mypy_cache|\.ruff_cache|\.tox|\.nox|\.srcx)(\/|$)/i.test(
+      path,
+    ) || /(^|\/)(dist|build|target|coverage)(\/|$)/i.test(outputPath)
+  );
+}
+
+/** Explicit rules override defaults, in order; the final matching rule wins. */
+export function fileDecision(
+  path: string,
+  policy: FilePolicy,
+  source?: string,
+): FileDecision {
+  let decision: FileDecision = { action: "semantic", reason: "source" };
+  if (excludedDirectory(path))
+    decision = { action: "exclude", reason: "dependency-or-build-output" };
+  else if (/\.(min\.(js|css)|map)$/i.test(path))
+    decision = { action: "exclude", reason: "generated-output" };
+  else if (/\.(log|log\.\d+)$/i.test(path))
+    decision = { action: "exclude", reason: "log-output" };
+  else if (
+    /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|go\.sum|[^/]+\.lock|packages\.lock\.json|Package\.resolved)$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "lockfile" };
+  else if (
+    /(^|\/)(licen[sc]e|copying|copyright|notice|authors|contributors)([.-][^/]*)?$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "legal-or-attribution" };
+  else if (
+    /\.(csv|tsv|jsonl|ndjson|snap)$/i.test(path) ||
+    /(^|\/)__snapshots__\//i.test(path)
+  )
+    decision = { action: "lexical", reason: "data-or-snapshot" };
+  else if (
+    /\.(pb\.go|generated\.[^/]+|g\.[^/]+)$/i.test(path) ||
+    /_pb2(?:_grpc)?\.py$/i.test(path) ||
+    (source !== undefined &&
+      /(?:^|\n)[ \t]*(?:\/\/|#|\/\*|\*|<!--)[^\r\n]*(?:code generated[^\r\n]*do not edit|(?:auto[- ]?generated|generated by)[^\r\n]*do not edit|do not edit[^\r\n]*generated|@generated(?:[ \t]|$))/i.test(
+        Buffer.from(source, "utf8").subarray(0, 8192).toString("utf8"),
+      ))
+  )
+    decision = { action: "lexical", reason: "generated-source" };
+  for (const rule of policy.rules)
+    if (glob(rule.pattern).test(path))
+      decision = { action: rule.action, reason: `file-rule:${rule.action}` };
+  return decision;
+}