@functional-systems/srcx 0.1.0-dev.23 → 0.1.0-dev.25

package/src/cli.ts

@@ -17,6 +17,7 @@ import {
   normalizeAnalyzers,
   ANALYZERS,
 } from "./build.js";
+import { normalizeFilePolicy } from "./file-policy.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
 import {
@@ -36,6 +37,7 @@ import {
 import { resolveVersion, syncTags } from "./releases.js";
 import { directHandle, loadHandle, readHandle, search } from "./search.js";
 import { candidateLimit } from "./rerank.js";
+import { manageSkill, SKILL_AGENTS, SKILL_SCOPES } from "./skills.js";
 const cli = new Command()
   .name("srcx")
   .description("Version-aware code search on LambdaDB")
@@ -63,10 +65,41 @@ const analyzerOption = (description: string) =>
   ).argParser((value) =>
     normalizeAnalyzers(value.split(",").map((name) => name.trim())),
   );
+const filePolicyOption = () =>
+  new Option(
+    "--file-policy <json>",
+    "Path to file policy JSON; pinned at registration or local dry run",
+  ).argParser((path) =>
+    normalizeFilePolicy(JSON.parse(readFileSync(path, "utf8"))),
+  );
 async function connected() {
   const settings = await loadSettings();
   return { settings, remote: new LambdaRemote(settings) };
 }
+const skills = cli
+  .command("skills")
+  .description(
+    "Install and manage the bundled agent skill (no credentials required)",
+  );
+for (const operation of ["install", "update", "remove", "status"] as const) {
+  skills
+    .command(operation)
+    .addOption(
+      new Option("--agent <name>", "Agent to configure")
+        .choices(SKILL_AGENTS)
+        .makeOptionMandatory(),
+    )
+    .addOption(
+      new Option("--scope <scope>", "Installation scope")
+        .choices(SKILL_SCOPES)
+        .default("user"),
+    )
+    .option(
+      "--path <directory>",
+      "Project directory (project scope only; default: current directory)",
+    )
+    .action(async (options) => output(await manageSkill(operation, options)));
+}
 cli
   .command("configure")
   .requiredOption("--endpoint <origin>")
@@ -106,6 +139,7 @@ repo
   .option("--description <text>")
   .option("--tag <key=value>", "One optional Collection metadata tag")
   .addOption(analyzerOption("Comma-separated text analyzers"))
+  .addOption(filePolicyOption())
   .addOption(
     new Option(
       "--embedding <model>",
@@ -128,7 +162,7 @@ repo
         remote: o.remote,
         description: o.description,
         labels,
-        preset: presetFor(o.embedding, o.analyzers),
+        preset: presetFor(o.embedding, o.analyzers, o.filePolicy),
       }),
     );
   });
@@ -156,6 +190,7 @@ cli
   .option("--repo <name>", "Registered remote repository")
   .option("--ref <ref>", "Local branch, Git tag, or commit OID")
   .option("--dry-run", "Only materialize and validate a local build")
+  .addOption(filePolicyOption())
   .addOption(
     new Option(
       "--embedding <model>",
@@ -187,6 +222,10 @@ cli
       !o.analyzers || (o.dryRun && o.path),
       "--analyzers is only accepted with --dry-run --path; connected imports use the repository preset.",
     );
+    invariant(
+      !o.filePolicy || (o.dryRun && o.path),
+      "--file-policy is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
     invariant(!(o.path && o.repo), "Choose --path or --repo.");
     invariant(
       !o.previous || o.dryRun,
@@ -236,7 +275,7 @@ cli
         await validateBuild(b);
       } else {
         let source;
-        let preset = presetFor(o.embedding, o.analyzers);
+        let preset = presetFor(o.embedding, o.analyzers, o.filePolicy);
         if (o.path) source = await identity(o.path, o.remote);
         else {
           invariant(o.repo, "Use --path for a credential-free dry run.");
@@ -264,6 +303,8 @@ cli
           path: e.path,
           status: e.status,
           reason: e.reason,
+          retrieval: e.retrieval,
+          policyReason: e.policyReason,
           parseStatus: e.parseStatus,
         })),
         uploaded: false,

package/src/file-policy.ts

@@ -0,0 +1,157 @@
+import { invariant } from "./common.js";
+
+export type FileAction = "exclude" | "lexical" | "semantic";
+export type FileRule = { pattern: string; action: FileAction };
+export type FilePolicy = { version: 1; rules: FileRule[] };
+export const FILE_POLICY: FilePolicy = { version: 1, rules: [] };
+export type FileDecision = { action: FileAction; reason: string };
+
+/** Deliberately small glob syntax: repo-relative paths, *, ?, and whole-segment **. */
+function glob(pattern: string): RegExp {
+  let expression = "^";
+  const parts = pattern.split("/");
+  for (const [index, part] of parts.entries()) {
+    if (part === "**")
+      expression += index === parts.length - 1 ? "[\\s\\S]*" : "(?:[^/]+/)*";
+    else {
+      for (const char of part)
+        expression +=
+          char === "*"
+            ? "[^/]*"
+            : char === "?"
+              ? "[^/]"
+              : char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      if (index !== parts.length - 1) expression += "/";
+    }
+  }
+  return new RegExp(expression + "$", "u");
+}
+
+export function normalizeFilePolicy(value: unknown): FilePolicy {
+  invariant(
+    value !== null && typeof value === "object" && !Array.isArray(value),
+    "File policy must be an object.",
+  );
+  const policy = value as FilePolicy;
+  invariant(
+    Object.keys(policy).sort().join(",") === "rules,version" &&
+      policy.version === 1 &&
+      Array.isArray(policy.rules) &&
+      policy.rules.length <= 100,
+    "File policy requires version: 1 and at most 100 rules.",
+  );
+  const rules = policy.rules.map((rule) => {
+    invariant(
+      rule &&
+        typeof rule === "object" &&
+        Object.keys(rule).sort().join(",") === "action,pattern" &&
+        ["exclude", "lexical", "semantic"].includes(rule.action),
+      "Each file rule requires pattern and action (exclude, lexical, semantic).",
+    );
+    invariant(
+      typeof rule.pattern === "string" &&
+        rule.pattern.length > 0 &&
+        rule.pattern.length <= 512 &&
+        !/[\\\x00-\x1f\x7f\[\]{}!]/.test(rule.pattern) &&
+        rule.pattern
+          .split("/")
+          .every(
+            (part) =>
+              part &&
+              part !== "." &&
+              part !== ".." &&
+              (!part.includes("**") || part === "**"),
+          ),
+      "File policy patterns must be relative paths using only *, ?, and whole-segment ** wildcards.",
+    );
+    glob(rule.pattern);
+    return { pattern: rule.pattern, action: rule.action };
+  });
+  return { version: 1, rules };
+}
+
+/** These bodies cannot be re-enabled by repository rules. This is not a secret scanner. */
+export function hardExclusion(path: string): string | undefined {
+  const name = path.split("/").at(-1)!;
+  if (
+    (/^\.env(?:\.|$)/i.test(name) &&
+      !/\.(example|sample|template)$/i.test(name)) ||
+    /^(\.npmrc|\.pypirc|\.netrc|_netrc|id_rsa|id_dsa|id_ecdsa|id_ed25519)$/i.test(
+      name,
+    ) ||
+    /(^|\/)\.(aws|ssh)(\/|$)/i.test(path)
+  )
+    return "credential-file";
+  if (
+    /\.(pem|key|crt|cer|csr|der|p12|pfx|p7b|p7c|jks|keystore|srl)$/i.test(path)
+  )
+    return "crypto-material";
+  if (
+    /\.(svgz?|png|apng|jpe?g|jfif|gif|webp|avif|bmp|dib|tiff?|ico|icns|heic|heif|jxl|psd|ai|eps|pnm|pbm|pgm|ppm|xbm|xpm)$/i.test(
+      path,
+    )
+  )
+    return "image-extension";
+  if (
+    /\.(pdf|zip|gz|bz2|xz|7z|tar|rar|jar|class|woff2?|ttf|otf|mp[34]|wav|ogg|mov|exe|dll|so|dylib|wasm|pyc|pyo|o|a|sqlite3?|db)$/i.test(
+      path,
+    )
+  )
+    return "binary-extension";
+  return undefined;
+}
+
+export function privateKeyContent(source: string): boolean {
+  return /(?:^|\r?\n)-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----\r?\n/.test(
+    source,
+  );
+}
+
+/** Explicit rules override defaults, in order; the final matching rule wins. */
+export function fileDecision(
+  path: string,
+  policy: FilePolicy,
+  source?: string,
+): FileDecision {
+  let decision: FileDecision = { action: "semantic", reason: "source" };
+  if (
+    /(^|\/)(node_modules|vendor|vendored|third_party|third-party|dist|build|target|coverage|\.git|\.next|\.nuxt|\.venv|venv|__pycache__|\.cache|\.pytest_cache|\.mypy_cache|\.ruff_cache|\.tox|\.nox|\.srcx)(\/|$)/i.test(
+      path,
+    )
+  )
+    decision = { action: "exclude", reason: "dependency-or-build-output" };
+  else if (/\.(min\.(js|css)|map)$/i.test(path))
+    decision = { action: "exclude", reason: "generated-output" };
+  else if (/\.(log|log\.\d+)$/i.test(path))
+    decision = { action: "exclude", reason: "log-output" };
+  else if (
+    /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|go\.sum|[^/]+\.lock|packages\.lock\.json|Package\.resolved)$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "lockfile" };
+  else if (
+    /(^|\/)(licen[sc]e|copying|copyright|notice|authors|contributors)([.-][^/]*)?$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "legal-or-attribution" };
+  else if (
+    /\.(csv|tsv|jsonl|ndjson|snap)$/i.test(path) ||
+    /(^|\/)__snapshots__\//i.test(path)
+  )
+    decision = { action: "lexical", reason: "data-or-snapshot" };
+  else if (
+    /\.(pb\.go|generated\.[^/]+|g\.[^/]+)$/i.test(path) ||
+    /_pb2(?:_grpc)?\.py$/i.test(path) ||
+    (source !== undefined &&
+      /(?:^|\n)[ \t]*(?:\/\/|#|\/\*|\*|<!--)[^\r\n]*(?:code generated[^\r\n]*do not edit|(?:auto[- ]?generated|generated by)[^\r\n]*do not edit|do not edit[^\r\n]*generated|@generated(?:[ \t]|$))/i.test(
+        Buffer.from(source, "utf8").subarray(0, 8192).toString("utf8"),
+      ))
+  )
+    decision = { action: "lexical", reason: "generated-source" };
+  for (const rule of policy.rules)
+    if (glob(rule.pattern).test(path))
+      decision = { action: rule.action, reason: `file-rule:${rule.action}` };
+  return decision;
+}

package/src/skills.ts

@@ -0,0 +1,205 @@
+import { readFileSync } from "node:fs";
+import {
+  lstat,
+  mkdir,
+  readFile,
+  readdir,
+  realpath,
+  rename,
+  rmdir,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
+import { homedir } from "node:os";
+import { join, resolve } from "node:path";
+import { hash, invariant } from "./common.js";
+
+export const SKILL_AGENTS = ["codex", "claude"] as const;
+export const SKILL_SCOPES = ["user", "project"] as const;
+type Agent = (typeof SKILL_AGENTS)[number];
+type Scope = (typeof SKILL_SCOPES)[number];
+type Operation = "install" | "update" | "remove" | "status";
+const name = "srcx-search";
+const owner = "@functional-systems/srcx";
+const pkg = JSON.parse(
+  readFileSync(new URL("../package.json", import.meta.url), "utf8"),
+);
+const marker = "<!-- srcx-install ";
+
+async function stat(path: string) {
+  try {
+    return await lstat(path);
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === "ENOENT") return undefined;
+    throw e;
+  }
+}
+async function directory(path: string, create = false): Promise<boolean> {
+  if (create) {
+    try {
+      await mkdir(path);
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
+    }
+  }
+  const s = await stat(path);
+  invariant(
+    !s || (s.isDirectory() && !s.isSymbolicLink()),
+    `Expected a directory, not a file or symlink: ${path}`,
+  );
+  return !!s;
+}
+
+export async function manageSkill(
+  operation: Operation,
+  options: {
+    agent: Agent;
+    scope: Scope;
+    path?: string;
+  },
+  userHome = homedir(),
+) {
+  invariant(
+    SKILL_AGENTS.includes(options.agent),
+    "Choose --agent codex or claude.",
+  );
+  invariant(
+    SKILL_SCOPES.includes(options.scope),
+    "Choose --scope user or project.",
+  );
+  invariant(
+    options.scope === "project" || options.path === undefined,
+    "--path requires --scope project.",
+  );
+  // Resolve the chosen root, then refuse redirected managed path components.
+  const root = await realpath(
+    options.scope === "user" ? userHome : resolve(options.path ?? "."),
+  );
+  invariant(
+    (await lstat(root)).isDirectory(),
+    "Skill scope root must be a directory.",
+  );
+  const dirs = [join(root, options.agent === "codex" ? ".agents" : ".claude")];
+  dirs.push(join(dirs[0]!, "skills"));
+  const parent = dirs[1]!,
+    target = join(parent, name),
+    file = join(target, "SKILL.md");
+  const bundled = await readFile(
+    new URL("../skills/srcx-search/SKILL.md", import.meta.url),
+    "utf8",
+  );
+  const bundledHash = hash(bundled);
+  const info = {
+    agent: options.agent,
+    scope: options.scope,
+    path: target,
+    bundledVersion: pkg.version as string,
+  };
+  const encoded = `${bundled}${marker}${JSON.stringify({ package: owner, version: pkg.version, sha256: bundledHash })} -->\n`;
+  async function inspect() {
+    if (!(await directory(target))) return { status: "not-installed" as const };
+    const entries = await readdir(target);
+    const s = await stat(file);
+    if (!s?.isFile() || s.isSymbolicLink())
+      return { status: "unmanaged" as const };
+    const text = await readFile(file, "utf8");
+    const at = text.lastIndexOf(marker);
+    let installed: { package?: unknown; version?: unknown; sha256?: unknown };
+    try {
+      installed = JSON.parse(text.slice(at + marker.length, -5));
+    } catch {
+      return { status: "unmanaged" as const };
+    }
+    if (
+      at < 0 ||
+      !text.endsWith(" -->\n") ||
+      installed?.package !== owner ||
+      typeof installed.version !== "string" ||
+      !installed.version ||
+      typeof installed.sha256 !== "string" ||
+      !/^[a-f0-9]{64}$/.test(installed.sha256)
+    ) {
+      return { status: "unmanaged" as const };
+    }
+    const status =
+      entries.length !== 1 || hash(text.slice(0, at)) !== installed.sha256
+        ? "modified"
+        : installed.sha256 === bundledHash && installed.version === pkg.version
+          ? "current"
+          : "update-available";
+    return { status, installedVersion: installed.version };
+  }
+  // Status and absent removals are read-only, including missing parent directories.
+  for (const dir of dirs) {
+    if (!(await directory(dir, operation === "install"))) {
+      invariant(
+        operation !== "update",
+        "Skill is not installed; run skills install first.",
+      );
+      return { ...info, status: "not-installed" };
+    }
+  }
+  if (operation === "status" || operation === "remove") {
+    const current = await inspect();
+    if (operation === "status" || current.status === "not-installed")
+      return { ...info, ...current };
+  }
+  const lock = join(parent, `.${name}.srcx-lock`);
+  try {
+    await mkdir(lock);
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === "EEXIST")
+      throw new Error(
+        `Another skill operation may be active. Retry later; inspect a stale lock before removing it: ${lock}`,
+      );
+    throw e;
+  }
+  try {
+    // Recheck after locking before changing an existing installation.
+    const current = await inspect();
+    invariant(
+      current.status !== "unmanaged" && current.status !== "modified",
+      `Existing skill is ${current.status}; preserve or move it before ${operation}: ${target}`,
+    );
+    if (operation === "remove") {
+      if (current.status === "not-installed")
+        return { ...info, status: "not-installed" };
+      await unlink(file);
+      await rmdir(target); // Never recursively remove unrecognized files.
+      return { ...info, status: "removed" };
+    }
+    if (current.status === "current") return { ...info, ...current };
+    invariant(
+      operation !== "update" || current.status !== "not-installed",
+      "Skill is not installed; run skills install first.",
+    );
+    invariant(
+      operation !== "install" || current.status === "not-installed",
+      "A different bundled skill is installed; run skills update.",
+    );
+    // One file carries both instructions and provenance, so updates cannot tear
+    // a separate ownership manifest away from the installed content.
+    const temp = join(lock, "SKILL.md");
+    await writeFile(temp, encoded, { flag: "wx", mode: 0o644 });
+    const fresh = current.status === "not-installed";
+    if (fresh) await mkdir(target);
+    try {
+      await rename(temp, file);
+    } catch (e) {
+      if (fresh) await rmdir(target);
+      throw e;
+    }
+    return {
+      ...info,
+      status: fresh ? "installed" : "updated",
+      installedVersion: pkg.version as string,
+    };
+  } finally {
+    try {
+      await unlink(join(lock, "SKILL.md"));
+    } catch (e) {
+      if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e;
+    }
+    await rmdir(lock);
+  }
+}