@functional-systems/srcx 0.1.0-dev.24 → 0.1.0-dev.25

package/src/build.ts

@@ -15,6 +15,15 @@ import {
   type Doc,
   optionalJson,
 } from "./common.js";
+import {
+  FILE_POLICY,
+  normalizeFilePolicy,
+  hardExclusion,
+  privateKeyContent,
+  fileDecision,
+  type FilePolicy,
+  type FileDecision,
+} from "./file-policy.js";
 export const ANALYZERS = ["english", "japanese", "korean", "standard"] as const;
 export type Analyzer = (typeof ANALYZERS)[number];
 export function normalizeAnalyzers(value: unknown): Analyzer[] {
@@ -42,6 +51,7 @@ export type Preset = {
   /** Internal evaluation override; the CLI keeps its original enrichment. */
   enrichment?: "path-only-v1";
   maxFileBytes: number;
+  filePolicy: FilePolicy;
   embedding: {
     model: string;
     dimensions: number;
@@ -57,6 +67,7 @@ export const PRESET: Preset = {
   chunker: CHUNKER,
   mode: "syntax",
   maxFileBytes: 1024 * 1024,
+  filePolicy: FILE_POLICY,
   embedding: null,
 };
 /** Embedding variants share the same chunking behavior. */
@@ -82,6 +93,7 @@ export const MANAGED_LARGE_PRESET: Preset = {
 export function presetFor(
   embedding = "none",
   analyzers: readonly string[] = PRESET.analyzers,
+  filePolicy: unknown = FILE_POLICY,
 ): Preset {
   invariant(
     ["none", "text-embedding-3-small", "text-embedding-3-large"].includes(
@@ -96,17 +108,21 @@ export function presetFor(
         ? MANAGED_LARGE_PRESET
         : MANAGED_PRESET;
   const normalized = normalizeAnalyzers(analyzers);
-  return hash(normalized) === hash(base.analyzers)
-    ? base
-    : { ...base, analyzers: normalized };
+  const configured = {
+    ...base,
+    analyzers: normalized,
+    filePolicy: normalizeFilePolicy(filePolicy),
+  };
+  return hash(configured) === hash(base) ? base : configured;
 }
 export function supportedPreset(preset: unknown): preset is Preset {
   if (!preset || typeof preset !== "object" || Array.isArray(preset))
     return false;
   try {
     const analyzers = normalizeAnalyzers((preset as Preset).analyzers);
+    const filePolicy = normalizeFilePolicy((preset as Preset).filePolicy);
     return [PRESET, MANAGED_PRESET, MANAGED_LARGE_PRESET].some(
-      (base) => hash({ ...base, analyzers }) === hash(preset),
+      (base) => hash({ ...base, analyzers, filePolicy }) === hash(preset),
     );
   } catch {
     return false;
@@ -154,6 +170,8 @@ export type InventoryItem = {
   bytes: number;
   status: "included" | "excluded";
   reason?: string;
+  retrieval?: "lexical" | "semantic";
+  policyReason?: string;
   targetBase64?: string;
   fileId?: string;
   chunkIds?: string[];
@@ -215,22 +233,6 @@ export async function loadBuild(directory: string): Promise<Build> {
   b.directory = resolve(directory);
   return b;
 }
-function exclude(path: string): string | undefined {
-  if (
-    /(^|\/)(node_modules|vendor|vendored|dist|build|target|coverage|\.git|\.next)(\/|$)/.test(
-      path,
-    )
-  )
-    return "dependency-or-build-output";
-  if (
-    /\.(png|jpe?g|gif|ico|pdf|zip|gz|jar|class|woff2?|mp[34]|exe|dll|so|dylib|wasm)$/i.test(
-      path,
-    )
-  )
-    return "binary-extension";
-  if (/\.(min\.(js|css)|map)$/.test(path)) return "generated-output";
-  return undefined;
-}
 export async function materialize(args: {
   identity: Identity;
   ref: string;
@@ -245,7 +247,8 @@ export async function materialize(args: {
     configHash = hash(preset),
     prev = args.previous;
   invariant(
-    hash(preset.analyzers) === hash(normalizeAnalyzers(preset.analyzers)) &&
+    hash(preset.filePolicy) === hash(normalizeFilePolicy(preset.filePolicy)) &&
+      hash(preset.analyzers) === hash(normalizeAnalyzers(preset.analyzers)) &&
       hash(preset.chunker) === hash(CHUNKER) &&
       ["syntax", "window"].includes(preset.mode) &&
       (preset.enrichment === undefined ||
@@ -317,7 +320,10 @@ export async function materialize(args: {
         item.targetBase64 = (await blobs.read(e)).toString("base64");
         continue;
       }
-      const reason = exclude(e.path);
+      const pathDecision = fileDecision(e.path, preset.filePolicy);
+      const reason =
+        hardExclusion(e.path) ??
+        (pathDecision.action === "exclude" ? pathDecision.reason : undefined);
       if (reason) {
         item.reason = reason;
         continue;
@@ -350,6 +356,11 @@ export async function materialize(args: {
         item.reason = "lfs-pointer";
         continue;
       }
+      if (privateKeyContent(source)) {
+        item.reason = "private-key-content";
+        continue;
+      }
+      const decision = fileDecision(e.path, preset.filePolicy, source);
       const parsed = await chunk(
         source,
         e.path,
@@ -378,6 +389,8 @@ export async function materialize(args: {
       }
       Object.assign(item, {
         status: "included",
+        retrieval: decision.action,
+        policyReason: decision.reason,
         contentHash,
         fileId,
         chunkIds: [],
@@ -404,7 +417,9 @@ export async function materialize(args: {
             ? "structural-only"
             : "embedding-disabled",
         };
-        const eligible = !["imports", "structural"].includes(span.chunkKind);
+        const eligible =
+          decision.action === "semantic" &&
+          !["imports", "structural"].includes(span.chunkKind);
         if (preset.embedding && eligible) {
           const inputHash = hash([preset.embedding, span.searchText]);
           if (preset.embedding.managed) {
@@ -443,9 +458,11 @@ export async function materialize(args: {
           }
         } else {
           doc.embeddingSkipReason = preset.embedding
-            ? span.chunkKind === "imports"
-              ? "imports-only"
-              : "structural-only"
+            ? decision.action === "lexical"
+              ? decision.reason
+              : span.chunkKind === "imports"
+                ? "imports-only"
+                : "structural-only"
             : "embedding-disabled";
           item.skipped!++;
         }
@@ -545,7 +562,10 @@ export async function validateBuild(b: Build): Promise<void> {
     "Build commit identity mismatch; rebuild from the intended Git commit.",
   );
   invariant(
-    hash(b.preset.analyzers) === hash(normalizeAnalyzers(b.preset.analyzers)) &&
+    hash(b.preset.filePolicy) ===
+      hash(normalizeFilePolicy(b.preset.filePolicy)) &&
+      hash(b.preset.analyzers) ===
+        hash(normalizeAnalyzers(b.preset.analyzers)) &&
       hash(b.preset) === b.configHash &&
       hash(b.inventory) === b.inventoryHash,
     "Build metadata hash mismatch.",
@@ -559,6 +579,7 @@ export async function validateBuild(b: Build): Promise<void> {
       .filter((e) => e.status === "included")
       .map((e) => [e.fileId!, e]),
   );
+  let activeDecision: FileDecision | undefined;
   const completedFiles = new Set<string>();
   const finish = () => {
     if (activeFile) {
@@ -598,6 +619,19 @@ export async function validateBuild(b: Build): Promise<void> {
       end = 0;
       count = 0;
       const entry = expectedFile.get(doc.id);
+      activeDecision = fileDecision(
+        doc.path as string,
+        b.preset.filePolicy,
+        doc.sourceText as string,
+      );
+      invariant(
+        !hardExclusion(doc.path as string) &&
+          !privateKeyContent(doc.sourceText as string) &&
+          activeDecision.action !== "exclude" &&
+          entry?.retrieval === activeDecision.action &&
+          entry?.policyReason === activeDecision.reason,
+        "File policy or inventory decision mismatch.",
+      );
       const raw = Buffer.from(doc.sourceText as string);
       invariant(
         entry &&
@@ -648,9 +682,9 @@ export async function validateBuild(b: Build): Promise<void> {
         "Invalid chunk source/position/token binding.",
       );
       if (b.preset.embedding?.managed) {
-        const eligible = !["imports", "structural"].includes(
-          String(doc.chunkKind),
-        );
+        const eligible =
+          activeDecision!.action === "semantic" &&
+          !["imports", "structural"].includes(String(doc.chunkKind));
         invariant(
           eligible
             ? doc.embeddingStatus === "managed" &&
@@ -662,14 +696,18 @@ export async function validateBuild(b: Build): Promise<void> {
                 doc.embeddingText === undefined &&
                 doc.embeddingInputHash === undefined &&
                 doc.embeddingSkipReason ===
-                  (doc.chunkKind === "imports"
-                    ? "imports-only"
-                    : "structural-only"),
+                  (activeDecision!.action === "lexical"
+                    ? activeDecision!.reason
+                    : doc.chunkKind === "imports"
+                      ? "imports-only"
+                      : "structural-only"),
           "Invalid managed embedding input or eligibility.",
         );
       } else if (doc.embeddingStatus === "embedded")
         invariant(
           b.preset.embedding &&
+            activeDecision!.action === "semantic" &&
+            !["imports", "structural"].includes(String(doc.chunkKind)) &&
             Array.isArray(doc.embedding) &&
             doc.embedding.length === b.preset.embedding.dimensions &&
             doc.embedding.every(Number.isFinite),

package/src/chunk.ts

@@ -14,12 +14,14 @@ export const CHUNKER = {
   grammars: "tree-sitter-wasms@0.1.13",
   sqlGrammar:
     "tree-sitter-wasm@2.0.2/sql/b77530893b1dd6d1d4814eacf34d25bcbe4a12ec9a25a8c45b320d447265f42b",
+  shellGrammar:
+    "tree-sitter-wasm@2.0.2/bash/5daf3f2ac1cea01a64cfd6878ef7d247310925a4c62238eb4b8c1feb5dce355a",
   tokenizer: "js-tiktoken@1.0.21/cl100k_base",
   targetTokens: 800,
   maxTokens: 1500,
   fallbackOverlap: 0.1,
   enrichment: "path-scope-symbol-v1",
-  policy: "source-v1",
+  policy: "source-v2",
 } as const;
 export type Span = {
   startByte: number;
@@ -49,16 +51,14 @@ async function language(name: string): Promise<Language> {
   let l = languages.get(name);
   if (!l) {
     l = await Language.load(
-      name === "sql"
+      name === "sql" || name === "shell"
         ? fileURLToPath(
             new URL(
-              "../runtime/grammars/tree-sitter-sql.wasm",
+              `../runtime/grammars/tree-sitter-${name === "shell" ? "bash" : "sql"}.wasm`,
               import.meta.url,
             ),
           )
-        : require.resolve(
-            `tree-sitter-wasms/out/tree-sitter-${name === "shell" ? "bash" : name}.wasm`,
-          ),
+        : require.resolve(`tree-sitter-wasms/out/tree-sitter-${name}.wasm`),
     );
     languages.set(name, l);
   }

package/src/cli.ts

@@ -17,6 +17,7 @@ import {
   normalizeAnalyzers,
   ANALYZERS,
 } from "./build.js";
+import { normalizeFilePolicy } from "./file-policy.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
 import {
@@ -64,6 +65,13 @@ const analyzerOption = (description: string) =>
   ).argParser((value) =>
     normalizeAnalyzers(value.split(",").map((name) => name.trim())),
   );
+const filePolicyOption = () =>
+  new Option(
+    "--file-policy <json>",
+    "Path to file policy JSON; pinned at registration or local dry run",
+  ).argParser((path) =>
+    normalizeFilePolicy(JSON.parse(readFileSync(path, "utf8"))),
+  );
 async function connected() {
   const settings = await loadSettings();
   return { settings, remote: new LambdaRemote(settings) };
@@ -131,6 +139,7 @@ repo
   .option("--description <text>")
   .option("--tag <key=value>", "One optional Collection metadata tag")
   .addOption(analyzerOption("Comma-separated text analyzers"))
+  .addOption(filePolicyOption())
   .addOption(
     new Option(
       "--embedding <model>",
@@ -153,7 +162,7 @@ repo
         remote: o.remote,
         description: o.description,
         labels,
-        preset: presetFor(o.embedding, o.analyzers),
+        preset: presetFor(o.embedding, o.analyzers, o.filePolicy),
       }),
     );
   });
@@ -181,6 +190,7 @@ cli
   .option("--repo <name>", "Registered remote repository")
   .option("--ref <ref>", "Local branch, Git tag, or commit OID")
   .option("--dry-run", "Only materialize and validate a local build")
+  .addOption(filePolicyOption())
   .addOption(
     new Option(
       "--embedding <model>",
@@ -212,6 +222,10 @@ cli
       !o.analyzers || (o.dryRun && o.path),
       "--analyzers is only accepted with --dry-run --path; connected imports use the repository preset.",
     );
+    invariant(
+      !o.filePolicy || (o.dryRun && o.path),
+      "--file-policy is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
     invariant(!(o.path && o.repo), "Choose --path or --repo.");
     invariant(
       !o.previous || o.dryRun,
@@ -261,7 +275,7 @@ cli
         await validateBuild(b);
       } else {
         let source;
-        let preset = presetFor(o.embedding, o.analyzers);
+        let preset = presetFor(o.embedding, o.analyzers, o.filePolicy);
         if (o.path) source = await identity(o.path, o.remote);
         else {
           invariant(o.repo, "Use --path for a credential-free dry run.");
@@ -289,6 +303,8 @@ cli
           path: e.path,
           status: e.status,
           reason: e.reason,
+          retrieval: e.retrieval,
+          policyReason: e.policyReason,
           parseStatus: e.parseStatus,
         })),
         uploaded: false,

package/src/file-policy.ts

@@ -0,0 +1,157 @@
+import { invariant } from "./common.js";
+
+export type FileAction = "exclude" | "lexical" | "semantic";
+export type FileRule = { pattern: string; action: FileAction };
+export type FilePolicy = { version: 1; rules: FileRule[] };
+export const FILE_POLICY: FilePolicy = { version: 1, rules: [] };
+export type FileDecision = { action: FileAction; reason: string };
+
+/** Deliberately small glob syntax: repo-relative paths, *, ?, and whole-segment **. */
+function glob(pattern: string): RegExp {
+  let expression = "^";
+  const parts = pattern.split("/");
+  for (const [index, part] of parts.entries()) {
+    if (part === "**")
+      expression += index === parts.length - 1 ? "[\\s\\S]*" : "(?:[^/]+/)*";
+    else {
+      for (const char of part)
+        expression +=
+          char === "*"
+            ? "[^/]*"
+            : char === "?"
+              ? "[^/]"
+              : char.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      if (index !== parts.length - 1) expression += "/";
+    }
+  }
+  return new RegExp(expression + "$", "u");
+}
+
+export function normalizeFilePolicy(value: unknown): FilePolicy {
+  invariant(
+    value !== null && typeof value === "object" && !Array.isArray(value),
+    "File policy must be an object.",
+  );
+  const policy = value as FilePolicy;
+  invariant(
+    Object.keys(policy).sort().join(",") === "rules,version" &&
+      policy.version === 1 &&
+      Array.isArray(policy.rules) &&
+      policy.rules.length <= 100,
+    "File policy requires version: 1 and at most 100 rules.",
+  );
+  const rules = policy.rules.map((rule) => {
+    invariant(
+      rule &&
+        typeof rule === "object" &&
+        Object.keys(rule).sort().join(",") === "action,pattern" &&
+        ["exclude", "lexical", "semantic"].includes(rule.action),
+      "Each file rule requires pattern and action (exclude, lexical, semantic).",
+    );
+    invariant(
+      typeof rule.pattern === "string" &&
+        rule.pattern.length > 0 &&
+        rule.pattern.length <= 512 &&
+        !/[\\\x00-\x1f\x7f\[\]{}!]/.test(rule.pattern) &&
+        rule.pattern
+          .split("/")
+          .every(
+            (part) =>
+              part &&
+              part !== "." &&
+              part !== ".." &&
+              (!part.includes("**") || part === "**"),
+          ),
+      "File policy patterns must be relative paths using only *, ?, and whole-segment ** wildcards.",
+    );
+    glob(rule.pattern);
+    return { pattern: rule.pattern, action: rule.action };
+  });
+  return { version: 1, rules };
+}
+
+/** These bodies cannot be re-enabled by repository rules. This is not a secret scanner. */
+export function hardExclusion(path: string): string | undefined {
+  const name = path.split("/").at(-1)!;
+  if (
+    (/^\.env(?:\.|$)/i.test(name) &&
+      !/\.(example|sample|template)$/i.test(name)) ||
+    /^(\.npmrc|\.pypirc|\.netrc|_netrc|id_rsa|id_dsa|id_ecdsa|id_ed25519)$/i.test(
+      name,
+    ) ||
+    /(^|\/)\.(aws|ssh)(\/|$)/i.test(path)
+  )
+    return "credential-file";
+  if (
+    /\.(pem|key|crt|cer|csr|der|p12|pfx|p7b|p7c|jks|keystore|srl)$/i.test(path)
+  )
+    return "crypto-material";
+  if (
+    /\.(svgz?|png|apng|jpe?g|jfif|gif|webp|avif|bmp|dib|tiff?|ico|icns|heic|heif|jxl|psd|ai|eps|pnm|pbm|pgm|ppm|xbm|xpm)$/i.test(
+      path,
+    )
+  )
+    return "image-extension";
+  if (
+    /\.(pdf|zip|gz|bz2|xz|7z|tar|rar|jar|class|woff2?|ttf|otf|mp[34]|wav|ogg|mov|exe|dll|so|dylib|wasm|pyc|pyo|o|a|sqlite3?|db)$/i.test(
+      path,
+    )
+  )
+    return "binary-extension";
+  return undefined;
+}
+
+export function privateKeyContent(source: string): boolean {
+  return /(?:^|\r?\n)-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY-----\r?\n/.test(
+    source,
+  );
+}
+
+/** Explicit rules override defaults, in order; the final matching rule wins. */
+export function fileDecision(
+  path: string,
+  policy: FilePolicy,
+  source?: string,
+): FileDecision {
+  let decision: FileDecision = { action: "semantic", reason: "source" };
+  if (
+    /(^|\/)(node_modules|vendor|vendored|third_party|third-party|dist|build|target|coverage|\.git|\.next|\.nuxt|\.venv|venv|__pycache__|\.cache|\.pytest_cache|\.mypy_cache|\.ruff_cache|\.tox|\.nox|\.srcx)(\/|$)/i.test(
+      path,
+    )
+  )
+    decision = { action: "exclude", reason: "dependency-or-build-output" };
+  else if (/\.(min\.(js|css)|map)$/i.test(path))
+    decision = { action: "exclude", reason: "generated-output" };
+  else if (/\.(log|log\.\d+)$/i.test(path))
+    decision = { action: "exclude", reason: "log-output" };
+  else if (
+    /(^|\/)(package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?|go\.sum|[^/]+\.lock|packages\.lock\.json|Package\.resolved)$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "lockfile" };
+  else if (
+    /(^|\/)(licen[sc]e|copying|copyright|notice|authors|contributors)([.-][^/]*)?$/i.test(
+      path,
+    )
+  )
+    decision = { action: "lexical", reason: "legal-or-attribution" };
+  else if (
+    /\.(csv|tsv|jsonl|ndjson|snap)$/i.test(path) ||
+    /(^|\/)__snapshots__\//i.test(path)
+  )
+    decision = { action: "lexical", reason: "data-or-snapshot" };
+  else if (
+    /\.(pb\.go|generated\.[^/]+|g\.[^/]+)$/i.test(path) ||
+    /_pb2(?:_grpc)?\.py$/i.test(path) ||
+    (source !== undefined &&
+      /(?:^|\n)[ \t]*(?:\/\/|#|\/\*|\*|<!--)[^\r\n]*(?:code generated[^\r\n]*do not edit|(?:auto[- ]?generated|generated by)[^\r\n]*do not edit|do not edit[^\r\n]*generated|@generated(?:[ \t]|$))/i.test(
+        Buffer.from(source, "utf8").subarray(0, 8192).toString("utf8"),
+      ))
+  )
+    decision = { action: "lexical", reason: "generated-source" };
+  for (const rule of policy.rules)
+    if (glob(rule.pattern).test(path))
+      decision = { action: rule.action, reason: `file-rule:${rule.action}` };
+  return decision;
+}