@functional-systems/srcx 0.1.0-dev.3 → 0.1.0-dev.31

package/src/build.ts

@@ -15,6 +15,26 @@ import {
   type Doc,
   optionalJson,
 } from "./common.js";
+import {
+  FILE_POLICY,
+  normalizeFilePolicy,
+  hardExclusion,
+  privateKeyContent,
+  fileDecision,
+  type FilePolicy,
+  type FileDecision,
+} from "./file-policy.js";
+export const ANALYZERS = ["english", "japanese", "korean", "standard"] as const;
+export type Analyzer = (typeof ANALYZERS)[number];
+export function normalizeAnalyzers(value: unknown): Analyzer[] {
+  invariant(
+    Array.isArray(value) &&
+      value.length > 0 &&
+      value.every((name) => ANALYZERS.includes(name)),
+    `Analyzers must be a nonempty list of: ${ANALYZERS.join(", ")}.`,
+  );
+  return [...new Set(value as Analyzer[])].sort();
+}
 export const INDEX_CONFIGS = {
   kind: { type: "keyword" },
   path: { type: "keyword" },
@@ -25,18 +45,123 @@ export const INDEX_CONFIGS = {
 } as const;
 export type Preset = {
   schemaVersion: 1;
+  analyzers: readonly Analyzer[];
   chunker: typeof CHUNKER;
   mode: "syntax" | "window";
+  /** Internal evaluation override; the CLI keeps its original enrichment. */
+  enrichment?: "path-only-v1";
   maxFileBytes: number;
-  embedding: { model: string; dimensions: number } | null;
+  filePolicy: FilePolicy;
+  embedding: {
+    model: string;
+    dimensions: number;
+    managed?: true;
+    provider?: "openai";
+    similarity?: "cosine";
+    sourceField?: "embeddingText";
+  } | null;
 };
 export const PRESET: Preset = {
   schemaVersion: 1,
+  analyzers: ["standard"],
   chunker: CHUNKER,
   mode: "syntax",
   maxFileBytes: 1024 * 1024,
+  filePolicy: FILE_POLICY,
   embedding: null,
 };
+/** Embedding variants share the same chunking behavior. */
+export const MANAGED_PRESET: Preset = {
+  ...PRESET,
+  embedding: {
+    managed: true,
+    provider: "openai",
+    model: "text-embedding-3-small",
+    dimensions: 1536,
+    similarity: "cosine",
+    sourceField: "embeddingText",
+  },
+};
+export const MANAGED_LARGE_PRESET: Preset = {
+  ...MANAGED_PRESET,
+  embedding: {
+    ...MANAGED_PRESET.embedding!,
+    model: "text-embedding-3-large",
+    dimensions: 3072,
+  },
+};
+export function presetFor(
+  embedding = "none",
+  analyzers: readonly string[] = PRESET.analyzers,
+  filePolicy: unknown = FILE_POLICY,
+): Preset {
+  invariant(
+    ["none", "text-embedding-3-small", "text-embedding-3-large"].includes(
+      embedding,
+    ),
+    "Embedding must be none, text-embedding-3-small or text-embedding-3-large.",
+  );
+  const base =
+    embedding === "none"
+      ? PRESET
+      : embedding === "text-embedding-3-large"
+        ? MANAGED_LARGE_PRESET
+        : MANAGED_PRESET;
+  const normalized = normalizeAnalyzers(analyzers);
+  const configured = {
+    ...base,
+    analyzers: normalized,
+    filePolicy: normalizeFilePolicy(filePolicy),
+  };
+  return hash(configured) === hash(base) ? base : configured;
+}
+export function supportedPreset(preset: unknown): preset is Preset {
+  if (!preset || typeof preset !== "object" || Array.isArray(preset))
+    return false;
+  try {
+    const analyzers = normalizeAnalyzers((preset as Preset).analyzers);
+    const filePolicy = normalizeFilePolicy((preset as Preset).filePolicy);
+    return [PRESET, MANAGED_PRESET, MANAGED_LARGE_PRESET].some(
+      (base) => hash({ ...base, analyzers, filePolicy }) === hash(preset),
+    );
+  } catch {
+    return false;
+  }
+}
+export function indexConfigs(preset: Preset = PRESET) {
+  const text = { type: "text" as const, analyzers: [...preset.analyzers] };
+  const indexes = { ...INDEX_CONFIGS, searchText: text };
+  if (!preset.embedding?.managed) return indexes;
+  const { managed: _, ...embedding } = preset.embedding;
+  return {
+    ...indexes,
+    embeddingText: { ...text, analyzers: [...preset.analyzers] },
+    embedding: { type: "vector", managedEmbedding: true, embedding },
+  };
+}
+/** Validate server enrichment separately; all other payload fields remain exact. */
+export function recordHash(doc: Doc, preset: Preset): string {
+  if (!preset.embedding?.managed) return hash(doc);
+  const { embedding, ...source } = doc;
+  if (doc.kind === "chunk" && doc.embeddingStatus === "managed") {
+    invariant(
+      typeof doc.embeddingText === "string" &&
+        doc.embeddingText === doc.searchText &&
+        doc.embeddingInputHash ===
+          hash([preset.embedding, doc.embeddingText]) &&
+        Array.isArray(embedding) &&
+        embedding.length === preset.embedding.dimensions &&
+        embedding.every((v) => typeof v === "number" && Number.isFinite(v)) &&
+        embedding.some((v) => v !== 0),
+      "Missing or invalid managed embedding.",
+    );
+  } else
+    invariant(
+      embedding === undefined && doc.embeddingText === undefined,
+      "Unexpected managed embedding on an ineligible record.",
+    );
+  return hash(source);
+}
 export type InventoryItem = {
   path: string;
   pathBase64: string;
@@ -45,6 +170,8 @@ export type InventoryItem = {
   bytes: number;
   status: "included" | "excluded";
   reason?: string;
+  retrieval?: "lexical" | "semantic";
+  policyReason?: string;
   targetBase64?: string;
   fileId?: string;
   chunkIds?: string[];
@@ -52,6 +179,8 @@ export type InventoryItem = {
   parseStatus?: string;
   tokens?: number;
   embedded?: number;
+  managed?: number;
+  managedTokens?: number;
   skipped?: number;
 };
 export type Build = {
@@ -104,25 +233,10 @@ export async function loadBuild(directory: string): Promise<Build> {
   b.directory = resolve(directory);
   return b;
 }
-function exclude(path: string): string | undefined {
-  if (
-    /(^|\/)(node_modules|vendor|vendored|dist|build|target|coverage|\.git|\.next)(\/|$)/.test(
-      path,
-    )
-  )
-    return "dependency-or-build-output";
-  if (
-    /\.(png|jpe?g|gif|ico|pdf|zip|gz|jar|class|woff2?|mp[34]|exe|dll|so|dylib|wasm)$/i.test(
-      path,
-    )
-  )
-    return "binary-extension";
-  if (/\.(min\.(js|css)|map)$/.test(path)) return "generated-output";
-  return undefined;
-}
 export async function materialize(args: {
   identity: Identity;
   ref: string;
+  resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
   output: string;
   previous?: Build;
   preset?: Preset;
@@ -133,8 +247,12 @@ export async function materialize(args: {
     configHash = hash(preset),
     prev = args.previous;
   invariant(
-    hash(preset.chunker) === hash(CHUNKER) &&
+    hash(preset.filePolicy) === hash(normalizeFilePolicy(preset.filePolicy)) &&
+      hash(preset.analyzers) === hash(normalizeAnalyzers(preset.analyzers)) &&
+      hash(preset.chunker) === hash(CHUNKER) &&
       ["syntax", "window"].includes(preset.mode) &&
+      (preset.enrichment === undefined ||
+        preset.enrichment === "path-only-v1") &&
       preset.maxFileBytes === PRESET.maxFileBytes,
     "Unsupported build preset.",
   );
@@ -147,11 +265,13 @@ export async function materialize(args: {
   }
   invariant(
     !preset.embedding ||
+      (preset.embedding.managed && supportedPreset(preset)) ||
       (args.embedder?.model === preset.embedding.model &&
         args.embedder.dimensions === preset.embedding.dimensions),
     "Embedding provider must match the pinned preset.",
   );
-  const commit = await resolveCommit(args.identity.path, args.ref);
+  const commit =
+    args.resolvedCommit ?? (await resolveCommit(args.identity.path, args.ref));
   const directory = resolve(args.output);
   await mkdir(directory, { recursive: false, mode: 0o700 });
   const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
@@ -200,7 +320,10 @@ export async function materialize(args: {
         item.targetBase64 = (await blobs.read(e)).toString("base64");
         continue;
       }
-      const reason = exclude(e.path);
+      const pathDecision = fileDecision(e.path, preset.filePolicy);
+      const reason =
+        hardExclusion(e.path) ??
+        (pathDecision.action === "exclude" ? pathDecision.reason : undefined);
       if (reason) {
         item.reason = reason;
         continue;
@@ -233,7 +356,17 @@ export async function materialize(args: {
         item.reason = "lfs-pointer";
         continue;
       }
-      const parsed = await chunk(source, e.path, preset.mode);
+      if (privateKeyContent(source)) {
+        item.reason = "private-key-content";
+        continue;
+      }
+      const decision = fileDecision(e.path, preset.filePolicy, source);
+      const parsed = await chunk(
+        source,
+        e.path,
+        preset.mode,
+        preset.enrichment,
+      );
       const contentHash = hash(data);
       const fileId = `f-${hash([e.pathBase64, contentHash])}`;
       const base = { schemaVersion: 1, configHash };
@@ -256,6 +389,8 @@ export async function materialize(args: {
       }
       Object.assign(item, {
         status: "included",
+        retrieval: decision.action,
+        policyReason: decision.reason,
         contentHash,
         fileId,
         chunkIds: [],
@@ -282,39 +417,52 @@ export async function materialize(args: {
             ? "structural-only"
             : "embedding-disabled",
         };
-        const eligible = !["imports", "structural"].includes(span.chunkKind);
+        const eligible =
+          decision.action === "semantic" &&
+          !["imports", "structural"].includes(span.chunkKind);
         if (preset.embedding && eligible) {
           const inputHash = hash([preset.embedding, span.searchText]);
-          const cachePath = args.cache
-            ? join(args.cache, inputHash + ".json")
-            : undefined;
-          let vector = cachePath
-            ? await optionalJson<number[]>(cachePath)
-            : undefined;
-          if (!vector) {
-            vector = await args.embedder!.embed(span.searchText);
+          if (preset.embedding.managed) {
+            doc.embeddingText = span.searchText;
+            doc.embeddingInputHash = inputHash;
+            doc.embeddingStatus = "managed";
+            delete doc.embeddingSkipReason;
+            item.managed = (item.managed ?? 0) + 1;
+            item.managedTokens = (item.managedTokens ?? 0) + span.tokenCount;
+          } else {
+            const cachePath = args.cache
+              ? join(args.cache, inputHash + ".json")
+              : undefined;
+            let vector = cachePath
+              ? await optionalJson<number[]>(cachePath)
+              : undefined;
+            if (!vector) {
+              vector = await args.embedder!.embed(span.searchText);
+              invariant(
+                vector.length === preset.embedding.dimensions &&
+                  vector.every(Number.isFinite),
+                "Invalid embedding output.",
+              );
+              if (cachePath) await atomic(cachePath, vector);
+            }
             invariant(
               vector.length === preset.embedding.dimensions &&
                 vector.every(Number.isFinite),
-              "Invalid embedding output.",
+              "Invalid cached vector.",
             );
-            if (cachePath) await atomic(cachePath, vector);
+            doc.embedding = vector;
+            doc.embeddingInputHash = inputHash;
+            doc.embeddingStatus = "embedded";
+            delete doc.embeddingSkipReason;
+            item.embedded!++;
           }
-          invariant(
-            vector.length === preset.embedding.dimensions &&
-              vector.every(Number.isFinite),
-            "Invalid cached vector.",
-          );
-          doc.embedding = vector;
-          doc.embeddingInputHash = inputHash;
-          doc.embeddingStatus = "embedded";
-          delete doc.embeddingSkipReason;
-          item.embedded!++;
         } else {
           doc.embeddingSkipReason = preset.embedding
-            ? span.chunkKind === "imports"
-              ? "imports-only"
-              : "structural-only"
+            ? decision.action === "lexical"
+              ? decision.reason
+              : span.chunkKind === "imports"
+                ? "imports-only"
+                : "structural-only"
             : "embedding-disabled";
           item.skipped!++;
         }
@@ -388,6 +536,15 @@ export async function materialize(args: {
       excluded: items.filter((e) => e.status === "excluded").length,
       chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
       tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
+      ...(preset.embedding?.managed
+        ? {
+            managed: items.reduce((n, e) => n + (e.managed ?? 0), 0),
+            managedTokens: items.reduce(
+              (n, e) => n + (e.managedTokens ?? 0),
+              0,
+            ),
+          }
+        : {}),
       embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
       skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
     },
@@ -405,7 +562,12 @@ export async function validateBuild(b: Build): Promise<void> {
     "Build commit identity mismatch; rebuild from the intended Git commit.",
   );
   invariant(
-    hash(b.preset) === b.configHash && hash(b.inventory) === b.inventoryHash,
+    hash(b.preset.filePolicy) ===
+      hash(normalizeFilePolicy(b.preset.filePolicy)) &&
+      hash(b.preset.analyzers) ===
+        hash(normalizeAnalyzers(b.preset.analyzers)) &&
+      hash(b.preset) === b.configHash &&
+      hash(b.inventory) === b.inventoryHash,
     "Build metadata hash mismatch.",
   );
   const seen = new Set<string>();
@@ -417,6 +579,7 @@ export async function validateBuild(b: Build): Promise<void> {
       .filter((e) => e.status === "included")
       .map((e) => [e.fileId!, e]),
   );
+  let activeDecision: FileDecision | undefined;
   const completedFiles = new Set<string>();
   const finish = () => {
     if (activeFile) {
@@ -439,12 +602,36 @@ export async function validateBuild(b: Build): Promise<void> {
       doc.configHash === b.configHash && doc.schemaVersion === 1,
       "Mixed configuration.",
     );
+    if (b.preset.embedding?.managed) {
+      invariant(
+        doc.embedding === undefined,
+        "Managed artifacts must not contain vectors.",
+      );
+      if (doc.kind !== "chunk")
+        invariant(
+          doc.embeddingText === undefined,
+          "Only eligible chunks can contain embedding input.",
+        );
+    }
     if (doc.kind === "file") {
       finish();
       activeFile = doc;
       end = 0;
       count = 0;
       const entry = expectedFile.get(doc.id);
+      activeDecision = fileDecision(
+        doc.path as string,
+        b.preset.filePolicy,
+        doc.sourceText as string,
+      );
+      invariant(
+        !hardExclusion(doc.path as string) &&
+          !privateKeyContent(doc.sourceText as string) &&
+          activeDecision.action !== "exclude" &&
+          entry?.retrieval === activeDecision.action &&
+          entry?.policyReason === activeDecision.reason,
+        "File policy or inventory decision mismatch.",
+      );
       const raw = Buffer.from(doc.sourceText as string);
       invariant(
         entry &&
@@ -494,9 +681,33 @@ export async function validateBuild(b: Build): Promise<void> {
           Number(doc.tokenCount) <= CHUNKER.maxTokens,
         "Invalid chunk source/position/token binding.",
       );
-      if (doc.embeddingStatus === "embedded")
+      if (b.preset.embedding?.managed) {
+        const eligible =
+          activeDecision!.action === "semantic" &&
+          !["imports", "structural"].includes(String(doc.chunkKind));
+        invariant(
+          eligible
+            ? doc.embeddingStatus === "managed" &&
+                doc.embeddingText === doc.searchText &&
+                doc.embeddingInputHash ===
+                  hash([b.preset.embedding, doc.searchText]) &&
+                doc.embeddingSkipReason === undefined
+            : doc.embeddingStatus === "skipped" &&
+                doc.embeddingText === undefined &&
+                doc.embeddingInputHash === undefined &&
+                doc.embeddingSkipReason ===
+                  (activeDecision!.action === "lexical"
+                    ? activeDecision!.reason
+                    : doc.chunkKind === "imports"
+                      ? "imports-only"
+                      : "structural-only"),
+          "Invalid managed embedding input or eligibility.",
+        );
+      } else if (doc.embeddingStatus === "embedded")
         invariant(
           b.preset.embedding &&
+            activeDecision!.action === "semantic" &&
+            !["imports", "structural"].includes(String(doc.chunkKind)) &&
             Array.isArray(doc.embedding) &&
             doc.embedding.length === b.preset.embedding.dimensions &&
             doc.embedding.every(Number.isFinite),