@functional-systems/srcx 0.1.0-dev.4 → 0.1.0-dev.40

package/src/build.ts

@@ -1,9 +1,14 @@
-import { mkdir, open, rm } from "node:fs/promises";
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
+import { mkdir, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { Blobs, inventory, resolveCommit, type Identity } from "./git.js";
 import { chunk, CHUNKER, tokens } from "./chunk.js";
+import {
+  ArtifactWriter,
+  artifactRecords,
+  validateStorage,
+  type ArtifactStorage,
+  type RecordPart,
+} from "./artifact.js";
 import {
   atomic,
   canonical,
@@ -15,6 +20,26 @@ import {
   type Doc,
   optionalJson,
 } from "./common.js";
+import {
+  FILE_POLICY,
+  normalizeFilePolicy,
+  hardExclusion,
+  privateKeyContent,
+  fileDecision,
+  type FilePolicy,
+  type FileDecision,
+} from "./file-policy.js";
+export const ANALYZERS = ["english", "japanese", "korean", "standard"] as const;
+export type Analyzer = (typeof ANALYZERS)[number];
+export function normalizeAnalyzers(value: unknown): Analyzer[] {
+  invariant(
+    Array.isArray(value) &&
+      value.length > 0 &&
+      value.every((name) => ANALYZERS.includes(name)),
+    `Analyzers must be a nonempty list of: ${ANALYZERS.join(", ")}.`,
+  );
+  return [...new Set(value as Analyzer[])].sort();
+}
 export const INDEX_CONFIGS = {
   kind: { type: "keyword" },
   path: { type: "keyword" },
@@ -25,18 +50,123 @@ export const INDEX_CONFIGS = {
 } as const;
 export type Preset = {
   schemaVersion: 1;
+  analyzers: readonly Analyzer[];
   chunker: typeof CHUNKER;
   mode: "syntax" | "window";
+  /** Internal evaluation override; the CLI keeps its original enrichment. */
+  enrichment?: "path-only-v1";
   maxFileBytes: number;
-  embedding: { model: string; dimensions: number } | null;
+  filePolicy: FilePolicy;
+  embedding: {
+    model: string;
+    dimensions: number;
+    managed?: true;
+    provider?: "openai";
+    similarity?: "cosine";
+    sourceField?: "embeddingText";
+  } | null;
 };
 export const PRESET: Preset = {
   schemaVersion: 1,
+  analyzers: ["standard"],
   chunker: CHUNKER,
   mode: "syntax",
   maxFileBytes: 1024 * 1024,
+  filePolicy: FILE_POLICY,
   embedding: null,
 };
+/** Embedding variants share the same chunking behavior. */
+export const MANAGED_PRESET: Preset = {
+  ...PRESET,
+  embedding: {
+    managed: true,
+    provider: "openai",
+    model: "text-embedding-3-small",
+    dimensions: 1536,
+    similarity: "cosine",
+    sourceField: "embeddingText",
+  },
+};
+export const MANAGED_LARGE_PRESET: Preset = {
+  ...MANAGED_PRESET,
+  embedding: {
+    ...MANAGED_PRESET.embedding!,
+    model: "text-embedding-3-large",
+    dimensions: 3072,
+  },
+};
+export function presetFor(
+  embedding = "none",
+  analyzers: readonly string[] = PRESET.analyzers,
+  filePolicy: unknown = FILE_POLICY,
+): Preset {
+  invariant(
+    ["none", "text-embedding-3-small", "text-embedding-3-large"].includes(
+      embedding,
+    ),
+    "Embedding must be none, text-embedding-3-small or text-embedding-3-large.",
+  );
+  const base =
+    embedding === "none"
+      ? PRESET
+      : embedding === "text-embedding-3-large"
+        ? MANAGED_LARGE_PRESET
+        : MANAGED_PRESET;
+  const normalized = normalizeAnalyzers(analyzers);
+  const configured = {
+    ...base,
+    analyzers: normalized,
+    filePolicy: normalizeFilePolicy(filePolicy),
+  };
+  return hash(configured) === hash(base) ? base : configured;
+}
+export function supportedPreset(preset: unknown): preset is Preset {
+  if (!preset || typeof preset !== "object" || Array.isArray(preset))
+    return false;
+  try {
+    const analyzers = normalizeAnalyzers((preset as Preset).analyzers);
+    const filePolicy = normalizeFilePolicy((preset as Preset).filePolicy);
+    return [PRESET, MANAGED_PRESET, MANAGED_LARGE_PRESET].some(
+      (base) => hash({ ...base, analyzers, filePolicy }) === hash(preset),
+    );
+  } catch {
+    return false;
+  }
+}
+export function indexConfigs(preset: Preset = PRESET) {
+  const text = { type: "text" as const, analyzers: [...preset.analyzers] };
+  const indexes = { ...INDEX_CONFIGS, searchText: text };
+  if (!preset.embedding?.managed) return indexes;
+  const { managed: _, ...embedding } = preset.embedding;
+  return {
+    ...indexes,
+    embeddingText: { ...text, analyzers: [...preset.analyzers] },
+    embedding: { type: "vector", managedEmbedding: true, embedding },
+  };
+}
+/** Validate server enrichment separately; all other payload fields remain exact. */
+export function recordHash(doc: Doc, preset: Preset): string {
+  if (!preset.embedding?.managed) return hash(doc);
+  const { embedding, ...source } = doc;
+  if (doc.kind === "chunk" && doc.embeddingStatus === "managed") {
+    invariant(
+      typeof doc.embeddingText === "string" &&
+        doc.embeddingText === doc.searchText &&
+        doc.embeddingInputHash ===
+          hash([preset.embedding, doc.embeddingText]) &&
+        Array.isArray(embedding) &&
+        embedding.length === preset.embedding.dimensions &&
+        embedding.every((v) => typeof v === "number" && Number.isFinite(v)) &&
+        embedding.some((v) => v !== 0),
+      "Missing or invalid managed embedding.",
+    );
+  } else
+    invariant(
+      embedding === undefined && doc.embeddingText === undefined,
+      "Unexpected managed embedding on an ineligible record.",
+    );
+  return hash(source);
+}
 export type InventoryItem = {
   path: string;
   pathBase64: string;
@@ -45,6 +175,8 @@ export type InventoryItem = {
   bytes: number;
   status: "included" | "excluded";
   reason?: string;
+  retrieval?: "lexical" | "semantic";
+  policyReason?: string;
   targetBase64?: string;
   fileId?: string;
   chunkIds?: string[];
@@ -52,10 +184,13 @@ export type InventoryItem = {
   parseStatus?: string;
   tokens?: number;
   embedded?: number;
+  managed?: number;
+  managedTokens?: number;
   skipped?: number;
 };
 export type Build = {
-  format: 1;
+  format: 1 | 2;
+  recordParts?: RecordPart[];
   directory: string;
   repoKey: string;
   configHash: string;
@@ -83,58 +218,46 @@ export type Embedder = {
   dimensions: number;
   embed: (text: string) => Promise<number[]>;
 };
-export async function* records(directory: string): AsyncGenerator<Doc> {
-  const input = createReadStream(join(directory, "records.jsonl"), {
-    encoding: "utf8",
-  });
-  const lines = createInterface({ input, crlfDelay: Infinity });
-  try {
-    for await (const line of lines) if (line) yield JSON.parse(line) as Doc;
-  } finally {
-    lines.close();
-    input.destroy();
-  }
+export async function* records(build: Build | string): AsyncGenerator<Doc> {
+  const b = typeof build === "string" ? await loadBuild(build) : build;
+  yield* artifactRecords(b);
 }
 export async function loadBuild(directory: string): Promise<Build> {
   const b = await json<Build>(join(directory, "build.json"));
   invariant(
-    b.format === 1 && b.configHash === hash(b.preset),
+    (b.format === 1 || b.format === 2) && b.configHash === hash(b.preset),
     "Invalid build configuration.",
   );
   b.directory = resolve(directory);
+  validateStorage(b);
   return b;
 }
-function exclude(path: string): string | undefined {
-  if (
-    /(^|\/)(node_modules|vendor|vendored|dist|build|target|coverage|\.git|\.next)(\/|$)/.test(
-      path,
-    )
-  )
-    return "dependency-or-build-output";
-  if (
-    /\.(png|jpe?g|gif|ico|pdf|zip|gz|jar|class|woff2?|mp[34]|exe|dll|so|dylib|wasm)$/i.test(
-      path,
-    )
-  )
-    return "binary-extension";
-  if (/\.(min\.(js|css)|map)$/.test(path)) return "generated-output";
-  return undefined;
-}
 export async function materialize(args: {
   identity: Identity;
   ref: string;
+  resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
   output: string;
   previous?: Build;
+  storage?: ArtifactStorage;
   preset?: Preset;
   embedder?: Embedder;
   cache?: string;
 }): Promise<Build> {
+  const storage = args.storage ?? "jsonl";
+  invariant(
+    ["jsonl", "segments"].includes(storage),
+    "Artifact storage must be jsonl or segments.",
+  );
   const preset = args.preset ?? PRESET,
     configHash = hash(preset),
     prev = args.previous;
   invariant(
-    hash(preset.chunker) === hash(CHUNKER) &&
+    hash(preset.filePolicy) === hash(normalizeFilePolicy(preset.filePolicy)) &&
+      hash(preset.analyzers) === hash(normalizeAnalyzers(preset.analyzers)) &&
+      hash(preset.chunker) === hash(CHUNKER) &&
       ["syntax", "window"].includes(preset.mode) &&
+      (preset.enrichment === undefined ||
+        preset.enrichment === "path-only-v1") &&
       preset.maxFileBytes === PRESET.maxFileBytes,
     "Unsupported build preset.",
   );
@@ -147,26 +270,25 @@ export async function materialize(args: {
   }
   invariant(
     !preset.embedding ||
+      (preset.embedding.managed && supportedPreset(preset)) ||
       (args.embedder?.model === preset.embedding.model &&
         args.embedder.dimensions === preset.embedding.dimensions),
     "Embedding provider must match the pinned preset.",
   );
-  const commit = await resolveCommit(args.identity.path, args.ref);
+  const commit =
+    args.resolvedCommit ?? (await resolveCommit(args.identity.path, args.ref));
   const directory = resolve(args.output);
   await mkdir(directory, { recursive: false, mode: 0o700 });
-  const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
-  const recordHashes: Record<string, string> = {};
-  let writeError = false;
-  async function emit(doc: Doc) {
-    const data = canonical(doc);
-    invariant(
-      Buffer.byteLength(data) <= 4_800_000,
-      `Serialized document exceeds safe limit: ${doc.id}`,
-    );
-    invariant(!recordHashes[doc.id], "Duplicate document ID.");
-    recordHashes[doc.id] = hash(doc);
-    await output.write(data + "\n");
+  let writer: ArtifactWriter;
+  try {
+    writer = await ArtifactWriter.create(directory, storage);
+  } catch (error) {
+    await rm(directory, { recursive: true, force: true });
+    throw error;
   }
+  const recordHashes = writer.recordHashes;
+  let writeError = false;
+  const emit = (doc: Doc) => writer.emit(doc);
   const items: InventoryItem[] = [];
   const blobs = new Blobs(args.identity.path);
   const old = new Map(prev?.inventory.map((e) => [e.pathBase64, e]) ?? []);
@@ -200,7 +322,10 @@ export async function materialize(args: {
         item.targetBase64 = (await blobs.read(e)).toString("base64");
         continue;
       }
-      const reason = exclude(e.path);
+      const pathDecision = fileDecision(e.path, preset.filePolicy);
+      const reason =
+        hardExclusion(e.path) ??
+        (pathDecision.action === "exclude" ? pathDecision.reason : undefined);
       if (reason) {
         item.reason = reason;
         continue;
@@ -233,7 +358,17 @@ export async function materialize(args: {
         item.reason = "lfs-pointer";
         continue;
       }
-      const parsed = await chunk(source, e.path, preset.mode);
+      if (privateKeyContent(source)) {
+        item.reason = "private-key-content";
+        continue;
+      }
+      const decision = fileDecision(e.path, preset.filePolicy, source);
+      const parsed = await chunk(
+        source,
+        e.path,
+        preset.mode,
+        preset.enrichment,
+      );
       const contentHash = hash(data);
       const fileId = `f-${hash([e.pathBase64, contentHash])}`;
       const base = { schemaVersion: 1, configHash };
@@ -256,6 +391,8 @@ export async function materialize(args: {
       }
       Object.assign(item, {
         status: "included",
+        retrieval: decision.action,
+        policyReason: decision.reason,
         contentHash,
         fileId,
         chunkIds: [],
@@ -282,39 +419,52 @@ export async function materialize(args: {
             ? "structural-only"
             : "embedding-disabled",
         };
-        const eligible = !["imports", "structural"].includes(span.chunkKind);
+        const eligible =
+          decision.action === "semantic" &&
+          !["imports", "structural"].includes(span.chunkKind);
         if (preset.embedding && eligible) {
           const inputHash = hash([preset.embedding, span.searchText]);
-          const cachePath = args.cache
-            ? join(args.cache, inputHash + ".json")
-            : undefined;
-          let vector = cachePath
-            ? await optionalJson<number[]>(cachePath)
-            : undefined;
-          if (!vector) {
-            vector = await args.embedder!.embed(span.searchText);
+          if (preset.embedding.managed) {
+            doc.embeddingText = span.searchText;
+            doc.embeddingInputHash = inputHash;
+            doc.embeddingStatus = "managed";
+            delete doc.embeddingSkipReason;
+            item.managed = (item.managed ?? 0) + 1;
+            item.managedTokens = (item.managedTokens ?? 0) + span.tokenCount;
+          } else {
+            const cachePath = args.cache
+              ? join(args.cache, inputHash + ".json")
+              : undefined;
+            let vector = cachePath
+              ? await optionalJson<number[]>(cachePath)
+              : undefined;
+            if (!vector) {
+              vector = await args.embedder!.embed(span.searchText);
+              invariant(
+                vector.length === preset.embedding.dimensions &&
+                  vector.every(Number.isFinite),
+                "Invalid embedding output.",
+              );
+              if (cachePath) await atomic(cachePath, vector);
+            }
             invariant(
               vector.length === preset.embedding.dimensions &&
                 vector.every(Number.isFinite),
-              "Invalid embedding output.",
+              "Invalid cached vector.",
             );
-            if (cachePath) await atomic(cachePath, vector);
+            doc.embedding = vector;
+            doc.embeddingInputHash = inputHash;
+            doc.embeddingStatus = "embedded";
+            delete doc.embeddingSkipReason;
+            item.embedded!++;
           }
-          invariant(
-            vector.length === preset.embedding.dimensions &&
-              vector.every(Number.isFinite),
-            "Invalid cached vector.",
-          );
-          doc.embedding = vector;
-          doc.embeddingInputHash = inputHash;
-          doc.embeddingStatus = "embedded";
-          delete doc.embeddingSkipReason;
-          item.embedded!++;
         } else {
           doc.embeddingSkipReason = preset.embedding
-            ? span.chunkKind === "imports"
-              ? "imports-only"
-              : "structural-only"
+            ? decision.action === "lexical"
+              ? decision.reason
+              : span.chunkKind === "imports"
+                ? "imports-only"
+                : "structural-only"
             : "embedding-disabled";
           item.skipped!++;
         }
@@ -323,20 +473,34 @@ export async function materialize(args: {
         await emit(doc);
       }
     }
-    if (prev && reuse.size)
-      for await (const doc of records(prev.directory))
-        if (reuse.has(doc.id)) {
-          await emit(doc);
-          reuse.delete(doc.id);
+    if (prev && reuse.size) {
+      if (storage === "segments" && prev.format === 2) {
+        const files = new Map(prev.inventory.map((e) => [e.fileId, e]));
+        for (const part of prev.recordParts!) {
+          if (!reuse.has(part.fileId)) continue;
+          const ids = [part.fileId, ...files.get(part.fileId)!.chunkIds!];
+          await writer.retain(prev, part, ids);
+          for (const id of ids) reuse.delete(id);
         }
+      } else
+        for await (const doc of records(prev))
+          if (reuse.has(doc.id)) {
+            await emit(doc);
+            reuse.delete(doc.id);
+          }
+    }
     invariant(reuse.size === 0, "Previous artifact lacks retained records.");
+    await writer.finish();
   } catch (e) {
     writeError = true;
     throw e;
   } finally {
     blobs.close();
-    await output.close();
-    if (writeError) await rm(directory, { recursive: true, force: true });
+    try {
+      await writer.close();
+    } finally {
+      if (writeError) await rm(directory, { recursive: true, force: true });
+    }
   }
   const changes: Build["changes"] = {
     added: [],
@@ -358,7 +522,8 @@ export async function materialize(args: {
   for (const e of old.values())
     if (!now.has(e.pathBase64)) changes.deleted.push(e.path);
   const build: Build = {
-    format: 1,
+    format: storage === "segments" ? 2 : 1,
+    ...(storage === "segments" ? { recordParts: writer.parts } : {}),
     directory,
     repoKey: args.identity.key,
     configHash,
@@ -388,6 +553,15 @@ export async function materialize(args: {
       excluded: items.filter((e) => e.status === "excluded").length,
       chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
       tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
+      ...(preset.embedding?.managed
+        ? {
+            managed: items.reduce((n, e) => n + (e.managed ?? 0), 0),
+            managedTokens: items.reduce(
+              (n, e) => n + (e.managedTokens ?? 0),
+              0,
+            ),
+          }
+        : {}),
       embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
       skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
     },
@@ -405,9 +579,15 @@ export async function validateBuild(b: Build): Promise<void> {
     "Build commit identity mismatch; rebuild from the intended Git commit.",
   );
   invariant(
-    hash(b.preset) === b.configHash && hash(b.inventory) === b.inventoryHash,
+    hash(b.preset.filePolicy) ===
+      hash(normalizeFilePolicy(b.preset.filePolicy)) &&
+      hash(b.preset.analyzers) ===
+        hash(normalizeAnalyzers(b.preset.analyzers)) &&
+      hash(b.preset) === b.configHash &&
+      hash(b.inventory) === b.inventoryHash,
     "Build metadata hash mismatch.",
   );
+  validateStorage(b);
   const seen = new Set<string>();
   let activeFile: Doc | undefined;
   let end = 0,
@@ -417,6 +597,7 @@ export async function validateBuild(b: Build): Promise<void> {
       .filter((e) => e.status === "included")
       .map((e) => [e.fileId!, e]),
   );
+  let activeDecision: FileDecision | undefined;
   const completedFiles = new Set<string>();
   const finish = () => {
     if (activeFile) {
@@ -429,7 +610,7 @@ export async function validateBuild(b: Build): Promise<void> {
       completedFiles.add(activeFile.id);
     }
   };
-  for await (const doc of records(b.directory)) {
+  for await (const doc of records(b)) {
     invariant(
       !seen.has(doc.id) && b.recordHashes[doc.id] === hash(doc),
       "Corrupted or unexpected artifact record.",
@@ -439,12 +620,36 @@ export async function validateBuild(b: Build): Promise<void> {
       doc.configHash === b.configHash && doc.schemaVersion === 1,
       "Mixed configuration.",
     );
+    if (b.preset.embedding?.managed) {
+      invariant(
+        doc.embedding === undefined,
+        "Managed artifacts must not contain vectors.",
+      );
+      if (doc.kind !== "chunk")
+        invariant(
+          doc.embeddingText === undefined,
+          "Only eligible chunks can contain embedding input.",
+        );
+    }
     if (doc.kind === "file") {
       finish();
       activeFile = doc;
       end = 0;
       count = 0;
       const entry = expectedFile.get(doc.id);
+      activeDecision = fileDecision(
+        doc.path as string,
+        b.preset.filePolicy,
+        doc.sourceText as string,
+      );
+      invariant(
+        !hardExclusion(doc.path as string) &&
+          !privateKeyContent(doc.sourceText as string) &&
+          activeDecision.action !== "exclude" &&
+          entry?.retrieval === activeDecision.action &&
+          entry?.policyReason === activeDecision.reason,
+        "File policy or inventory decision mismatch.",
+      );
       const raw = Buffer.from(doc.sourceText as string);
       invariant(
         entry &&
@@ -494,9 +699,33 @@ export async function validateBuild(b: Build): Promise<void> {
           Number(doc.tokenCount) <= CHUNKER.maxTokens,
         "Invalid chunk source/position/token binding.",
       );
-      if (doc.embeddingStatus === "embedded")
+      if (b.preset.embedding?.managed) {
+        const eligible =
+          activeDecision!.action === "semantic" &&
+          !["imports", "structural"].includes(String(doc.chunkKind));
+        invariant(
+          eligible
+            ? doc.embeddingStatus === "managed" &&
+                doc.embeddingText === doc.searchText &&
+                doc.embeddingInputHash ===
+                  hash([b.preset.embedding, doc.searchText]) &&
+                doc.embeddingSkipReason === undefined
+            : doc.embeddingStatus === "skipped" &&
+                doc.embeddingText === undefined &&
+                doc.embeddingInputHash === undefined &&
+                doc.embeddingSkipReason ===
+                  (activeDecision!.action === "lexical"
+                    ? activeDecision!.reason
+                    : doc.chunkKind === "imports"
+                      ? "imports-only"
+                      : "structural-only"),
+          "Invalid managed embedding input or eligibility.",
+        );
+      } else if (doc.embeddingStatus === "embedded")
         invariant(
           b.preset.embedding &&
+            activeDecision!.action === "semantic" &&
+            !["imports", "structural"].includes(String(doc.chunkKind)) &&
             Array.isArray(doc.embedding) &&
             doc.embedding.length === b.preset.embedding.dimensions &&
             doc.embedding.every(Number.isFinite),