@functional-systems/srcx 0.1.0-dev.38 → 0.1.0-dev.39

package/src/artifact.ts

@@ -0,0 +1,303 @@
+import { constants, createReadStream } from "node:fs";
+import {
+  copyFile,
+  link,
+  lstat,
+  mkdir,
+  open,
+  realpath,
+  rename,
+  type FileHandle,
+} from "node:fs/promises";
+import { createHash, randomUUID, type Hash } from "node:crypto";
+import { createInterface } from "node:readline";
+import { join } from "node:path";
+import { canonical, hash, invariant, type Doc } from "./common.js";
+import type { Build } from "./build.js";
+
+export type ArtifactStorage = "jsonl" | "segments";
+export type RecordPart = {
+  fileId: string;
+  digest: string;
+  byteLength: number;
+  recordCount: number;
+};
+const bufferLimit = 64 * 1024;
+const documentLimit = 4_800_000;
+function requireSegmentSupport(): void {
+  invariant(
+    constants.O_NOFOLLOW && constants.O_NONBLOCK,
+    "Segmented artifacts require no-follow file-open support.",
+  );
+}
+const objectPath = (directory: string, digest: string) => {
+  invariant(/^[a-f0-9]{64}$/.test(digest), "Invalid artifact object digest.");
+  return join(directory, "objects", `${digest}.jsonl`);
+};
+
+export function validateStorage(b: Build): void {
+  invariant(b.format === 1 || b.format === 2, "Unsupported artifact format.");
+  if (b.format === 1) return;
+  const included = b.inventory.filter((e) => e.status === "included");
+  const files = new Map(included.map((e) => [e.fileId, e]));
+  invariant(
+    files.size === included.length &&
+      Array.isArray(b.recordParts) &&
+      b.recordParts.length === included.length,
+    "Invalid segmented artifact inventory.",
+  );
+  const seen = new Set<string>();
+  for (const part of b.recordParts) {
+    const entry = files.get(part?.fileId);
+    invariant(
+      part &&
+        typeof part.fileId === "string" &&
+        /^f-[a-f0-9]{64}$/.test(part.fileId) &&
+        entry &&
+        Array.isArray(entry.chunkIds) &&
+        !seen.has(part.fileId) &&
+        typeof part.digest === "string" &&
+        /^[a-f0-9]{64}$/.test(part.digest) &&
+        Number.isSafeInteger(part.byteLength) &&
+        part.byteLength > 0 &&
+        Number.isSafeInteger(part.recordCount) &&
+        part.recordCount === entry.chunkIds.length + 1,
+      "Invalid segmented artifact part.",
+    );
+    seen.add(part.fileId);
+  }
+}
+
+async function objectDirectory(directory: string): Promise<void> {
+  requireSegmentSupport();
+  const path = join(directory, "objects");
+  const info = await lstat(path);
+  invariant(
+    info.isDirectory() &&
+      !info.isSymbolicLink() &&
+      (await realpath(path)) === join(await realpath(directory), "objects"),
+    "Artifact objects must be a local regular directory.",
+  );
+}
+async function openObject(
+  directory: string,
+  part: RecordPart,
+): Promise<FileHandle> {
+  requireSegmentSupport();
+  const file = await open(
+    objectPath(directory, part.digest),
+    constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
+  );
+  try {
+    const info = await file.stat();
+    invariant(
+      info.isFile() && info.size === part.byteLength,
+      "Artifact object is not a regular file of the expected size.",
+    );
+    return file;
+  } catch (error) {
+    await file.close();
+    throw error;
+  }
+}
+
+/** Digest verification completes when the iterator is exhausted; validateBuild always exhausts it. */
+export async function* artifactRecords(b: Build): AsyncGenerator<Doc> {
+  if (b.format === 1) {
+    const input = createReadStream(join(b.directory, "records.jsonl"), {
+      encoding: "utf8",
+    });
+    const lines = createInterface({ input, crlfDelay: Infinity });
+    try {
+      for await (const line of lines) if (line) yield JSON.parse(line) as Doc;
+    } finally {
+      lines.close();
+      input.destroy();
+    }
+    return;
+  }
+  validateStorage(b);
+  await objectDirectory(b.directory);
+  const files = new Map(b.inventory.map((e) => [e.fileId, e]));
+  for (const part of b.recordParts!) {
+    const file = await openObject(b.directory, part);
+    const input = file.createReadStream({
+      autoClose: false,
+      highWaterMark: bufferLimit,
+    });
+    const digest = createHash("sha256"),
+      decoder = new TextDecoder("utf-8", { fatal: true });
+    let pending = "",
+      bytes = 0,
+      count = 0;
+    try {
+      for await (const chunk of input) {
+        const data = chunk as Buffer;
+        digest.update(data);
+        bytes += data.length;
+        pending += decoder.decode(data, { stream: true });
+        let newline: number;
+        while ((newline = pending.indexOf("\n")) >= 0) {
+          const line = pending.slice(0, newline);
+          pending = pending.slice(newline + 1);
+          invariant(
+            line.length && Buffer.byteLength(line) <= documentLimit,
+            "Invalid artifact record size.",
+          );
+          const doc = JSON.parse(line) as Doc;
+          const entry = files.get(part.fileId)!;
+          invariant(
+            count === 0
+              ? doc.kind === "file" && doc.id === part.fileId
+              : doc.kind === "chunk" &&
+                  doc.fileId === part.fileId &&
+                  doc.id === entry.chunkIds![count - 1],
+            "Artifact part membership mismatch.",
+          );
+          count++;
+          yield doc;
+        }
+        invariant(
+          Buffer.byteLength(pending) <= documentLimit,
+          "Artifact record exceeds safe document limit.",
+        );
+      }
+      pending += decoder.decode();
+      invariant(
+        !pending &&
+          bytes === part.byteLength &&
+          count === part.recordCount &&
+          digest.digest("hex") === part.digest,
+        "Artifact object digest/count mismatch.",
+      );
+    } finally {
+      input.destroy();
+      await file.close();
+    }
+  }
+}
+
+/** Owns only this build's output; retained objects are verified again by full build validation. */
+export class ArtifactWriter {
+  readonly recordHashes: Record<string, string> = {};
+  readonly parts: RecordPart[] = [];
+  private output?: FileHandle;
+  private pending: string[] = [];
+  private pendingBytes = 0;
+  private part?: {
+    fileId: string;
+    temp: string;
+    digest: Hash;
+    bytes: number;
+    count: number;
+  };
+  private constructor(
+    readonly directory: string,
+    readonly storage: ArtifactStorage,
+  ) {}
+  static async create(
+    directory: string,
+    storage: ArtifactStorage,
+  ): Promise<ArtifactWriter> {
+    const writer = new ArtifactWriter(directory, storage);
+    if (storage === "jsonl")
+      writer.output = await open(join(directory, "records.jsonl"), "wx", 0o600);
+    else {
+      requireSegmentSupport();
+      await mkdir(join(directory, "objects"), { mode: 0o700 });
+    }
+    return writer;
+  }
+  private async flush(): Promise<void> {
+    if (!this.pending.length) return;
+    await this.output!.write(this.pending.join(""));
+    this.pending = [];
+    this.pendingBytes = 0;
+  }
+  private async finishPart(): Promise<void> {
+    await this.flush();
+    if (!this.part) return;
+    await this.output!.close();
+    this.output = undefined;
+    const part = this.part;
+    const digest = part.digest.digest("hex");
+    // Output is private to this newly created artifact; no other writer shares it.
+    await rename(part.temp, objectPath(this.directory, digest));
+    this.parts.push({
+      fileId: part.fileId,
+      digest,
+      byteLength: part.bytes,
+      recordCount: part.count,
+    });
+    this.part = undefined;
+  }
+  async emit(doc: Doc): Promise<void> {
+    const data = canonical(doc),
+      bytes = Buffer.byteLength(data);
+    invariant(
+      bytes <= documentLimit,
+      `Serialized document exceeds safe limit: ${doc.id}`,
+    );
+    invariant(!this.recordHashes[doc.id], "Duplicate document ID.");
+    this.recordHashes[doc.id] = hash(data);
+    if (this.storage === "segments") {
+      if (doc.kind === "file") {
+        await this.finishPart();
+        const temp = join(this.directory, "objects", `${randomUUID()}.tmp`);
+        this.output = await open(temp, "wx", 0o600);
+        this.part = {
+          fileId: doc.id,
+          temp,
+          digest: createHash("sha256"),
+          bytes: 0,
+          count: 0,
+        };
+      }
+      invariant(this.part, "Orphan artifact chunk.");
+      this.part.digest.update(data + "\n");
+      this.part.bytes += bytes + 1;
+      this.part.count++;
+    }
+    if (this.pendingBytes + bytes + 1 > bufferLimit) await this.flush();
+    if (bytes + 1 >= bufferLimit) await this.output!.write(data + "\n");
+    else {
+      this.pending.push(data + "\n");
+      this.pendingBytes += bytes + 1;
+    }
+  }
+  async retain(
+    previous: Build,
+    part: RecordPart,
+    ids: readonly string[],
+  ): Promise<void> {
+    await this.finishPart();
+    await objectDirectory(previous.directory);
+    const file = await openObject(previous.directory, part);
+    await file.close();
+    const source = objectPath(previous.directory, part.digest),
+      target = objectPath(this.directory, part.digest);
+    try {
+      await link(source, target);
+    } catch (error) {
+      if (
+        !["EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"].includes(
+          (error as NodeJS.ErrnoException).code ?? "",
+        )
+      )
+        throw error;
+      await copyFile(source, target, constants.COPYFILE_EXCL);
+    }
+    for (const id of ids) {
+      invariant(!this.recordHashes[id], "Duplicate document ID.");
+      this.recordHashes[id] = previous.recordHashes[id]!;
+    }
+    this.parts.push({ ...part });
+  }
+  async finish(): Promise<void> {
+    await this.finishPart();
+  }
+  async close(): Promise<void> {
+    await this.output?.close();
+    this.output = undefined;
+  }
+}

package/src/build.ts

@@ -1,9 +1,14 @@
-import { mkdir, open, rm } from "node:fs/promises";
-import { createReadStream } from "node:fs";
-import { createInterface } from "node:readline";
+import { mkdir, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { Blobs, inventory, resolveCommit, type Identity } from "./git.js";
 import { chunk, CHUNKER, tokens } from "./chunk.js";
+import {
+  ArtifactWriter,
+  artifactRecords,
+  validateStorage,
+  type ArtifactStorage,
+  type RecordPart,
+} from "./artifact.js";
 import {
   atomic,
   canonical,
@@ -184,7 +189,8 @@ export type InventoryItem = {
   skipped?: number;
 };
 export type Build = {
-  format: 1;
+  format: 1 | 2;
+  recordParts?: RecordPart[];
   directory: string;
   repoKey: string;
   configHash: string;
@@ -212,25 +218,18 @@ export type Embedder = {
   dimensions: number;
   embed: (text: string) => Promise<number[]>;
 };
-export async function* records(directory: string): AsyncGenerator<Doc> {
-  const input = createReadStream(join(directory, "records.jsonl"), {
-    encoding: "utf8",
-  });
-  const lines = createInterface({ input, crlfDelay: Infinity });
-  try {
-    for await (const line of lines) if (line) yield JSON.parse(line) as Doc;
-  } finally {
-    lines.close();
-    input.destroy();
-  }
+export async function* records(build: Build | string): AsyncGenerator<Doc> {
+  const b = typeof build === "string" ? await loadBuild(build) : build;
+  yield* artifactRecords(b);
 }
 export async function loadBuild(directory: string): Promise<Build> {
   const b = await json<Build>(join(directory, "build.json"));
   invariant(
-    b.format === 1 && b.configHash === hash(b.preset),
+    (b.format === 1 || b.format === 2) && b.configHash === hash(b.preset),
     "Invalid build configuration.",
   );
   b.directory = resolve(directory);
+  validateStorage(b);
   return b;
 }
 export async function materialize(args: {
@@ -239,10 +238,16 @@ export async function materialize(args: {
   resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
   output: string;
   previous?: Build;
+  storage?: ArtifactStorage;
   preset?: Preset;
   embedder?: Embedder;
   cache?: string;
 }): Promise<Build> {
+  const storage = args.storage ?? "jsonl";
+  invariant(
+    ["jsonl", "segments"].includes(storage),
+    "Artifact storage must be jsonl or segments.",
+  );
   const preset = args.preset ?? PRESET,
     configHash = hash(preset),
     prev = args.previous;
@@ -274,35 +279,16 @@ export async function materialize(args: {
     args.resolvedCommit ?? (await resolveCommit(args.identity.path, args.ref));
   const directory = resolve(args.output);
   await mkdir(directory, { recursive: false, mode: 0o700 });
-  const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
-  const recordHashes: Record<string, string> = {};
-  let writeError = false;
-  const pending: string[] = [];
-  let pendingBytes = 0;
-  const writeBufferBytes = 64 * 1024;
-  async function flush() {
-    if (!pending.length) return;
-    await output.write(pending.join(""));
-    pending.length = 0;
-    pendingBytes = 0;
-  }
-  async function emit(doc: Doc) {
-    const data = canonical(doc);
-    const bytes = Buffer.byteLength(data);
-    invariant(
-      bytes <= 4_800_000,
-      `Serialized document exceeds safe limit: ${doc.id}`,
-    );
-    invariant(!recordHashes[doc.id], "Duplicate document ID.");
-    // Hash the same canonical bytes written, without serializing the record twice.
-    recordHashes[doc.id] = hash(data);
-    if (pendingBytes + bytes + 1 > writeBufferBytes) await flush();
-    if (bytes + 1 >= writeBufferBytes) await output.write(data + "\n");
-    else {
-      pending.push(data + "\n");
-      pendingBytes += bytes + 1;
-    }
+  let writer: ArtifactWriter;
+  try {
+    writer = await ArtifactWriter.create(directory, storage);
+  } catch (error) {
+    await rm(directory, { recursive: true, force: true });
+    throw error;
   }
+  const recordHashes = writer.recordHashes;
+  let writeError = false;
+  const emit = (doc: Doc) => writer.emit(doc);
   const items: InventoryItem[] = [];
   const blobs = new Blobs(args.identity.path);
   const old = new Map(prev?.inventory.map((e) => [e.pathBase64, e]) ?? []);
@@ -487,21 +473,34 @@ export async function materialize(args: {
         await emit(doc);
       }
     }
-    if (prev && reuse.size)
-      for await (const doc of records(prev.directory))
-        if (reuse.has(doc.id)) {
-          await emit(doc);
-          reuse.delete(doc.id);
+    if (prev && reuse.size) {
+      if (storage === "segments" && prev.format === 2) {
+        const files = new Map(prev.inventory.map((e) => [e.fileId, e]));
+        for (const part of prev.recordParts!) {
+          if (!reuse.has(part.fileId)) continue;
+          const ids = [part.fileId, ...files.get(part.fileId)!.chunkIds!];
+          await writer.retain(prev, part, ids);
+          for (const id of ids) reuse.delete(id);
         }
+      } else
+        for await (const doc of records(prev))
+          if (reuse.has(doc.id)) {
+            await emit(doc);
+            reuse.delete(doc.id);
+          }
+    }
     invariant(reuse.size === 0, "Previous artifact lacks retained records.");
-    await flush();
+    await writer.finish();
   } catch (e) {
     writeError = true;
     throw e;
   } finally {
     blobs.close();
-    await output.close();
-    if (writeError) await rm(directory, { recursive: true, force: true });
+    try {
+      await writer.close();
+    } finally {
+      if (writeError) await rm(directory, { recursive: true, force: true });
+    }
   }
   const changes: Build["changes"] = {
     added: [],
@@ -523,7 +522,8 @@ export async function materialize(args: {
   for (const e of old.values())
     if (!now.has(e.pathBase64)) changes.deleted.push(e.path);
   const build: Build = {
-    format: 1,
+    format: storage === "segments" ? 2 : 1,
+    ...(storage === "segments" ? { recordParts: writer.parts } : {}),
     directory,
     repoKey: args.identity.key,
     configHash,
@@ -587,6 +587,7 @@ export async function validateBuild(b: Build): Promise<void> {
       hash(b.inventory) === b.inventoryHash,
     "Build metadata hash mismatch.",
   );
+  validateStorage(b);
   const seen = new Set<string>();
   let activeFile: Doc | undefined;
   let end = 0,
@@ -609,7 +610,7 @@ export async function validateBuild(b: Build): Promise<void> {
       completedFiles.add(activeFile.id);
     }
   };
-  for await (const doc of records(b.directory)) {
+  for await (const doc of records(b)) {
     invariant(
       !seen.has(doc.id) && b.recordHashes[doc.id] === hash(doc),
       "Corrupted or unexpected artifact record.",

package/src/cli.ts

@@ -209,6 +209,14 @@ cli
     ),
   )
   .option("--output <directory>", "New artifact directory")
+  .addOption(
+    new Option(
+      "--artifact-storage <storage>",
+      "Experimental local artifact storage",
+    )
+      .choices(["jsonl", "segments"])
+      .default("jsonl"),
+  )
   .option(
     "--previous <directory>",
     "Previous build artifact for a dry-run diff",
@@ -219,7 +227,7 @@ cli
     "Resume the same pending artifact after inspecting its journal",
   )
   .option("--timeout <seconds>", "Validation wait deadline", integer, 120)
-  .action(async (o) => {
+  .action(async (o, command) => {
     invariant(
       !o.embedding || (o.dryRun && o.path),
       "--embedding is only accepted with --dry-run --path; connected imports use the repository preset.",
@@ -242,6 +250,10 @@ cli
       !(o.artifact && (o.ref || o.path || o.output || o.previous)),
       "--artifact cannot be combined with --ref/--path/--output/--previous.",
     );
+    invariant(
+      !o.artifact || command.getOptionValueSource("artifactStorage") !== "cli",
+      "--artifact-storage cannot be combined with --artifact; the artifact records its storage format.",
+    );
     const buildLocal = async (
       source: Awaited<ReturnType<typeof identity>>,
       previous?: Build,
@@ -271,6 +283,7 @@ cli
         resolvedCommit: commit,
         output: path,
         previous,
+        storage: o.artifactStorage,
         preset,
       });
     };
@@ -300,6 +313,7 @@ cli
       output({
         status: "locally-validated",
         artifact: b.directory,
+        artifactFormat: b.format,
         commitOid: b.commitOid,
         configHash: b.configHash,
         counts: b.counts,

package/src/publish.ts

@@ -174,7 +174,7 @@ export async function validateCandidate(
     let query: Record<string, unknown> = {
       queryString: { query: "kind:chunk" },
     };
-    for await (const d of records(b.directory)) {
+    for await (const d of records(b)) {
       if (d.kind !== "chunk") continue;
       // Probe an indexed chunk ID: a valid source may contain only stop words
       // or punctuation, so text-query matches cannot establish readiness.
@@ -623,7 +623,7 @@ export async function publish(args: {
       await mutate("delete", ids, () => store.delete(j!.writer, ids));
     }
     async function* changed() {
-      for await (const d of records(b.directory))
+      for await (const d of records(b))
         if (oldHashes.get(d.id) !== hash(d)) yield d;
       for (const d of metadata.slice(0, -1))
         if (oldHashes.get(d.id) !== hash(d)) yield d;