@h-rig/runtime 0.0.6-alpha.0

package/dist/src/control-plane/runtime/snapshot-sidecar.js

@@ -0,0 +1,498 @@
+// @bun
+// packages/runtime/src/control-plane/runtime/snapshot-sidecar.ts
+import { copyFileSync as copyFileSync2, existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, rmSync as rmSync2, writeFileSync } from "fs";
+import { dirname as dirname2, relative, resolve as resolve2 } from "path";
+
+// packages/runtime/src/control-plane/runtime/snapshot.ts
+import { ptr as ptr2, toBuffer as toBuffer2 } from "bun:ffi";
+
+// packages/runtime/src/control-plane/native/runtime-native.ts
+import { dlopen, ptr, suffix, toBuffer } from "bun:ffi";
+import { copyFileSync, existsSync, mkdirSync, renameSync, rmSync, statSync } from "fs";
+import { tmpdir } from "os";
+import { dirname, resolve } from "path";
+var sharedNativeRuntimeOutputDir = resolve(tmpdir(), "rig-native");
+var sharedNativeRuntimeOutputPath = resolve(sharedNativeRuntimeOutputDir, `runtime-native-${process.platform}-${process.arch}.${suffix}`);
+var colocatedNativeRuntimeFileName = `runtime-native.${suffix}`;
+var nativeRuntimeLibrary = await loadNativeRuntimeLibrary();
+function requireNativeRuntimeLibrary(feature) {
+  if (!nativeRuntimeLibrary) {
+    throw new Error(`Native Zig runtime is required for ${feature}`);
+  }
+  return nativeRuntimeLibrary;
+}
+async function ensureNativeRuntimeLibraryPath(outputPath = sharedNativeRuntimeOutputPath, options = {}) {
+  if (await buildNativeRuntimeLibrary(outputPath, options)) {
+    return outputPath;
+  }
+  return !options.force && existsSync(outputPath) ? outputPath : null;
+}
+async function loadNativeRuntimeLibrary() {
+  if (process.env.RIG_DISABLE_ZIG_NATIVE === "1") {
+    return null;
+  }
+  for (const candidate of nativeRuntimeLibraryCandidates()) {
+    if (!candidate || !existsSync(candidate)) {
+      continue;
+    }
+    const loaded = tryDlopenNativeRuntimeLibrary(candidate);
+    if (loaded) {
+      return loaded;
+    }
+  }
+  const builtLibraryPath = await ensureNativeRuntimeLibraryPath(sharedNativeRuntimeOutputPath, { force: true });
+  if (!builtLibraryPath) {
+    return null;
+  }
+  return tryDlopenNativeRuntimeLibrary(builtLibraryPath);
+}
+function nativePackageLibraryCandidates(fromDir, names) {
+  const candidates = [];
+  let cursor = resolve(fromDir);
+  for (let index = 0;index < 8; index += 1) {
+    for (const name of names) {
+      candidates.push(resolve(cursor, "native", `${process.platform}-${process.arch}`, name), resolve(cursor, "native", `${process.platform}-${process.arch}`, "lib", name), resolve(cursor, "native", name), resolve(cursor, "native", "lib", name));
+    }
+    const parent = dirname(cursor);
+    if (parent === cursor)
+      break;
+    cursor = parent;
+  }
+  return candidates;
+}
+function nativeRuntimeLibraryCandidates() {
+  const explicit = process.env.RIG_NATIVE_RUNTIME_LIB?.trim() || "";
+  const execDir = process.execPath?.trim() ? dirname(process.execPath.trim()) : "";
+  const platformSpecific = `runtime-native-${process.platform}-${process.arch}.${suffix}`;
+  return [...new Set([
+    explicit,
+    ...nativePackageLibraryCandidates(import.meta.dir, [colocatedNativeRuntimeFileName, platformSpecific]),
+    execDir ? resolve(execDir, colocatedNativeRuntimeFileName) : "",
+    execDir ? resolve(execDir, platformSpecific) : "",
+    execDir ? resolve(execDir, "..", colocatedNativeRuntimeFileName) : "",
+    execDir ? resolve(execDir, "..", platformSpecific) : "",
+    execDir ? resolve(execDir, "lib", colocatedNativeRuntimeFileName) : "",
+    execDir ? resolve(execDir, "..", "lib", colocatedNativeRuntimeFileName) : "",
+    sharedNativeRuntimeOutputPath
+  ].filter(Boolean))];
+}
+function resolveNativeRuntimeSourcePath() {
+  const explicit = process.env.RIG_NATIVE_RUNTIME_SOURCE?.trim();
+  if (explicit && existsSync(explicit)) {
+    return explicit;
+  }
+  const bundled = resolve(import.meta.dir, "../../../native/snapshot.zig");
+  return existsSync(bundled) ? bundled : null;
+}
+async function buildNativeRuntimeLibrary(outputPath, options = {}) {
+  if (process.env.RIG_DISABLE_ZIG_NATIVE === "1") {
+    return false;
+  }
+  const zigBinary = Bun.which("zig");
+  const sourcePath = resolveNativeRuntimeSourcePath();
+  if (!zigBinary || !sourcePath) {
+    return false;
+  }
+  const tempOutputPath = `${outputPath}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
+  try {
+    mkdirSync(dirname(outputPath), { recursive: true });
+    const needsBuild = options.force === true || !existsSync(outputPath) || statSync(sourcePath).mtimeMs > statSync(outputPath).mtimeMs;
+    if (!needsBuild) {
+      return true;
+    }
+    const build = Bun.spawn([
+      zigBinary,
+      "build-lib",
+      sourcePath,
+      "-dynamic",
+      "-O",
+      "ReleaseFast",
+      `-femit-bin=${tempOutputPath}`
+    ], {
+      cwd: import.meta.dir,
+      stdout: "pipe",
+      stderr: "pipe"
+    });
+    const exitCode = await build.exited;
+    if (exitCode !== 0 || !existsSync(tempOutputPath)) {
+      rmSync(tempOutputPath, { force: true });
+      return false;
+    }
+    renameSync(tempOutputPath, outputPath);
+    return true;
+  } catch {
+    rmSync(tempOutputPath, { force: true });
+    return false;
+  }
+}
+function tryDlopenNativeRuntimeLibrary(outputPath) {
+  try {
+    return dlopen(outputPath, {
+      rig_scope_match: {
+        args: ["ptr", "ptr"],
+        returns: "u8"
+      },
+      snapshot_capture: {
+        args: ["ptr", "u64", "ptr", "u64"],
+        returns: "ptr"
+      },
+      snapshot_delta: {
+        args: ["ptr", "ptr"],
+        returns: "ptr"
+      },
+      snapshot_store_delta: {
+        args: ["ptr", "ptr", "ptr", "u64", "ptr", "u64", "ptr", "u64", "ptr", "u64"],
+        returns: "ptr"
+      },
+      snapshot_inspect_delta: {
+        args: ["ptr", "u64"],
+        returns: "ptr"
+      },
+      snapshot_apply_delta: {
+        args: ["ptr", "u64", "ptr", "u64"],
+        returns: "ptr"
+      },
+      snapshot_release: {
+        args: ["ptr"],
+        returns: "void"
+      },
+      runtime_hash_file: {
+        args: ["ptr", "u64"],
+        returns: "ptr"
+      },
+      runtime_hash_tree: {
+        args: ["ptr", "u64"],
+        returns: "ptr"
+      },
+      runtime_prepare_paths: {
+        args: ["ptr", "u64", "ptr", "u64", "ptr", "u64", "ptr", "u64", "ptr", "u64"],
+        returns: "ptr"
+      },
+      runtime_link_dependency_layer: {
+        args: ["ptr", "u64", "ptr", "u64"],
+        returns: "ptr"
+      },
+      runtime_scan_worktrees: {
+        args: ["ptr", "u64"],
+        returns: "ptr"
+      }
+    });
+  } catch {
+    return null;
+  }
+}
+
+// packages/runtime/src/control-plane/runtime/snapshot.ts
+var MANIFEST_HANDLE_SIZE = 40;
+var DELTA_RESULT_SIZE = 72;
+var FILE_ENTRY_SIZE = 40;
+var DELTA_ENTRY_SIZE = 24;
+var STRING_SLICE_SIZE = 16;
+var STORE_RESULT_SIZE = 104;
+var MANIFEST_NATIVE_HANDLE = Symbol("manifest-native-handle");
+var DEFAULT_EXCLUDES = new Set([
+  "node_modules",
+  ".git",
+  ".worktrees",
+  ".tmp",
+  ".rig",
+  "artifacts"
+]);
+var nativeSnapshotLibrary = requireNativeRuntimeLibrary("runtime snapshots");
+var manifestReleaseRegistry = typeof FinalizationRegistry !== "undefined" ? new FinalizationRegistry((resourcePtr) => {
+  try {
+    nativeSnapshotLibrary.symbols.snapshot_release(resourcePtr);
+  } catch {}
+}) : null;
+function captureManifest(rootDir, options) {
+  const exclude = options?.exclude ?? DEFAULT_EXCLUDES;
+  const captured = captureManifestEntries(rootDir, [...exclude]);
+  const manifest = {
+    root: rootDir,
+    capturedAt: new Date().toISOString(),
+    entries: captured.entries
+  };
+  manifest[MANIFEST_NATIVE_HANDLE] = captured.nativeHandle;
+  manifestReleaseRegistry?.register(manifest, captured.nativeHandle, manifest);
+  return manifest;
+}
+function releaseManifest(manifest) {
+  if (!manifest) {
+    return;
+  }
+  const nativeManifest = manifest;
+  const handle = nativeManifest[MANIFEST_NATIVE_HANDLE];
+  if (!handle) {
+    return;
+  }
+  manifestReleaseRegistry?.unregister(manifest);
+  delete nativeManifest[MANIFEST_NATIVE_HANDLE];
+  nativeSnapshotLibrary.symbols.snapshot_release(handle);
+}
+function computeDelta(before, after, taskId) {
+  const deltaEntries = computeDeltaEntries(before, after);
+  return {
+    taskId,
+    baseManifestHash: computeManifestHash(before),
+    capturedAt: new Date().toISOString(),
+    modified: deltaEntries.modified,
+    created: deltaEntries.created,
+    deleted: deltaEntries.deleted
+  };
+}
+function snapshotBefore(worktreeDir, exclude) {
+  return captureManifest(worktreeDir, { exclude });
+}
+function snapshotAfter(worktreeDir, beforeManifest, taskId, outputPath, exclude) {
+  const afterManifest = captureManifest(worktreeDir, { exclude });
+  const delta = computeDelta(beforeManifest, afterManifest, taskId);
+  const capturedAt = new Date().toISOString();
+  const meta = storeSnapshotArchive(beforeManifest, afterManifest, worktreeDir, taskId, capturedAt, outputPath);
+  meta.beforeEntries = beforeManifest.entries.size;
+  meta.afterEntries = afterManifest.entries.size;
+  return { delta: { ...delta, capturedAt: meta.capturedAt, baseManifestHash: meta.baseManifestHash ?? delta.baseManifestHash }, meta, afterManifest };
+}
+function captureManifestEntries(rootDir, exclude) {
+  const rootBuffer = Buffer.from(rootDir, "utf8");
+  const excludeBuffer = Buffer.from(exclude.join("\x00"), "utf8");
+  const handlePtr = nativeSnapshotLibrary.symbols.snapshot_capture(Number(ptr2(rootBuffer)), rootBuffer.byteLength, Number(ptr2(excludeBuffer)), excludeBuffer.byteLength);
+  if (!handlePtr) {
+    throw new Error(`snapshot_capture returned null for ${rootDir}`);
+  }
+  const handleView = viewAt(handlePtr, MANIFEST_HANDLE_SIZE);
+  try {
+    throwIfError(nativeSnapshotLibrary.symbols, handleView, 24, 32, "snapshot_capture");
+    const entriesPtr = readU64(handleView, 8);
+    const entriesLen = readU64(handleView, 16);
+    const entries = new Map;
+    for (let index = 0;index < entriesLen; index += 1) {
+      const entryView = viewAt(entriesPtr + index * FILE_ENTRY_SIZE, FILE_ENTRY_SIZE);
+      const path = readString(readU64(entryView, 0), readU64(entryView, 8));
+      entries.set(path, {
+        path,
+        hash: readBigU64(entryView, 16).toString(16),
+        size: readU64(entryView, 24),
+        mode: readU64(entryView, 32)
+      });
+    }
+    return {
+      entries,
+      nativeHandle: handlePtr
+    };
+  } catch (error) {
+    nativeSnapshotLibrary.symbols.snapshot_release(handlePtr);
+    throw error;
+  }
+}
+function computeDeltaEntries(before, after) {
+  const beforeHandle = before[MANIFEST_NATIVE_HANDLE];
+  const afterHandle = after[MANIFEST_NATIVE_HANDLE];
+  if (!beforeHandle || !afterHandle) {
+    throw new Error("snapshot manifests must carry native handles");
+  }
+  const resultPtr = nativeSnapshotLibrary.symbols.snapshot_delta(beforeHandle, afterHandle);
+  if (!resultPtr) {
+    throw new Error("snapshot_delta returned null");
+  }
+  try {
+    const resultView = viewAt(resultPtr, DELTA_RESULT_SIZE);
+    throwIfError(nativeSnapshotLibrary.symbols, resultView, 56, 64, "snapshot_delta");
+    return {
+      modified: readDeltaEntries(readU64(resultView, 8), readU64(resultView, 16)),
+      created: readDeltaEntries(readU64(resultView, 24), readU64(resultView, 32)),
+      deleted: readStringSliceArray(readU64(resultView, 40), readU64(resultView, 48))
+    };
+  } finally {
+    nativeSnapshotLibrary.symbols.snapshot_release(resultPtr);
+  }
+}
+function storeSnapshotArchive(beforeManifest, afterManifest, worktreeDir, taskId, capturedAt, outputPath) {
+  const beforeHandle = beforeManifest[MANIFEST_NATIVE_HANDLE];
+  const afterHandle = afterManifest[MANIFEST_NATIVE_HANDLE];
+  if (!beforeHandle || !afterHandle) {
+    throw new Error("snapshot manifests must carry native handles");
+  }
+  const worktreeBuffer = Buffer.from(worktreeDir, "utf8");
+  const taskBuffer = Buffer.from(taskId, "utf8");
+  const capturedAtBuffer = Buffer.from(capturedAt, "utf8");
+  const outputBuffer = Buffer.from(outputPath, "utf8");
+  const resultPtr = nativeSnapshotLibrary.symbols.snapshot_store_delta(beforeHandle, afterHandle, Number(ptr2(worktreeBuffer)), worktreeBuffer.byteLength, Number(ptr2(taskBuffer)), taskBuffer.byteLength, Number(ptr2(capturedAtBuffer)), capturedAtBuffer.byteLength, Number(ptr2(outputBuffer)), outputBuffer.byteLength);
+  if (!resultPtr) {
+    throw new Error(`snapshot_store_delta returned null for ${outputPath}`);
+  }
+  try {
+    const resultView = viewAt(resultPtr, STORE_RESULT_SIZE);
+    throwIfError(nativeSnapshotLibrary.symbols, resultView, 88, 96, "snapshot_store_delta");
+    return {
+      taskId,
+      baseManifestHash: readString(readU64(resultView, 8), readU64(resultView, 16)),
+      capturedAt: readString(readU64(resultView, 24), readU64(resultView, 32)),
+      beforeEntries: 0,
+      afterEntries: 0,
+      modifiedCount: readU64(resultView, 40),
+      createdCount: readU64(resultView, 48),
+      deletedCount: readU64(resultView, 56),
+      blobCount: readU64(resultView, 64),
+      blobsSize: readU64(resultView, 72),
+      compressedSize: readU64(resultView, 80)
+    };
+  } finally {
+    nativeSnapshotLibrary.symbols.snapshot_release(resultPtr);
+  }
+}
+function computeManifestHash(manifest) {
+  const sorted = [...manifest.entries.values()].sort((left, right) => left.path.localeCompare(right.path));
+  const hasher = new Bun.CryptoHasher("sha256");
+  for (const entry of sorted) {
+    hasher.update(entry.path);
+    hasher.update("\x00");
+    hasher.update(entry.hash);
+    hasher.update("\x00");
+    hasher.update(String(entry.size));
+    hasher.update("\x00");
+    hasher.update(String(entry.mode));
+  }
+  return hasher.digest("hex");
+}
+function readDeltaEntries(pointer, count) {
+  const values = [];
+  for (let index = 0;index < count; index += 1) {
+    const entryView = viewAt(pointer + index * DELTA_ENTRY_SIZE, DELTA_ENTRY_SIZE);
+    values.push({
+      path: readString(readU64(entryView, 0), readU64(entryView, 8)),
+      hash: readBigU64(entryView, 16).toString(16)
+    });
+  }
+  return values;
+}
+function readStringSliceArray(pointer, count) {
+  const values = [];
+  for (let index = 0;index < count; index += 1) {
+    const entryView = viewAt(pointer + index * STRING_SLICE_SIZE, STRING_SLICE_SIZE);
+    values.push(readString(readU64(entryView, 0), readU64(entryView, 8)));
+  }
+  return values;
+}
+function viewAt(pointer, size) {
+  const buffer = toBuffer2(pointer, 0, size);
+  return new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+}
+function readString(pointer, length) {
+  if (!pointer || length === 0) {
+    return "";
+  }
+  return Buffer.from(toBuffer2(pointer, 0, length)).toString("utf8");
+}
+function readU64(view, offset) {
+  return Number(view.getBigUint64(offset, true));
+}
+function readBigU64(view, offset) {
+  return view.getBigUint64(offset, true);
+}
+function readError(view, pointerOffset, lengthOffset) {
+  const errorPointer = readU64(view, pointerOffset);
+  const errorLength = readU64(view, lengthOffset);
+  if (!errorPointer || !errorLength) {
+    return null;
+  }
+  return readString(errorPointer, errorLength);
+}
+function throwIfError(_symbols, view, pointerOffset, lengthOffset, label) {
+  const error = readError(view, pointerOffset, lengthOffset);
+  if (error) {
+    throw new Error(`${label} failed: ${error}`);
+  }
+}
+var __testOnly = {
+  defaultExcludes: [...DEFAULT_EXCLUDES]
+};
+
+// packages/runtime/src/control-plane/runtime/snapshot-sidecar.ts
+function parseArgs(argv) {
+  const values = new Map;
+  for (let index = 0;index < argv.length; index += 2) {
+    values.set(argv[index] ?? "", argv[index + 1] ?? "");
+  }
+  const workspaceDir = values.get("--workspace") ?? "";
+  const taskId = values.get("--task") ?? "";
+  const runtimeId = values.get("--runtime-id") ?? "";
+  const instanceId = values.get("--instance-id") ?? runtimeId;
+  const readyFile = values.get("--ready-file") ?? "";
+  const requestFile = values.get("--request-file") ?? "";
+  if (!workspaceDir || !taskId || !runtimeId || !instanceId || !readyFile || !requestFile) {
+    throw new Error("Usage: snapshot-sidecar.ts --workspace <dir> --task <id> --runtime-id <id> [--instance-id <id>] --ready-file <path> --request-file <path>");
+  }
+  return { workspaceDir, taskId, runtimeId, instanceId, readyFile, requestFile };
+}
+function sanitizeRuntimeRefSegment(value) {
+  const sanitized = value.trim().replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
+  return (sanitized || "runtime").slice(0, 64);
+}
+var waitBuffer = new Int32Array(new SharedArrayBuffer(4));
+function readPayload(requestFile) {
+  while (!existsSync2(requestFile)) {
+    Atomics.wait(waitBuffer, 0, 0, 25);
+  }
+  const raw = readFileSync(requestFile, "utf-8").trim();
+  const payload = JSON.parse(raw);
+  if (!Array.isArray(payload.command) || typeof payload.exitCode !== "number") {
+    throw new Error("snapshot sidecar received malformed finalize payload");
+  }
+  return {
+    command: payload.command.map((part) => String(part)),
+    exitCode: payload.exitCode
+  };
+}
+async function runRuntimeSnapshotSidecar(argv = process.argv.slice(2)) {
+  const options = parseArgs(argv);
+  const beforeManifest = snapshotBefore(options.workspaceDir);
+  let afterManifest;
+  try {
+    mkdirSync2(dirname2(options.readyFile), { recursive: true });
+    writeFileSync(options.readyFile, `ready
+`, "utf-8");
+    const payload = readPayload(options.requestFile);
+    const artifactDir = resolve2(options.workspaceDir, "artifacts", options.taskId, "runtime-snapshots");
+    mkdirSync2(artifactDir, { recursive: true });
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const historyStem = `${timestamp}-${sanitizeRuntimeRefSegment(options.instanceId)}`;
+    const snapshotPath = resolve2(artifactDir, `${historyStem}.snap.bin`);
+    const latestSnapshotPath = resolve2(artifactDir, "latest.snap.bin");
+    const historyMetaPath = resolve2(artifactDir, `${historyStem}.json`);
+    const latestMetaPath = resolve2(artifactDir, "latest.json");
+    const result = snapshotAfter(options.workspaceDir, beforeManifest, options.taskId, snapshotPath);
+    afterManifest = result.afterManifest;
+    const metaPayload = {
+      taskId: options.taskId,
+      runtimeId: options.runtimeId,
+      exitCode: payload.exitCode,
+      command: payload.command,
+      workspaceDir: options.workspaceDir,
+      sidecarInstanceId: options.instanceId,
+      snapshotPath: relative(options.workspaceDir, snapshotPath).replaceAll("\\", "/"),
+      latestSnapshotPath: relative(options.workspaceDir, latestSnapshotPath).replaceAll("\\", "/"),
+      meta: result.meta,
+      recordedAt: new Date().toISOString()
+    };
+    copyFileSync2(snapshotPath, latestSnapshotPath);
+    writeFileSync(historyMetaPath, `${JSON.stringify(metaPayload, null, 2)}
+`, "utf-8");
+    writeFileSync(latestMetaPath, `${JSON.stringify(metaPayload, null, 2)}
+`, "utf-8");
+    console.log(JSON.stringify(metaPayload));
+  } finally {
+    releaseManifest(beforeManifest);
+    releaseManifest(afterManifest);
+    rmSync2(options.readyFile, { force: true });
+    rmSync2(options.requestFile, { force: true });
+  }
+}
+if (import.meta.main) {
+  runRuntimeSnapshotSidecar().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}
+export {
+  runRuntimeSnapshotSidecar
+};