mlclaw 0.1.0

package/src/hf-state-sync/restore.ts

@@ -0,0 +1,109 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { extractTarZst, sha256File } from "./archive.js";
+import type { BucketHub } from "./hub.js";
+import { MANIFEST_REMOTE_NAME, type SnapshotEntry, parseManifest } from "./manifest.js";
+import { type SyncConfig, log, logError, remotePath } from "./paths.js";
+import { checkIntegrity, findSqliteFiles } from "./sqlite.js";
+
+export type RestoreOutcome =
+  | { kind: "restored"; entry: SnapshotEntry }
+  | { kind: "fresh-start"; reason: "no-bucket" | "no-manifest" | "live-dir-exists" }
+  // Both failure kinds must stop the boot: starting fresh and snapshotting
+  // would overwrite the rollback index of a bucket that still holds state.
+  | { kind: "invalid-manifest"; reason: string }
+  | { kind: "all-snapshots-failed"; tried: string[] };
+
+async function tryRestoreEntry(params: {
+  hub: BucketHub;
+  entry: SnapshotEntry;
+  workDir: string;
+  liveDir: string;
+}): Promise<"restored" | "failed"> {
+  const { hub, entry, workDir, liveDir } = params;
+  const archivePath = path.join(workDir, `candidate-${entry.id}.tar.zst`);
+  const downloaded = await hub.download(entry.path, archivePath);
+  if (downloaded === "not-found") {
+    logError(`snapshot ${entry.id} missing from bucket`);
+    return "failed";
+  }
+  const digest = await sha256File(archivePath);
+  if (digest !== entry.sha256) {
+    logError(`snapshot ${entry.id} checksum mismatch`);
+    return "failed";
+  }
+  // Extract next to the live dir, not in tmpdir: the final rename into place
+  // must stay on one filesystem (rename across mounts fails with EXDEV).
+  const extractDir = `${liveDir}.restoring-${entry.id}`;
+  await fs.rm(extractDir, { recursive: true, force: true });
+  try {
+    try {
+      await extractTarZst(archivePath, extractDir);
+    } catch (err) {
+      // A checksum can match a manifest that was written for a broken upload;
+      // extraction failure must fall back to older entries, not abort restore.
+      const detail = err instanceof Error ? err.message : String(err);
+      logError(`snapshot ${entry.id} failed to extract: ${detail}`);
+      return "failed";
+    }
+    for (const database of await findSqliteFiles(extractDir)) {
+      const integrity = checkIntegrity(database);
+      if (integrity.kind === "corrupt") {
+        logError(`snapshot ${entry.id} db ${path.basename(database)} corrupt: ${integrity.detail}`);
+        return "failed";
+      }
+    }
+    // Verified: move into place atomically. The live dir must not exist yet
+    // (fresh container) — restore never overwrites live state.
+    await fs.mkdir(path.dirname(liveDir), { recursive: true });
+    await fs.rename(extractDir, liveDir);
+    return "restored";
+  } finally {
+    // No-op after a successful rename; clears partial extractions otherwise.
+    await fs.rm(extractDir, { recursive: true, force: true });
+  }
+}
+
+/** Restore the newest verified snapshot into the (not yet existing) live dir. */
+export async function runRestore(params: {
+  config: SyncConfig;
+  hub: BucketHub;
+}): Promise<RestoreOutcome> {
+  const { config, hub } = params;
+  if (!config.bucket) {
+    return { kind: "fresh-start", reason: "no-bucket" };
+  }
+  try {
+    await fs.access(config.liveDir);
+    return { kind: "fresh-start", reason: "live-dir-exists" };
+  } catch {
+    // expected: fresh container
+  }
+
+  const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "hf-state-restore-"));
+  try {
+    const manifestPath = path.join(workDir, "manifest.json");
+    const downloaded = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), manifestPath);
+    if (downloaded === "not-found") {
+      return { kind: "fresh-start", reason: "no-manifest" };
+    }
+    const parsed = parseManifest(await fs.readFile(manifestPath, "utf8"));
+    if (parsed.kind === "invalid") {
+      return { kind: "invalid-manifest", reason: parsed.reason };
+    }
+
+    const candidates = [parsed.manifest.current, ...parsed.manifest.previous];
+    const tried: string[] = [];
+    for (const entry of candidates) {
+      tried.push(entry.id);
+      if ((await tryRestoreEntry({ hub, entry, workDir, liveDir: config.liveDir })) === "restored") {
+        log(`restored snapshot ${entry.id} (created ${entry.createdAt})`);
+        return { kind: "restored", entry };
+      }
+    }
+    return { kind: "all-snapshots-failed", tried };
+  } finally {
+    await fs.rm(workDir, { recursive: true, force: true });
+  }
+}

package/src/hf-state-sync/snapshot.ts

@@ -0,0 +1,133 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { createTarZst, sha256File, stageLiveDir } from "./archive.js";
+import type { BucketHub } from "./hub.js";
+import {
+  MANIFEST_REMOTE_NAME,
+  type Manifest,
+  type SnapshotEntry,
+  parseManifest,
+  promoteSnapshot,
+  serializeManifest,
+} from "./manifest.js";
+import { type SyncConfig, log, remotePath } from "./paths.js";
+
+export type SnapshotOutcome =
+  | { kind: "uploaded"; entry: SnapshotEntry }
+  | { kind: "skipped"; reason: "no-bucket" | "empty-state" }
+  | { kind: "failed"; detail: string };
+
+// Object names must be unique across overlapping containers (run-id suffix)
+// AND within one run (counter): a final snapshot can land in the same second
+// as the last interval snapshot, and an overwritten tarball behind a distinct
+// manifest entry would poison rollback and confuse retention pruning.
+let snapshotCounter = 0;
+function snapshotId(now: Date, runId: string): string {
+  snapshotCounter += 1;
+  const stamp = now.toISOString().replaceAll(":", "-").replace(".", "-");
+  return `${stamp}-${runId.slice(0, 8)}-${snapshotCounter}`;
+}
+
+type FetchManifestResult =
+  | { kind: "none" }
+  | { kind: "ok"; manifest: Manifest }
+  | { kind: "invalid"; reason: string };
+
+async function fetchManifest(
+  config: SyncConfig,
+  hub: BucketHub,
+  workDir: string,
+): Promise<FetchManifestResult> {
+  const localPath = path.join(workDir, "manifest.remote.json");
+  const result = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), localPath);
+  if (result === "not-found") {
+    return { kind: "none" };
+  }
+  const parsed = parseManifest(await fs.readFile(localPath, "utf8"));
+  return parsed.kind === "ok"
+    ? { kind: "ok", manifest: parsed.manifest }
+    : { kind: "invalid", reason: parsed.reason };
+}
+
+/**
+ * Stage, verify, upload, then promote: the manifest object is only
+ * overwritten after the tarball upload succeeded, so it never references a
+ * partial archive. Object writes are atomic; no locking (last verified
+ * writer wins across overlapping containers).
+ */
+export async function runSnapshot(params: {
+  config: SyncConfig;
+  hub: BucketHub;
+  bootTime: string;
+  now?: () => Date;
+}): Promise<SnapshotOutcome> {
+  const { config, hub } = params;
+  if (!config.bucket) {
+    return { kind: "skipped", reason: "no-bucket" };
+  }
+  try {
+    await fs.access(config.liveDir);
+  } catch {
+    return { kind: "skipped", reason: "empty-state" };
+  }
+
+  const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "hf-state-snapshot-"));
+  try {
+    const stagingDir = path.join(workDir, "stage");
+    const staged = await stageLiveDir(config.liveDir, stagingDir);
+    if (staged.kind === "corrupt-database") {
+      return {
+        kind: "failed",
+        detail: `live database ${staged.database} failed integrity check: ${staged.detail}`,
+      };
+    }
+
+    const now = (params.now ?? (() => new Date()))();
+    const id = snapshotId(now, config.runId);
+    const archiveName = `state-${id}.tar.zst`;
+    const archivePath = path.join(workDir, archiveName);
+    await createTarZst(stagingDir, archivePath);
+
+    const entry: SnapshotEntry = {
+      id,
+      path: remotePath(config, `snapshots/${archiveName}`),
+      createdAt: now.toISOString(),
+      sha256: await sha256File(archivePath),
+      sizeBytes: (await fs.stat(archivePath)).size,
+      runId: config.runId,
+      bootTime: params.bootTime,
+    };
+
+    await hub.upload(archivePath, entry.path);
+
+    const existing = await fetchManifest(config, hub, workDir);
+    if (existing.kind === "invalid") {
+      // The manifest is the only rollback index (buckets are non-versioned).
+      // Never overwrite an index we cannot read — its snapshots may be the
+      // only good copy of the user's data.
+      return {
+        kind: "failed",
+        detail: `remote manifest is invalid, refusing to overwrite it: ${existing.reason}`,
+      };
+    }
+    const { manifest, expired } = promoteSnapshot({
+      existing: existing.kind === "ok" ? existing.manifest : null,
+      entry,
+      keep: config.keepSnapshots,
+    });
+    const manifestPath = path.join(workDir, "manifest.json");
+    await fs.writeFile(manifestPath, serializeManifest(manifest));
+    await hub.upload(manifestPath, remotePath(config, MANIFEST_REMOTE_NAME));
+
+    if (expired.length > 0) {
+      await hub.delete(expired.map((e) => e.path));
+    }
+    log(`snapshot ${entry.id} uploaded (${entry.sizeBytes} bytes, ${staged.databases.length} dbs)`);
+    return { kind: "uploaded", entry };
+  } catch (err) {
+    return { kind: "failed", detail: err instanceof Error ? err.message : String(err) };
+  } finally {
+    await fs.rm(workDir, { recursive: true, force: true });
+  }
+}

package/src/hf-state-sync/sqlite.ts

@@ -0,0 +1,57 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+/** Recursively find live SQLite database files under a directory. */
+export async function findSqliteFiles(root: string): Promise<string[]> {
+  const found: string[] = [];
+  let entries;
+  try {
+    entries = await fs.readdir(root, { withFileTypes: true, recursive: true });
+  } catch {
+    return found;
+  }
+  for (const entry of entries) {
+    if (entry.isFile() && entry.name.endsWith(".sqlite")) {
+      found.push(path.join(entry.parentPath, entry.name));
+    }
+  }
+  return found.sort();
+}
+
+/**
+ * Produce a consistent standalone copy of a live SQLite DB. VACUUM INTO takes
+ * its own read transaction, so this is safe while OpenClaw is writing; raw
+ * file copies of a WAL-mode DB are not.
+ */
+export function vacuumInto(sourceDb: string, destDb: string): void {
+  const db = new DatabaseSync(sourceDb);
+  try {
+    db.prepare("VACUUM INTO ?").run(destDb);
+  } finally {
+    db.close();
+  }
+}
+
+export type IntegrityResult = { kind: "ok" } | { kind: "corrupt"; detail: string };
+
+export function checkIntegrity(dbPath: string): IntegrityResult {
+  let db: DatabaseSync;
+  try {
+    db = new DatabaseSync(dbPath, { readOnly: true });
+  } catch (err) {
+    return { kind: "corrupt", detail: `cannot open: ${String(err)}` };
+  }
+  try {
+    const row = db.prepare("PRAGMA integrity_check").get() as
+      | { integrity_check?: unknown }
+      | undefined;
+    return row?.integrity_check === "ok"
+      ? { kind: "ok" }
+      : { kind: "corrupt", detail: String(row?.integrity_check ?? "no result") };
+  } catch (err) {
+    return { kind: "corrupt", detail: String(err) };
+  } finally {
+    db.close();
+  }
+}

package/src/hf-state-sync/supervise.ts

@@ -0,0 +1,256 @@
+import { type ChildProcess, spawn } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import type { BucketHub } from "./hub.js";
+import { type SyncConfig, log, logError, remotePath } from "./paths.js";
+import { runSnapshot, type SnapshotOutcome } from "./snapshot.js";
+
+const LEASE_HEARTBEAT_MS = 60_000;
+const HANDOFF_REQUEST_TTL_MS = 10 * 60 * 1000;
+
+/**
+ * Run OpenClaw as a child process with a periodic snapshot loop. On SIGTERM/
+ * SIGINT (Space shutdown/rebuild) the signal is forwarded and a best-effort
+ * final snapshot runs after the child exits, so at most one interval of state
+ * is lost. If the platform hard-kills us first, the interval loop has already
+ * bounded the loss the same way.
+ */
+export async function supervise(params: {
+  config: SyncConfig;
+  hub: BucketHub;
+  command: string[];
+}): Promise<number> {
+  const { config, hub, command } = params;
+  const [binary, ...args] = command;
+  if (!binary) {
+    throw new Error("supervise: missing child command");
+  }
+  const bootTime = new Date().toISOString();
+  let lastSnapshotId: string | undefined;
+  const handoffState: { request: RuntimeHandoffRequest | null } = { request: null };
+
+  const writeLease = async () => {
+    const status = {
+      schemaVersion: 1,
+      agent: config.agentName,
+      runtimeId: config.runtimeId,
+      gatewayLocation: config.gatewayLocation,
+      runtimeImage: config.runtimeImage,
+      startedAt: bootTime,
+      lastHeartbeatAt: new Date().toISOString(),
+      ...(lastSnapshotId ? { lastSnapshotId } : {}),
+    };
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "hf-state-lease-"));
+    try {
+      const file = path.join(tmpDir, "status.json");
+      await fs.writeFile(file, JSON.stringify(status, null, 2) + "\n");
+      await hub.upload(file, remotePath(config, "runtime/status.json"));
+    } finally {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  };
+  const readHandoffRequest = async (): Promise<RuntimeHandoffRequest | null> => {
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "hf-state-handoff-"));
+    try {
+      const file = path.join(tmpDir, "request.json");
+      const result = await hub.download(remotePath(config, "runtime/handoff-request.json"), file);
+      if (result === "not-found") {
+        return null;
+      }
+      const parsed = JSON.parse(await fs.readFile(file, "utf8")) as RuntimeHandoffRequest;
+      if (
+        parsed?.schemaVersion !== 1 ||
+        parsed.agent !== config.agentName ||
+        parsed.runtimeId !== config.runtimeId ||
+        typeof parsed.requestedAt !== "string" ||
+        typeof parsed.requestId !== "string" ||
+        !parsed.requestId
+      ) {
+        return null;
+      }
+      const requestedAt = Date.parse(parsed.requestedAt);
+      if (!Number.isFinite(requestedAt) || Date.now() - requestedAt > HANDOFF_REQUEST_TTL_MS) {
+        log(`ignoring expired handoff request ${parsed.requestId}`);
+        await hub.delete([remotePath(config, "runtime/handoff-request.json")]).catch((err) => {
+          logError(`failed to clear expired handoff request: ${err instanceof Error ? err.message : String(err)}`);
+        });
+        return null;
+      }
+      return parsed;
+    } finally {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  };
+  const writeHandoffAck = async (request: RuntimeHandoffRequest) => {
+    const ack = {
+      schemaVersion: 1,
+      requestId: request.requestId,
+      agent: config.agentName,
+      runtimeId: config.runtimeId,
+      gatewayLocation: config.gatewayLocation,
+      completedAt: new Date().toISOString(),
+      ...(lastSnapshotId ? { lastSnapshotId } : {}),
+    };
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "hf-state-handoff-ack-"));
+    try {
+      const file = path.join(tmpDir, "ack.json");
+      await fs.writeFile(file, JSON.stringify(ack, null, 2) + "\n");
+      await hub.upload(file, remotePath(config, "runtime/handoff-ack.json"));
+      await hub.delete([remotePath(config, "runtime/handoff-request.json")]);
+    } finally {
+      await fs.rm(tmpDir, { recursive: true, force: true });
+    }
+  };
+
+  const child: ChildProcess = spawn(binary, args, { stdio: "inherit" });
+  const childExit = new Promise<number>((resolve) => {
+    child.on("exit", (code, signal) => resolve(code ?? (signal ? 128 : 1)));
+    child.on("error", (err) => {
+      logError(`child failed to start: ${err.message}`);
+      resolve(1);
+    });
+  });
+
+  let stopping = false;
+  let inFlight: Promise<SnapshotOutcome> | null = null;
+  const runOnce = async (label: string): Promise<SnapshotOutcome> => {
+    try {
+      const outcome = await runSnapshot({ config, hub, bootTime });
+      if (outcome.kind === "failed") {
+        logError(`${label}: snapshot failed: ${outcome.detail}`);
+      } else if (outcome.kind === "uploaded") {
+        lastSnapshotId = outcome.entry.path;
+      }
+      await writeLease().catch((err) => {
+        logError(`${label}: lease heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
+      });
+      return outcome;
+    } finally {
+      inFlight = null;
+    }
+  };
+  const snapshotInterval = async () => {
+    if (inFlight) {
+      log("interval: previous snapshot still running, skipping");
+      return;
+    }
+    inFlight = runOnce("interval");
+    await inFlight;
+  };
+  // The final snapshot must neither be skipped nor kill an in-flight upload:
+  // wait the in-flight one out, then take a fresh snapshot of the quiesced
+  // state before the process is allowed to exit.
+  const snapshotFinal = async (): Promise<SnapshotOutcome> => {
+    if (inFlight) {
+      await inFlight;
+    }
+    inFlight = runOnce("final");
+    return await inFlight;
+  };
+
+  const snapshotLoop = (async () => {
+    while (!stopping) {
+      await delay(config.intervalSeconds * 1000);
+      if (stopping) {
+        return;
+      }
+      await snapshotInterval();
+    }
+  })();
+  void snapshotLoop;
+
+  const heartbeatLoop = (async () => {
+    await writeLease().catch((err) => logError(`initial lease failed: ${err instanceof Error ? err.message : String(err)}`));
+    while (!stopping) {
+      await delay(LEASE_HEARTBEAT_MS);
+      if (stopping) {
+        return;
+      }
+      await writeLease().catch((err) => {
+        logError(`lease heartbeat failed: ${err instanceof Error ? err.message : String(err)}`);
+      });
+    }
+  })();
+  void heartbeatLoop;
+
+  const handoffLoop = (async () => {
+    while (!stopping && !handoffState.request) {
+      await delay(config.handoffPollSeconds * 1000);
+      if (stopping || handoffState.request) {
+        return;
+      }
+      try {
+        const request = await readHandoffRequest();
+        if (!request) {
+          continue;
+        }
+        handoffState.request = request;
+        log(`handoff ${request.requestId} requested for ${request.targetRuntimeId}`);
+        stopping = true;
+        child.kill("SIGTERM");
+      } catch (err) {
+        logError(`handoff poll failed: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  })();
+  void handoffLoop;
+
+  const forwardSignal = (signal: NodeJS.Signals) => {
+    log(`received ${signal}, shutting down`);
+    stopping = true;
+    child.kill(signal);
+  };
+  process.on("SIGTERM", forwardSignal);
+  process.on("SIGINT", forwardSignal);
+
+  const exitCode = await childExit;
+  stopping = true;
+
+  log(`child exited with code ${exitCode}, taking final snapshot`);
+  if (!handoffState.request) {
+    try {
+      const shutdownRequest = await readHandoffRequest();
+      if (shutdownRequest) {
+        handoffState.request = shutdownRequest;
+        log(`handoff ${shutdownRequest.requestId} requested for ${shutdownRequest.targetRuntimeId}`);
+      }
+    } catch (err) {
+      logError(`shutdown handoff check failed: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  const finalOutcome = await snapshotFinal();
+  if (handoffState.request) {
+    const request = handoffState.request;
+    if (finalOutcome.kind !== "uploaded") {
+      throw new Error(`handoff ${request.requestId} final snapshot did not upload: ${snapshotFailureDetail(finalOutcome)}`);
+    }
+    await writeHandoffAck(request).catch((err) => {
+      throw new Error(`handoff ${request.requestId} snapshot completed but ack failed: ${err instanceof Error ? err.message : String(err)}`);
+    });
+    log(`handoff ${request.requestId} acknowledged`);
+    return 0;
+  }
+  return exitCode;
+}
+
+function snapshotFailureDetail(outcome: SnapshotOutcome): string {
+  switch (outcome.kind) {
+    case "uploaded":
+      return "uploaded";
+    case "failed":
+      return outcome.detail;
+    case "skipped":
+      return outcome.reason;
+  }
+}
+
+type RuntimeHandoffRequest = {
+  schemaVersion: 1;
+  requestId: string;
+  agent: string;
+  runtimeId: string;
+  requestedAt: string;
+  targetRuntimeId: string;
+};

package/src/vendor/hfjs-xet/error.ts

@@ -0,0 +1,52 @@
+// @ts-nocheck -- vendored upstream code, kept verbatim; our strict tsconfig does not apply.
+// Vendored from huggingface/huggingface.js@f8fdf6be (packages/hub/src/error.ts), MIT License.
+// Delete this directory when bucket support is upstreamed to @huggingface/hub.
+import type { JsonObject } from "./vendor/type-fest/basic";
+
+export async function createApiError(
+	response: Response,
+	opts?: { requestId?: string; message?: string },
+): Promise<never> {
+	const error = new HubApiError(response.url, response.status, response.headers.get("X-Request-Id") ?? opts?.requestId);
+
+	error.message = `Api error with status ${error.statusCode}${opts?.message ? `. ${opts.message}` : ""}`;
+
+	const trailer = [`URL: ${error.url}`, error.requestId ? `Request ID: ${error.requestId}` : undefined]
+		.filter(Boolean)
+		.join(". ");
+
+	if (response.headers.get("Content-Type")?.startsWith("application/json")) {
+		const json = await response.json();
+		error.message = json.error || json.message || error.message;
+		if (json.error_description) {
+			error.message = error.message ? error.message + `: ${json.error_description}` : json.error_description;
+		}
+		error.data = json;
+	} else {
+		error.data = { message: await response.text() };
+	}
+
+	error.message += `. ${trailer}`;
+
+	throw error;
+}
+
+/**
+ * Error thrown when an API call to the Hugging Face Hub fails.
+ */
+export class HubApiError extends Error {
+	statusCode: number;
+	url: string;
+	requestId?: string;
+	data?: JsonObject;
+
+	constructor(url: string, statusCode: number, requestId?: string, message?: string) {
+		super(message);
+
+		this.statusCode = statusCode;
+		this.requestId = requestId;
+		this.url = url;
+	}
+}
+
+export class InvalidApiResponseFormatError extends Error {}