mlclaw 0.2.3 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,7 @@ var __export = (target, all) => {
7
7
 
8
8
  // src/hf-state-sync/hub.ts
9
9
  import fs from "node:fs/promises";
10
+ import path from "node:path";
10
11
 
11
12
  // src/vendor/hfjs-xet/error.ts
12
13
  async function createApiError(response, opts) {
@@ -3828,10 +3829,10 @@ var CurrentXorbInfo = class {
3828
3829
  hash: computeXorbHash(xorbChunksCleaned),
3829
3830
  chunks: xorbChunksCleaned,
3830
3831
  id: this.id,
3831
- files: Object.entries(this.fileProcessedBytes).map(([path6, processedBytes]) => ({
3832
- path: path6,
3833
- progress: processedBytes / this.fileSize[path6],
3834
- lastSentProgress: ((this.fileUploadedBytes[path6] ?? 0) + (processedBytes - (this.fileUploadedBytes[path6] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path6]
3832
+ files: Object.entries(this.fileProcessedBytes).map(([path9, processedBytes]) => ({
3833
+ path: path9,
3834
+ progress: processedBytes / this.fileSize[path9],
3835
+ lastSentProgress: ((this.fileUploadedBytes[path9] ?? 0) + (processedBytes - (this.fileUploadedBytes[path9] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path9]
3835
3836
  }))
3836
3837
  };
3837
3838
  }
@@ -4704,7 +4705,7 @@ var BucketClient = class {
4704
4705
  if (paths.length === 0) {
4705
4706
  return;
4706
4707
  }
4707
- await this.batch(paths.map((path6) => ({ type: "deleteFile", path: path6 })));
4708
+ await this.batch(paths.map((path9) => ({ type: "deleteFile", path: path9 })));
4708
4709
  }
4709
4710
  async batch(operations) {
4710
4711
  const body = `${operations.map((op) => JSON.stringify(op)).join("\n")}
@@ -4720,8 +4721,8 @@ var BucketClient = class {
4720
4721
  * any other failure (including bucket/auth errors), so a missing object is
4721
4722
  * never conflated with an unreachable bucket.
4722
4723
  */
4723
- async downloadFile(path6) {
4724
- const url = `${this.hubUrl}/buckets/${this.bucket}/resolve/${encodeURIComponent(path6)}`;
4724
+ async downloadFile(path9) {
4725
+ const url = `${this.hubUrl}/buckets/${this.bucket}/resolve/${encodeURIComponent(path9)}`;
4725
4726
  const response = await this.fetchWithRetry(url);
4726
4727
  if (response.status === 404) {
4727
4728
  await this.assertBucketAccessible();
@@ -4813,10 +4814,82 @@ function createHfBucketHub(params) {
4813
4814
  }
4814
4815
  };
4815
4816
  }
4817
+ function createMountedBucketHub(params) {
4818
+ const root = path.resolve(params.mountDir);
4819
+ const assertMountRoot = async () => {
4820
+ let stat;
4821
+ try {
4822
+ stat = await fs.stat(root);
4823
+ } catch (err) {
4824
+ if (isNotFound(err)) {
4825
+ throw new Error(`mounted bucket root is missing: ${root}`);
4826
+ }
4827
+ const message = err instanceof Error ? err.message : String(err);
4828
+ throw new Error(`mounted bucket root is not accessible: ${root}: ${message}`);
4829
+ }
4830
+ if (!stat.isDirectory()) {
4831
+ throw new Error(`mounted bucket root is not a directory: ${root}`);
4832
+ }
4833
+ };
4834
+ const localPathFor = (remotePath2) => {
4835
+ const normalized = remotePath2.replace(/^\/+/, "");
4836
+ const resolved = path.resolve(root, normalized);
4837
+ if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) {
4838
+ throw new Error(`invalid bucket path outside mount: ${remotePath2}`);
4839
+ }
4840
+ return resolved;
4841
+ };
4842
+ return {
4843
+ async download(remotePath2, localPath) {
4844
+ await assertMountRoot();
4845
+ const source = localPathFor(remotePath2);
4846
+ try {
4847
+ await fs.copyFile(source, localPath);
4848
+ return "downloaded";
4849
+ } catch (err) {
4850
+ if (isNotFound(err)) {
4851
+ return "not-found";
4852
+ }
4853
+ const message = err instanceof Error ? err.message : String(err);
4854
+ throw new Error(`mounted bucket download failed for ${remotePath2}: ${message}`);
4855
+ }
4856
+ },
4857
+ async upload(localPath, remotePath2) {
4858
+ await assertMountRoot();
4859
+ const target = localPathFor(remotePath2);
4860
+ const dir = path.dirname(target);
4861
+ const tmp = path.join(dir, `.tmp-${path.basename(target)}-${process.pid}-${Date.now()}`);
4862
+ try {
4863
+ await fs.mkdir(dir, { recursive: true });
4864
+ await fs.copyFile(localPath, tmp);
4865
+ await fs.chmod(tmp, 420);
4866
+ await fs.rename(tmp, target);
4867
+ } catch (err) {
4868
+ await fs.rm(tmp, { force: true }).catch(() => void 0);
4869
+ const message = err instanceof Error ? err.message : String(err);
4870
+ throw new Error(`mounted bucket upload failed for ${remotePath2}: ${message}`);
4871
+ }
4872
+ },
4873
+ async delete(remotePaths) {
4874
+ await assertMountRoot();
4875
+ for (const remotePath2 of remotePaths) {
4876
+ try {
4877
+ await fs.rm(localPathFor(remotePath2), { force: true });
4878
+ } catch (err) {
4879
+ const message = err instanceof Error ? err.message : String(err);
4880
+ console.error(`[hf-state-sync] prune failed for ${remotePath2}: ${message}`);
4881
+ }
4882
+ }
4883
+ }
4884
+ };
4885
+ }
4886
+ function isNotFound(err) {
4887
+ return err instanceof Error && "code" in err && err.code === "ENOENT";
4888
+ }
4816
4889
 
4817
4890
  // src/hf-state-sync/paths.ts
4818
4891
  import { randomUUID } from "node:crypto";
4819
- var DEFAULT_LIVE_DIR = "/tmp/openclaw-live";
4892
+ var DEFAULT_LIVE_DIR = "/home/node/.local/share/mlclaw/live";
4820
4893
  var DEFAULT_BUCKET_PREFIX = "openclaw-state";
4821
4894
  var DEFAULT_INTERVAL_SECONDS = 60;
4822
4895
  var DEFAULT_HANDOFF_POLL_SECONDS = 5;
@@ -4827,9 +4900,12 @@ function positiveIntFromEnv(value, fallback) {
4827
4900
  }
4828
4901
  function resolveSyncConfig(env = process.env) {
4829
4902
  const runId = env.MLCLAW_RUN_ID?.trim() || randomUUID();
4903
+ const snapshotUid = nonNegativeIntFromEnv(env.MLCLAW_OPENCLAW_UID);
4904
+ const snapshotGid = nonNegativeIntFromEnv(env.MLCLAW_OPENCLAW_GID);
4830
4905
  return {
4831
4906
  liveDir: env.OPENCLAW_LIVE_DIR?.trim() || DEFAULT_LIVE_DIR,
4832
4907
  bucket: env.OPENCLAW_HF_STATE_BUCKET?.trim() || null,
4908
+ stateMountDir: env.MLCLAW_STATE_MOUNT_DIR?.trim() || null,
4833
4909
  bucketPrefix: normalizeBucketPrefix(env.OPENCLAW_HF_STATE_PREFIX),
4834
4910
  intervalSeconds: positiveIntFromEnv(env.HF_STATE_SYNC_INTERVAL_SECONDS, DEFAULT_INTERVAL_SECONDS),
4835
4911
  handoffPollSeconds: positiveIntFromEnv(env.HF_STATE_SYNC_HANDOFF_POLL_SECONDS, DEFAULT_HANDOFF_POLL_SECONDS),
@@ -4838,9 +4914,16 @@ function resolveSyncConfig(env = process.env) {
4838
4914
  runtimeId: env.MLCLAW_RUNTIME_ID?.trim() || runId,
4839
4915
  agentName: env.OPENCLAW_AGENT_NAME?.trim() || "openclaw",
4840
4916
  gatewayLocation: env.MLCLAW_GATEWAY_LOCATION === "local" || env.MLCLAW_GATEWAY_LOCATION === "space" ? env.MLCLAW_GATEWAY_LOCATION : "unknown",
4841
- runtimeImage: env.MLCLAW_RUNTIME_IMAGE?.trim() || "unknown"
4917
+ runtimeImage: env.MLCLAW_RUNTIME_IMAGE?.trim() || "unknown",
4918
+ ...snapshotUid !== void 0 ? { snapshotUid } : {},
4919
+ ...snapshotGid !== void 0 ? { snapshotGid } : {},
4920
+ ...env.MLCLAW_PROTECTED_STATE_DIR?.trim() ? { protectedStateDir: env.MLCLAW_PROTECTED_STATE_DIR.trim() } : {}
4842
4921
  };
4843
4922
  }
4923
+ function nonNegativeIntFromEnv(value) {
4924
+ const parsed = Number(value);
4925
+ return Number.isInteger(parsed) && parsed >= 0 ? parsed : void 0;
4926
+ }
4844
4927
  function remotePath(config, name) {
4845
4928
  return `${normalizeBucketPrefix(config.bucketPrefix)}/${name.replace(/^\/+/, "")}`;
4846
4929
  }
@@ -4858,20 +4941,20 @@ function logError(message) {
4858
4941
  // src/hf-state-sync/restore.ts
4859
4942
  import fs4 from "node:fs/promises";
4860
4943
  import os from "node:os";
4861
- import path3 from "node:path";
4944
+ import path4 from "node:path";
4862
4945
 
4863
4946
  // src/hf-state-sync/archive.ts
4864
4947
  import { execFile } from "node:child_process";
4865
4948
  import { createHash } from "node:crypto";
4866
4949
  import { createReadStream } from "node:fs";
4867
4950
  import fs3 from "node:fs/promises";
4868
- import path2 from "node:path";
4951
+ import path3 from "node:path";
4869
4952
  import { pipeline } from "node:stream/promises";
4870
4953
  import { promisify } from "node:util";
4871
4954
 
4872
4955
  // src/hf-state-sync/sqlite.ts
4873
4956
  import fs2 from "node:fs/promises";
4874
- import path from "node:path";
4957
+ import path2 from "node:path";
4875
4958
  import { DatabaseSync } from "node:sqlite";
4876
4959
  async function findSqliteFiles(root) {
4877
4960
  const found = [];
@@ -4883,7 +4966,7 @@ async function findSqliteFiles(root) {
4883
4966
  }
4884
4967
  for (const entry of entries) {
4885
4968
  if (entry.isFile() && entry.name.endsWith(".sqlite")) {
4886
- found.push(path.join(entry.parentPath, entry.name));
4969
+ found.push(path2.join(entry.parentPath, entry.name));
4887
4970
  }
4888
4971
  }
4889
4972
  return found.sort();
@@ -4919,6 +5002,7 @@ var STATE_EXCLUDED_NAMES = /* @__PURE__ */ new Set([".env", "credentials", "tmp"
4919
5002
  var STATE_EXCLUDED_SUFFIXES = [".log"];
4920
5003
  var SIDECAR_SUFFIXES = [".sqlite-wal", ".sqlite-shm"];
4921
5004
  var STATE_DIR_NAME = ".openclaw";
5005
+ var PROTECTED_STATE_DIR_NAME = ".mlclaw-protected";
4922
5006
  function isExcluded(name, inStateDir) {
4923
5007
  if (SIDECAR_SUFFIXES.some((suffix) => name.endsWith(suffix))) {
4924
5008
  return true;
@@ -4929,15 +5013,18 @@ function isExcluded(name, inStateDir) {
4929
5013
  return STATE_EXCLUDED_NAMES.has(name) || STATE_EXCLUDED_SUFFIXES.some((suffix) => name.endsWith(suffix));
4930
5014
  }
4931
5015
  async function copyTreeFiltered(params) {
4932
- const { sourceDir, destDir, databases, rootDir, inStateDir, depth } = params;
5016
+ const { sourceDir, destDir, databases, rootDir, inStateDir, depth, excludeProtectedState } = params;
4933
5017
  await fs3.mkdir(destDir, { recursive: true });
4934
5018
  const entries = await fs3.readdir(sourceDir, { withFileTypes: true });
4935
5019
  for (const entry of entries) {
5020
+ if (excludeProtectedState && depth === 0 && entry.name === PROTECTED_STATE_DIR_NAME) {
5021
+ continue;
5022
+ }
4936
5023
  if (isExcluded(entry.name, inStateDir)) {
4937
5024
  continue;
4938
5025
  }
4939
- const source = path2.join(sourceDir, entry.name);
4940
- const dest = path2.join(destDir, entry.name);
5026
+ const source = path3.join(sourceDir, entry.name);
5027
+ const dest = path3.join(destDir, entry.name);
4941
5028
  if (entry.isDirectory()) {
4942
5029
  await copyTreeFiltered({
4943
5030
  sourceDir: source,
@@ -4947,11 +5034,12 @@ async function copyTreeFiltered(params) {
4947
5034
  // Only the top-level .openclaw dir is OpenClaw state; a workspace
4948
5035
  // project may legitimately contain its own .openclaw directory.
4949
5036
  inStateDir: inStateDir || depth === 0 && entry.name === STATE_DIR_NAME,
4950
- depth: depth + 1
5037
+ depth: depth + 1,
5038
+ excludeProtectedState
4951
5039
  });
4952
5040
  } else if (entry.isFile()) {
4953
5041
  if (entry.name.endsWith(".sqlite")) {
4954
- databases.push(path2.relative(rootDir, source));
5042
+ databases.push(path3.relative(rootDir, source));
4955
5043
  } else {
4956
5044
  await fs3.copyFile(source, dest);
4957
5045
  }
@@ -4960,7 +5048,7 @@ async function copyTreeFiltered(params) {
4960
5048
  }
4961
5049
  }
4962
5050
  }
4963
- async function stageLiveDir(liveDir, stagingDir) {
5051
+ async function stageLiveDir(liveDir, stagingDir, options = {}) {
4964
5052
  const databases = [];
4965
5053
  await copyTreeFiltered({
4966
5054
  sourceDir: liveDir,
@@ -4968,12 +5056,13 @@ async function stageLiveDir(liveDir, stagingDir) {
4968
5056
  databases,
4969
5057
  rootDir: liveDir,
4970
5058
  inStateDir: false,
4971
- depth: 0
5059
+ depth: 0,
5060
+ excludeProtectedState: options.excludeProtectedState ?? false
4972
5061
  });
4973
5062
  for (const relative of databases) {
4974
- const staged = path2.join(stagingDir, relative);
4975
- await fs3.mkdir(path2.dirname(staged), { recursive: true });
4976
- vacuumInto(path2.join(liveDir, relative), staged);
5063
+ const staged = path3.join(stagingDir, relative);
5064
+ await fs3.mkdir(path3.dirname(staged), { recursive: true });
5065
+ vacuumInto(path3.join(liveDir, relative), staged);
4977
5066
  const integrity = checkIntegrity(staged);
4978
5067
  if (integrity.kind === "corrupt") {
4979
5068
  return { kind: "corrupt-database", database: relative, detail: integrity.detail };
@@ -5480,8 +5569,8 @@ function getErrorMap() {
5480
5569
 
5481
5570
  // node_modules/zod/v3/helpers/parseUtil.js
5482
5571
  var makeIssue = (params) => {
5483
- const { data, path: path6, errorMaps, issueData } = params;
5484
- const fullPath = [...path6, ...issueData.path || []];
5572
+ const { data, path: path9, errorMaps, issueData } = params;
5573
+ const fullPath = [...path9, ...issueData.path || []];
5485
5574
  const fullIssue = {
5486
5575
  ...issueData,
5487
5576
  path: fullPath
@@ -5597,11 +5686,11 @@ var errorUtil;
5597
5686
 
5598
5687
  // node_modules/zod/v3/types.js
5599
5688
  var ParseInputLazyPath = class {
5600
- constructor(parent, value, path6, key) {
5689
+ constructor(parent, value, path9, key) {
5601
5690
  this._cachedPath = [];
5602
5691
  this.parent = parent;
5603
5692
  this.data = value;
5604
- this._path = path6;
5693
+ this._path = path9;
5605
5694
  this._key = key;
5606
5695
  }
5607
5696
  get path() {
@@ -9086,7 +9175,7 @@ function promoteSnapshot(params) {
9086
9175
  // src/hf-state-sync/restore.ts
9087
9176
  async function tryRestoreEntry(params) {
9088
9177
  const { hub, entry, workDir, liveDir } = params;
9089
- const archivePath = path3.join(workDir, `candidate-${entry.id}.tar.zst`);
9178
+ const archivePath = path4.join(workDir, `candidate-${entry.id}.tar.zst`);
9090
9179
  const downloaded = await hub.download(entry.path, archivePath);
9091
9180
  if (downloaded === "not-found") {
9092
9181
  logError(`snapshot ${entry.id} missing from bucket`);
@@ -9110,11 +9199,11 @@ async function tryRestoreEntry(params) {
9110
9199
  for (const database of await findSqliteFiles(extractDir)) {
9111
9200
  const integrity = checkIntegrity(database);
9112
9201
  if (integrity.kind === "corrupt") {
9113
- logError(`snapshot ${entry.id} db ${path3.basename(database)} corrupt: ${integrity.detail}`);
9202
+ logError(`snapshot ${entry.id} db ${path4.basename(database)} corrupt: ${integrity.detail}`);
9114
9203
  return "failed";
9115
9204
  }
9116
9205
  }
9117
- await fs4.mkdir(path3.dirname(liveDir), { recursive: true });
9206
+ await fs4.mkdir(path4.dirname(liveDir), { recursive: true });
9118
9207
  await fs4.rename(extractDir, liveDir);
9119
9208
  return "restored";
9120
9209
  } finally {
@@ -9131,9 +9220,9 @@ async function runRestore(params) {
9131
9220
  return { kind: "fresh-start", reason: "live-dir-exists" };
9132
9221
  } catch {
9133
9222
  }
9134
- const workDir = await fs4.mkdtemp(path3.join(os.tmpdir(), "hf-state-restore-"));
9223
+ const workDir = await fs4.mkdtemp(path4.join(os.tmpdir(), "hf-state-restore-"));
9135
9224
  try {
9136
- const manifestPath = path3.join(workDir, "manifest.json");
9225
+ const manifestPath = path4.join(workDir, "manifest.json");
9137
9226
  const downloaded = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), manifestPath);
9138
9227
  if (downloaded === "not-found") {
9139
9228
  return { kind: "fresh-start", reason: "no-manifest" };
@@ -9157,10 +9246,84 @@ async function runRestore(params) {
9157
9246
  }
9158
9247
  }
9159
9248
 
9160
- // src/hf-state-sync/snapshot.ts
9249
+ // src/hf-state-sync/prepare.ts
9161
9250
  import fs5 from "node:fs/promises";
9251
+ import path5 from "node:path";
9252
+ async function prepareRestore(config) {
9253
+ if (config.snapshotUid === void 0 || config.snapshotGid === void 0) {
9254
+ throw new Error("restore preparation requires MLCLAW_OPENCLAW_UID and MLCLAW_OPENCLAW_GID");
9255
+ }
9256
+ const liveParent = path5.dirname(config.liveDir);
9257
+ await fs5.mkdir(liveParent, { recursive: true });
9258
+ await fs5.chown(liveParent, config.snapshotUid, config.snapshotGid);
9259
+ if (!config.stateMountDir) {
9260
+ return;
9261
+ }
9262
+ await makeTraversableDirectory(config.stateMountDir);
9263
+ const prefixRoot = confinedPath(config.stateMountDir, config.bucketPrefix);
9264
+ let prefixPart = config.stateMountDir;
9265
+ for (const part of config.bucketPrefix.split("/").filter(Boolean)) {
9266
+ prefixPart = path5.join(prefixPart, part);
9267
+ await makeTraversableDirectory(prefixPart);
9268
+ }
9269
+ await makeReadableIfFile(path5.join(prefixRoot, "manifest.json"));
9270
+ const snapshotsDir = path5.join(prefixRoot, "snapshots");
9271
+ await makeTraversableDirectory(snapshotsDir);
9272
+ let entries;
9273
+ try {
9274
+ entries = await fs5.readdir(snapshotsDir, { withFileTypes: true });
9275
+ } catch (err) {
9276
+ if (isNotFound2(err)) {
9277
+ return;
9278
+ }
9279
+ throw err;
9280
+ }
9281
+ for (const entry of entries) {
9282
+ if (entry.isFile()) {
9283
+ await fs5.chmod(path5.join(snapshotsDir, entry.name), 420);
9284
+ }
9285
+ }
9286
+ }
9287
+ async function makeTraversableDirectory(directory) {
9288
+ try {
9289
+ const stat = await fs5.lstat(directory);
9290
+ if (stat.isDirectory()) {
9291
+ await fs5.chmod(directory, 457);
9292
+ }
9293
+ } catch (err) {
9294
+ if (!isNotFound2(err)) {
9295
+ throw err;
9296
+ }
9297
+ }
9298
+ }
9299
+ function confinedPath(root, relative) {
9300
+ const resolvedRoot = path5.resolve(root);
9301
+ const resolved = path5.resolve(resolvedRoot, relative);
9302
+ if (resolved !== resolvedRoot && !resolved.startsWith(`${resolvedRoot}${path5.sep}`)) {
9303
+ throw new Error(`invalid state prefix outside mounted bucket: ${relative}`);
9304
+ }
9305
+ return resolved;
9306
+ }
9307
+ async function makeReadableIfFile(file) {
9308
+ try {
9309
+ const stat = await fs5.lstat(file);
9310
+ if (stat.isFile()) {
9311
+ await fs5.chmod(file, 420);
9312
+ }
9313
+ } catch (err) {
9314
+ if (!isNotFound2(err)) {
9315
+ throw err;
9316
+ }
9317
+ }
9318
+ }
9319
+ function isNotFound2(err) {
9320
+ return err instanceof Error && "code" in err && err.code === "ENOENT";
9321
+ }
9322
+
9323
+ // src/hf-state-sync/snapshot.ts
9324
+ import fs6 from "node:fs/promises";
9162
9325
  import os2 from "node:os";
9163
- import path4 from "node:path";
9326
+ import path6 from "node:path";
9164
9327
  var snapshotCounter = 0;
9165
9328
  function snapshotId(now, runId) {
9166
9329
  snapshotCounter += 1;
@@ -9168,12 +9331,12 @@ function snapshotId(now, runId) {
9168
9331
  return `${stamp}-${runId.slice(0, 8)}-${snapshotCounter}`;
9169
9332
  }
9170
9333
  async function fetchManifest(config, hub, workDir) {
9171
- const localPath = path4.join(workDir, "manifest.remote.json");
9334
+ const localPath = path6.join(workDir, "manifest.remote.json");
9172
9335
  const result = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), localPath);
9173
9336
  if (result === "not-found") {
9174
9337
  return { kind: "none" };
9175
9338
  }
9176
- const parsed = parseManifest(await fs5.readFile(localPath, "utf8"));
9339
+ const parsed = parseManifest(await fs6.readFile(localPath, "utf8"));
9177
9340
  return parsed.kind === "ok" ? { kind: "ok", manifest: parsed.manifest } : { kind: "invalid", reason: parsed.reason };
9178
9341
  }
9179
9342
  async function runSnapshot(params) {
@@ -9182,31 +9345,29 @@ async function runSnapshot(params) {
9182
9345
  return { kind: "skipped", reason: "no-bucket" };
9183
9346
  }
9184
9347
  try {
9185
- await fs5.access(config.liveDir);
9348
+ await fs6.access(config.liveDir);
9186
9349
  } catch {
9187
9350
  return { kind: "skipped", reason: "empty-state" };
9188
9351
  }
9189
- const workDir = await fs5.mkdtemp(path4.join(os2.tmpdir(), "hf-state-snapshot-"));
9352
+ const workDir = await fs6.mkdtemp(path6.join(os2.tmpdir(), "hf-state-snapshot-"));
9190
9353
  try {
9191
- const stagingDir = path4.join(workDir, "stage");
9192
- const staged = await stageLiveDir(config.liveDir, stagingDir);
9354
+ const now = (params.now ?? (() => /* @__PURE__ */ new Date()))();
9355
+ const id = snapshotId(now, config.runId);
9356
+ const archiveName = `state-${id}.tar.zst`;
9357
+ const archivePath = path6.join(workDir, archiveName);
9358
+ const staged = params.stageArchive ? await params.stageArchive({ liveDir: config.liveDir, archivePath }) : await stageArchiveInProcess(config.liveDir, path6.join(workDir, "stage"), archivePath);
9193
9359
  if (staged.kind === "corrupt-database") {
9194
9360
  return {
9195
9361
  kind: "failed",
9196
9362
  detail: `live database ${staged.database} failed integrity check: ${staged.detail}`
9197
9363
  };
9198
9364
  }
9199
- const now = (params.now ?? (() => /* @__PURE__ */ new Date()))();
9200
- const id = snapshotId(now, config.runId);
9201
- const archiveName = `state-${id}.tar.zst`;
9202
- const archivePath = path4.join(workDir, archiveName);
9203
- await createTarZst(stagingDir, archivePath);
9204
9365
  const entry = {
9205
9366
  id,
9206
9367
  path: remotePath(config, `snapshots/${archiveName}`),
9207
9368
  createdAt: now.toISOString(),
9208
9369
  sha256: await sha256File(archivePath),
9209
- sizeBytes: (await fs5.stat(archivePath)).size,
9370
+ sizeBytes: (await fs6.stat(archivePath)).size,
9210
9371
  runId: config.runId,
9211
9372
  bootTime: params.bootTime
9212
9373
  };
@@ -9223,27 +9384,182 @@ async function runSnapshot(params) {
9223
9384
  entry,
9224
9385
  keep: config.keepSnapshots
9225
9386
  });
9226
- const manifestPath = path4.join(workDir, "manifest.json");
9227
- await fs5.writeFile(manifestPath, serializeManifest(manifest));
9387
+ const manifestPath = path6.join(workDir, "manifest.json");
9388
+ await fs6.writeFile(manifestPath, serializeManifest(manifest));
9228
9389
  await hub.upload(manifestPath, remotePath(config, MANIFEST_REMOTE_NAME));
9229
9390
  if (expired.length > 0) {
9230
9391
  await hub.delete(expired.map((e) => e.path));
9231
9392
  }
9232
- log(`snapshot ${entry.id} uploaded (${entry.sizeBytes} bytes, ${staged.databases.length} dbs)`);
9393
+ log(`snapshot ${entry.id} uploaded (${entry.sizeBytes} bytes, ${staged.databaseCount} dbs)`);
9233
9394
  return { kind: "uploaded", entry };
9234
9395
  } catch (err) {
9235
9396
  return { kind: "failed", detail: err instanceof Error ? err.message : String(err) };
9236
9397
  } finally {
9237
- await fs5.rm(workDir, { recursive: true, force: true });
9398
+ await fs6.rm(workDir, { recursive: true, force: true });
9399
+ }
9400
+ }
9401
+ async function stageArchiveInProcess(liveDir, stagingDir, archivePath) {
9402
+ const staged = await stageLiveDir(liveDir, stagingDir);
9403
+ if (staged.kind === "corrupt-database") {
9404
+ return staged;
9238
9405
  }
9406
+ await createTarZst(stagingDir, archivePath);
9407
+ return { kind: "staged", databaseCount: staged.databases.length };
9239
9408
  }
9240
9409
 
9241
9410
  // src/hf-state-sync/supervise.ts
9411
+ import { spawn as spawn2 } from "node:child_process";
9412
+ import fs8 from "node:fs/promises";
9413
+ import os4 from "node:os";
9414
+ import path8 from "node:path";
9415
+ import { setTimeout as delay } from "node:timers/promises";
9416
+
9417
+ // src/hf-state-sync/stage-worker.ts
9242
9418
  import { spawn } from "node:child_process";
9243
- import fs6 from "node:fs/promises";
9419
+ import { createReadStream as createReadStream2, createWriteStream, writeFileSync } from "node:fs";
9420
+ import fs7 from "node:fs/promises";
9244
9421
  import os3 from "node:os";
9245
- import path5 from "node:path";
9246
- import { setTimeout as delay } from "node:timers/promises";
9422
+ import path7 from "node:path";
9423
+ import { pipeline as pipeline2 } from "node:stream/promises";
9424
+ function unprivilegedStageArchive(params) {
9425
+ return async ({ liveDir, archivePath }) => {
9426
+ const child = spawn(process.execPath, [params.scriptPath, "stage-worker", liveDir], {
9427
+ uid: params.uid,
9428
+ gid: params.gid,
9429
+ env: snapshotWorkerEnvironment(process.env),
9430
+ stdio: ["ignore", "pipe", "inherit", "pipe"]
9431
+ });
9432
+ if (!child.stdout || !child.stdio[3]) {
9433
+ child.kill("SIGKILL");
9434
+ throw new Error("snapshot staging worker pipes are unavailable");
9435
+ }
9436
+ const archiveOutput = createWriteStream(archivePath, { flags: "wx", mode: 384 });
9437
+ const metadata = collect(child.stdio[3]);
9438
+ const archive = pipeline2(child.stdout, archiveOutput);
9439
+ const exitCode = await new Promise((resolve, reject) => {
9440
+ child.once("error", reject);
9441
+ child.once("exit", (code, signal) => resolve(code ?? (signal ? 128 : 1)));
9442
+ });
9443
+ await archive;
9444
+ const message = parseWorkerMessage(await metadata);
9445
+ if (exitCode !== 0 || message.kind === "failed") {
9446
+ throw new Error(
9447
+ message.kind === "failed" ? message.detail : `snapshot staging worker exited with code ${exitCode}`
9448
+ );
9449
+ }
9450
+ return message;
9451
+ };
9452
+ }
9453
+ function protectedStageArchive(params) {
9454
+ return async (request) => {
9455
+ const outcome = await params.base(request);
9456
+ if (outcome.kind !== "staged") {
9457
+ return outcome;
9458
+ }
9459
+ const workDir = await fs7.mkdtemp(path7.join(os3.tmpdir(), "hf-state-protected-stage-"));
9460
+ try {
9461
+ const stagingDir = path7.join(workDir, "stage");
9462
+ await extractTarZst(request.archivePath, stagingDir);
9463
+ const destination = path7.join(stagingDir, params.archiveName);
9464
+ await fs7.cp(params.sourceDir, destination, {
9465
+ recursive: true,
9466
+ force: false,
9467
+ preserveTimestamps: true,
9468
+ filter: (source) => includeProtectedSnapshotPath(params.sourceDir, source)
9469
+ });
9470
+ await fs7.chmod(destination, 448);
9471
+ await fs7.rm(request.archivePath, { force: true });
9472
+ await createTarZst(stagingDir, request.archivePath);
9473
+ await fs7.chmod(request.archivePath, 384);
9474
+ return outcome;
9475
+ } finally {
9476
+ await fs7.rm(workDir, { recursive: true, force: true });
9477
+ }
9478
+ };
9479
+ }
9480
+ function includeProtectedSnapshotPath(sourceDir, source) {
9481
+ const relative = path7.relative(sourceDir, source);
9482
+ return relative !== "hf-broker/mirrors" && !relative.startsWith(`hf-broker/mirrors${path7.sep}`);
9483
+ }
9484
+ function trustedStageArchive(config, scriptPath) {
9485
+ const canStageAsOpenClaw = process.getuid?.() === 0 && Boolean(scriptPath) && config.snapshotUid !== void 0 && config.snapshotGid !== void 0;
9486
+ if (!canStageAsOpenClaw) {
9487
+ if (config.protectedStateDir) {
9488
+ throw new Error("protected runtime state requires root snapshot staging with an OpenClaw UID and GID");
9489
+ }
9490
+ return void 0;
9491
+ }
9492
+ let stageArchive = unprivilegedStageArchive({
9493
+ uid: config.snapshotUid,
9494
+ gid: config.snapshotGid,
9495
+ scriptPath
9496
+ });
9497
+ if (config.protectedStateDir) {
9498
+ stageArchive = protectedStageArchive({
9499
+ base: stageArchive,
9500
+ sourceDir: config.protectedStateDir,
9501
+ archiveName: PROTECTED_STATE_DIR_NAME
9502
+ });
9503
+ }
9504
+ return stageArchive;
9505
+ }
9506
+ async function runStageWorker(liveDir) {
9507
+ const workDir = await fs7.mkdtemp(path7.join(os3.tmpdir(), "hf-state-stage-worker-"));
9508
+ try {
9509
+ const stagingDir = path7.join(workDir, "stage");
9510
+ const archivePath = path7.join(workDir, "snapshot.tar.zst");
9511
+ const staged = await stageLiveDir(liveDir, stagingDir, { excludeProtectedState: true });
9512
+ if (staged.kind === "corrupt-database") {
9513
+ writeWorkerMessage(staged);
9514
+ return 0;
9515
+ }
9516
+ await createTarZst(stagingDir, archivePath);
9517
+ writeWorkerMessage({ kind: "staged", databaseCount: staged.databases.length });
9518
+ await pipeline2(createReadStream2(archivePath), process.stdout);
9519
+ return 0;
9520
+ } catch (err) {
9521
+ writeWorkerMessage({ kind: "failed", detail: err instanceof Error ? err.message : String(err) });
9522
+ return 1;
9523
+ } finally {
9524
+ await fs7.rm(workDir, { recursive: true, force: true });
9525
+ }
9526
+ }
9527
+ function snapshotWorkerEnvironment(env) {
9528
+ return {
9529
+ HOME: "/home/node",
9530
+ PATH: env.PATH,
9531
+ TMPDIR: env.TMPDIR
9532
+ };
9533
+ }
9534
+ function writeWorkerMessage(message) {
9535
+ writeFileSync(3, `${JSON.stringify(message)}
9536
+ `);
9537
+ }
9538
+ async function collect(stream) {
9539
+ if (!stream) {
9540
+ throw new Error("snapshot staging worker metadata pipe is unavailable");
9541
+ }
9542
+ const chunks = [];
9543
+ for await (const chunk of stream) {
9544
+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
9545
+ }
9546
+ return Buffer.concat(chunks).toString("utf8");
9547
+ }
9548
+ function parseWorkerMessage(raw) {
9549
+ const parsed = JSON.parse(raw);
9550
+ if (parsed.kind === "staged" && Number.isInteger(parsed.databaseCount) && parsed.databaseCount >= 0) {
9551
+ return parsed;
9552
+ }
9553
+ if (parsed.kind === "corrupt-database" && parsed.database && parsed.detail) {
9554
+ return parsed;
9555
+ }
9556
+ if (parsed.kind === "failed" && parsed.detail) {
9557
+ return parsed;
9558
+ }
9559
+ throw new Error("snapshot staging worker returned invalid metadata");
9560
+ }
9561
+
9562
+ // src/hf-state-sync/supervise.ts
9247
9563
  var LEASE_HEARTBEAT_MS = 6e4;
9248
9564
  var HANDOFF_REQUEST_TTL_MS = 10 * 60 * 1e3;
9249
9565
  async function supervise(params) {
@@ -9253,6 +9569,8 @@ async function supervise(params) {
9253
9569
  throw new Error("supervise: missing child command");
9254
9570
  }
9255
9571
  const bootTime = (/* @__PURE__ */ new Date()).toISOString();
9572
+ const scriptPath = process.argv[1];
9573
+ const stageArchive = trustedStageArchive(config, scriptPath);
9256
9574
  let lastSnapshotId;
9257
9575
  const handoffState = { request: null };
9258
9576
  const writeLease = async () => {
@@ -9266,24 +9584,24 @@ async function supervise(params) {
9266
9584
  lastHeartbeatAt: (/* @__PURE__ */ new Date()).toISOString(),
9267
9585
  ...lastSnapshotId ? { lastSnapshotId } : {}
9268
9586
  };
9269
- const tmpDir = await fs6.mkdtemp(path5.join(os3.tmpdir(), "hf-state-lease-"));
9587
+ const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-lease-"));
9270
9588
  try {
9271
- const file = path5.join(tmpDir, "status.json");
9272
- await fs6.writeFile(file, JSON.stringify(status, null, 2) + "\n");
9589
+ const file = path8.join(tmpDir, "status.json");
9590
+ await fs8.writeFile(file, JSON.stringify(status, null, 2) + "\n");
9273
9591
  await hub.upload(file, remotePath(config, "runtime/status.json"));
9274
9592
  } finally {
9275
- await fs6.rm(tmpDir, { recursive: true, force: true });
9593
+ await fs8.rm(tmpDir, { recursive: true, force: true });
9276
9594
  }
9277
9595
  };
9278
9596
  const readHandoffRequest = async () => {
9279
- const tmpDir = await fs6.mkdtemp(path5.join(os3.tmpdir(), "hf-state-handoff-"));
9597
+ const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-handoff-"));
9280
9598
  try {
9281
- const file = path5.join(tmpDir, "request.json");
9599
+ const file = path8.join(tmpDir, "request.json");
9282
9600
  const result = await hub.download(remotePath(config, "runtime/handoff-request.json"), file);
9283
9601
  if (result === "not-found") {
9284
9602
  return null;
9285
9603
  }
9286
- const parsed = JSON.parse(await fs6.readFile(file, "utf8"));
9604
+ const parsed = JSON.parse(await fs8.readFile(file, "utf8"));
9287
9605
  if (parsed?.schemaVersion !== 1 || parsed.agent !== config.agentName || parsed.runtimeId !== config.runtimeId || typeof parsed.requestedAt !== "string" || typeof parsed.requestId !== "string" || !parsed.requestId) {
9288
9606
  return null;
9289
9607
  }
@@ -9297,7 +9615,7 @@ async function supervise(params) {
9297
9615
  }
9298
9616
  return parsed;
9299
9617
  } finally {
9300
- await fs6.rm(tmpDir, { recursive: true, force: true });
9618
+ await fs8.rm(tmpDir, { recursive: true, force: true });
9301
9619
  }
9302
9620
  };
9303
9621
  const writeHandoffAck = async (request) => {
@@ -9310,17 +9628,20 @@ async function supervise(params) {
9310
9628
  completedAt: (/* @__PURE__ */ new Date()).toISOString(),
9311
9629
  ...lastSnapshotId ? { lastSnapshotId } : {}
9312
9630
  };
9313
- const tmpDir = await fs6.mkdtemp(path5.join(os3.tmpdir(), "hf-state-handoff-ack-"));
9631
+ const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-handoff-ack-"));
9314
9632
  try {
9315
- const file = path5.join(tmpDir, "ack.json");
9316
- await fs6.writeFile(file, JSON.stringify(ack, null, 2) + "\n");
9633
+ const file = path8.join(tmpDir, "ack.json");
9634
+ await fs8.writeFile(file, JSON.stringify(ack, null, 2) + "\n");
9317
9635
  await hub.upload(file, remotePath(config, "runtime/handoff-ack.json"));
9318
9636
  await hub.delete([remotePath(config, "runtime/handoff-request.json")]);
9319
9637
  } finally {
9320
- await fs6.rm(tmpDir, { recursive: true, force: true });
9638
+ await fs8.rm(tmpDir, { recursive: true, force: true });
9321
9639
  }
9322
9640
  };
9323
- const child = spawn(binary, args, { stdio: "inherit" });
9641
+ const child = spawn2(binary, args, {
9642
+ stdio: "inherit",
9643
+ env: supervisedChildEnvironment(process.env)
9644
+ });
9324
9645
  const childExit = new Promise((resolve) => {
9325
9646
  child.on("exit", (code, signal) => resolve(code ?? (signal ? 128 : 1)));
9326
9647
  child.on("error", (err) => {
@@ -9332,7 +9653,12 @@ async function supervise(params) {
9332
9653
  let inFlight = null;
9333
9654
  const runOnce = async (label) => {
9334
9655
  try {
9335
- const outcome = await runSnapshot({ config, hub, bootTime });
9656
+ const outcome = await runSnapshot({
9657
+ config,
9658
+ hub,
9659
+ bootTime,
9660
+ ...stageArchive ? { stageArchive } : {}
9661
+ });
9336
9662
  if (outcome.kind === "failed") {
9337
9663
  logError(`${label}: snapshot failed: ${outcome.detail}`);
9338
9664
  } else if (outcome.kind === "uploaded") {
@@ -9372,7 +9698,9 @@ async function supervise(params) {
9372
9698
  })();
9373
9699
  void snapshotLoop;
9374
9700
  const heartbeatLoop = (async () => {
9375
- await writeLease().catch((err) => logError(`initial lease failed: ${err instanceof Error ? err.message : String(err)}`));
9701
+ await writeLease().catch(
9702
+ (err) => logError(`initial lease failed: ${err instanceof Error ? err.message : String(err)}`)
9703
+ );
9376
9704
  while (!stopping) {
9377
9705
  await delay(LEASE_HEARTBEAT_MS);
9378
9706
  if (stopping) {
@@ -9430,16 +9758,25 @@ async function supervise(params) {
9430
9758
  if (handoffState.request) {
9431
9759
  const request = handoffState.request;
9432
9760
  if (finalOutcome.kind !== "uploaded") {
9433
- throw new Error(`handoff ${request.requestId} final snapshot did not upload: ${snapshotFailureDetail(finalOutcome)}`);
9761
+ throw new Error(
9762
+ `handoff ${request.requestId} final snapshot did not upload: ${snapshotFailureDetail(finalOutcome)}`
9763
+ );
9434
9764
  }
9435
9765
  await writeHandoffAck(request).catch((err) => {
9436
- throw new Error(`handoff ${request.requestId} snapshot completed but ack failed: ${err instanceof Error ? err.message : String(err)}`);
9766
+ throw new Error(
9767
+ `handoff ${request.requestId} snapshot completed but ack failed: ${err instanceof Error ? err.message : String(err)}`
9768
+ );
9437
9769
  });
9438
9770
  log(`handoff ${request.requestId} acknowledged`);
9439
9771
  return 0;
9440
9772
  }
9441
9773
  return exitCode;
9442
9774
  }
9775
+ function supervisedChildEnvironment(source) {
9776
+ const env = { ...source };
9777
+ delete env.MLCLAW_STATE_HF_TOKEN;
9778
+ return env;
9779
+ }
9443
9780
  function snapshotFailureDetail(outcome) {
9444
9781
  switch (outcome.kind) {
9445
9782
  case "uploaded":
@@ -9456,17 +9793,36 @@ var USAGE = `usage:
9456
9793
  hf-state-sync restore
9457
9794
  hf-state-sync snapshot
9458
9795
  hf-state-sync supervise -- <command> [args...]`;
9459
- function makeHub(bucket) {
9460
- return bucket ? createHfBucketHub({ bucket }) : null;
9796
+ function makeHub(config) {
9797
+ if (!config.bucket) {
9798
+ return null;
9799
+ }
9800
+ if (config.stateMountDir) {
9801
+ log(`using mounted state bucket at ${config.stateMountDir}`);
9802
+ return createMountedBucketHub({ mountDir: config.stateMountDir });
9803
+ }
9804
+ const token = process.env.MLCLAW_STATE_HF_TOKEN ?? process.env.HF_TOKEN;
9805
+ return createHfBucketHub({ bucket: config.bucket, ...token ? { token } : {} });
9461
9806
  }
9462
9807
  async function main(argv) {
9808
+ if (argv[0] === "stage-worker") {
9809
+ const liveDir = argv[1];
9810
+ if (!liveDir) {
9811
+ logError("stage-worker: missing live directory");
9812
+ return 2;
9813
+ }
9814
+ return runStageWorker(liveDir);
9815
+ }
9463
9816
  const config = resolveSyncConfig();
9464
- const hub = makeHub(config.bucket);
9817
+ const hub = makeHub(config);
9465
9818
  if (!hub) {
9466
9819
  logError("OPENCLAW_HF_STATE_BUCKET is not set; state will NOT survive restarts");
9467
9820
  }
9468
9821
  const mode = argv[0];
9469
9822
  switch (mode) {
9823
+ case "prepare-restore":
9824
+ await prepareRestore(config);
9825
+ return 0;
9470
9826
  case "restore": {
9471
9827
  if (!hub) {
9472
9828
  return 0;
@@ -9491,7 +9847,13 @@ async function main(argv) {
9491
9847
  if (!hub) {
9492
9848
  return 1;
9493
9849
  }
9494
- const outcome = await runSnapshot({ config, hub, bootTime: (/* @__PURE__ */ new Date()).toISOString() });
9850
+ const stageArchive = trustedStageArchive(config, process.argv[1]);
9851
+ const outcome = await runSnapshot({
9852
+ config,
9853
+ hub,
9854
+ bootTime: (/* @__PURE__ */ new Date()).toISOString(),
9855
+ ...stageArchive ? { stageArchive } : {}
9856
+ });
9495
9857
  if (outcome.kind === "failed") {
9496
9858
  logError(outcome.detail);
9497
9859
  return 1;
@@ -9507,8 +9869,8 @@ async function main(argv) {
9507
9869
  return 2;
9508
9870
  }
9509
9871
  if (!hub) {
9510
- const { spawn: spawn2 } = await import("node:child_process");
9511
- const child = spawn2(command[0], command.slice(1), { stdio: "inherit" });
9872
+ const { spawn: spawn3 } = await import("node:child_process");
9873
+ const child = spawn3(command[0], command.slice(1), { stdio: "inherit" });
9512
9874
  return await new Promise((resolve) => {
9513
9875
  process.on("SIGTERM", () => child.kill("SIGTERM"));
9514
9876
  process.on("SIGINT", () => child.kill("SIGINT"));