mlclaw 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/mlclaw/SKILL.md +58 -19
- package/Dockerfile +46 -8
- package/README.md +75 -17
- package/assets/mlclaw-control-ui/assets/index-92PjW54g.js +9 -0
- package/assets/mlclaw-control-ui/assets/index-D1bHaYpf.css +1 -0
- package/assets/mlclaw-control-ui/index.html +2 -2
- package/dist/hf-state-sync.js +438 -76
- package/dist/hf-tooling-seed.js +64 -11
- package/dist/mlclaw-space-runtime.js +8403 -5935
- package/dist/mlclaw.mjs +406 -42
- package/entrypoint.sh +167 -17
- package/hf-broker.scope.json +38 -0
- package/package.json +26 -2
- package/space/README.md +14 -4
- package/src/hf-state-sync/archive.ts +15 -8
- package/src/hf-state-sync/cli.ts +33 -6
- package/src/hf-state-sync/hub.ts +82 -1
- package/src/hf-state-sync/paths.ts +23 -4
- package/src/hf-state-sync/prepare.ts +79 -0
- package/src/hf-state-sync/snapshot.ts +31 -9
- package/src/hf-state-sync/stage-worker.ts +165 -0
- package/src/hf-state-sync/supervise.ts +29 -10
- package/src/vendor/hfjs-xet/utils/ChunkCache.ts +1 -1
- package/src/vendor/hfjs-xet/utils/XetBlob.ts +2 -2
- package/src/vendor/hfjs-xet/utils/createXorbs.ts +2 -2
- package/assets/mlclaw-control-ui/assets/index-D2TFes32.js +0 -9
- package/assets/mlclaw-control-ui/assets/index-DP72PFuv.css +0 -1
package/dist/hf-state-sync.js
CHANGED
|
@@ -7,6 +7,7 @@ var __export = (target, all) => {
|
|
|
7
7
|
|
|
8
8
|
// src/hf-state-sync/hub.ts
|
|
9
9
|
import fs from "node:fs/promises";
|
|
10
|
+
import path from "node:path";
|
|
10
11
|
|
|
11
12
|
// src/vendor/hfjs-xet/error.ts
|
|
12
13
|
async function createApiError(response, opts) {
|
|
@@ -3828,10 +3829,10 @@ var CurrentXorbInfo = class {
|
|
|
3828
3829
|
hash: computeXorbHash(xorbChunksCleaned),
|
|
3829
3830
|
chunks: xorbChunksCleaned,
|
|
3830
3831
|
id: this.id,
|
|
3831
|
-
files: Object.entries(this.fileProcessedBytes).map(([
|
|
3832
|
-
path:
|
|
3833
|
-
progress: processedBytes / this.fileSize[
|
|
3834
|
-
lastSentProgress: ((this.fileUploadedBytes[
|
|
3832
|
+
files: Object.entries(this.fileProcessedBytes).map(([path9, processedBytes]) => ({
|
|
3833
|
+
path: path9,
|
|
3834
|
+
progress: processedBytes / this.fileSize[path9],
|
|
3835
|
+
lastSentProgress: ((this.fileUploadedBytes[path9] ?? 0) + (processedBytes - (this.fileUploadedBytes[path9] ?? 0)) * PROCESSING_PROGRESS_RATIO) / this.fileSize[path9]
|
|
3835
3836
|
}))
|
|
3836
3837
|
};
|
|
3837
3838
|
}
|
|
@@ -4704,7 +4705,7 @@ var BucketClient = class {
|
|
|
4704
4705
|
if (paths.length === 0) {
|
|
4705
4706
|
return;
|
|
4706
4707
|
}
|
|
4707
|
-
await this.batch(paths.map((
|
|
4708
|
+
await this.batch(paths.map((path9) => ({ type: "deleteFile", path: path9 })));
|
|
4708
4709
|
}
|
|
4709
4710
|
async batch(operations) {
|
|
4710
4711
|
const body = `${operations.map((op) => JSON.stringify(op)).join("\n")}
|
|
@@ -4720,8 +4721,8 @@ var BucketClient = class {
|
|
|
4720
4721
|
* any other failure (including bucket/auth errors), so a missing object is
|
|
4721
4722
|
* never conflated with an unreachable bucket.
|
|
4722
4723
|
*/
|
|
4723
|
-
async downloadFile(
|
|
4724
|
-
const url = `${this.hubUrl}/buckets/${this.bucket}/resolve/${encodeURIComponent(
|
|
4724
|
+
async downloadFile(path9) {
|
|
4725
|
+
const url = `${this.hubUrl}/buckets/${this.bucket}/resolve/${encodeURIComponent(path9)}`;
|
|
4725
4726
|
const response = await this.fetchWithRetry(url);
|
|
4726
4727
|
if (response.status === 404) {
|
|
4727
4728
|
await this.assertBucketAccessible();
|
|
@@ -4813,10 +4814,82 @@ function createHfBucketHub(params) {
|
|
|
4813
4814
|
}
|
|
4814
4815
|
};
|
|
4815
4816
|
}
|
|
4817
|
+
function createMountedBucketHub(params) {
|
|
4818
|
+
const root = path.resolve(params.mountDir);
|
|
4819
|
+
const assertMountRoot = async () => {
|
|
4820
|
+
let stat;
|
|
4821
|
+
try {
|
|
4822
|
+
stat = await fs.stat(root);
|
|
4823
|
+
} catch (err) {
|
|
4824
|
+
if (isNotFound(err)) {
|
|
4825
|
+
throw new Error(`mounted bucket root is missing: ${root}`);
|
|
4826
|
+
}
|
|
4827
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4828
|
+
throw new Error(`mounted bucket root is not accessible: ${root}: ${message}`);
|
|
4829
|
+
}
|
|
4830
|
+
if (!stat.isDirectory()) {
|
|
4831
|
+
throw new Error(`mounted bucket root is not a directory: ${root}`);
|
|
4832
|
+
}
|
|
4833
|
+
};
|
|
4834
|
+
const localPathFor = (remotePath2) => {
|
|
4835
|
+
const normalized = remotePath2.replace(/^\/+/, "");
|
|
4836
|
+
const resolved = path.resolve(root, normalized);
|
|
4837
|
+
if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) {
|
|
4838
|
+
throw new Error(`invalid bucket path outside mount: ${remotePath2}`);
|
|
4839
|
+
}
|
|
4840
|
+
return resolved;
|
|
4841
|
+
};
|
|
4842
|
+
return {
|
|
4843
|
+
async download(remotePath2, localPath) {
|
|
4844
|
+
await assertMountRoot();
|
|
4845
|
+
const source = localPathFor(remotePath2);
|
|
4846
|
+
try {
|
|
4847
|
+
await fs.copyFile(source, localPath);
|
|
4848
|
+
return "downloaded";
|
|
4849
|
+
} catch (err) {
|
|
4850
|
+
if (isNotFound(err)) {
|
|
4851
|
+
return "not-found";
|
|
4852
|
+
}
|
|
4853
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4854
|
+
throw new Error(`mounted bucket download failed for ${remotePath2}: ${message}`);
|
|
4855
|
+
}
|
|
4856
|
+
},
|
|
4857
|
+
async upload(localPath, remotePath2) {
|
|
4858
|
+
await assertMountRoot();
|
|
4859
|
+
const target = localPathFor(remotePath2);
|
|
4860
|
+
const dir = path.dirname(target);
|
|
4861
|
+
const tmp = path.join(dir, `.tmp-${path.basename(target)}-${process.pid}-${Date.now()}`);
|
|
4862
|
+
try {
|
|
4863
|
+
await fs.mkdir(dir, { recursive: true });
|
|
4864
|
+
await fs.copyFile(localPath, tmp);
|
|
4865
|
+
await fs.chmod(tmp, 420);
|
|
4866
|
+
await fs.rename(tmp, target);
|
|
4867
|
+
} catch (err) {
|
|
4868
|
+
await fs.rm(tmp, { force: true }).catch(() => void 0);
|
|
4869
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4870
|
+
throw new Error(`mounted bucket upload failed for ${remotePath2}: ${message}`);
|
|
4871
|
+
}
|
|
4872
|
+
},
|
|
4873
|
+
async delete(remotePaths) {
|
|
4874
|
+
await assertMountRoot();
|
|
4875
|
+
for (const remotePath2 of remotePaths) {
|
|
4876
|
+
try {
|
|
4877
|
+
await fs.rm(localPathFor(remotePath2), { force: true });
|
|
4878
|
+
} catch (err) {
|
|
4879
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4880
|
+
console.error(`[hf-state-sync] prune failed for ${remotePath2}: ${message}`);
|
|
4881
|
+
}
|
|
4882
|
+
}
|
|
4883
|
+
}
|
|
4884
|
+
};
|
|
4885
|
+
}
|
|
4886
|
+
function isNotFound(err) {
|
|
4887
|
+
return err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
4888
|
+
}
|
|
4816
4889
|
|
|
4817
4890
|
// src/hf-state-sync/paths.ts
|
|
4818
4891
|
import { randomUUID } from "node:crypto";
|
|
4819
|
-
var DEFAULT_LIVE_DIR = "/
|
|
4892
|
+
var DEFAULT_LIVE_DIR = "/home/node/.local/share/mlclaw/live";
|
|
4820
4893
|
var DEFAULT_BUCKET_PREFIX = "openclaw-state";
|
|
4821
4894
|
var DEFAULT_INTERVAL_SECONDS = 60;
|
|
4822
4895
|
var DEFAULT_HANDOFF_POLL_SECONDS = 5;
|
|
@@ -4827,9 +4900,12 @@ function positiveIntFromEnv(value, fallback) {
|
|
|
4827
4900
|
}
|
|
4828
4901
|
function resolveSyncConfig(env = process.env) {
|
|
4829
4902
|
const runId = env.MLCLAW_RUN_ID?.trim() || randomUUID();
|
|
4903
|
+
const snapshotUid = nonNegativeIntFromEnv(env.MLCLAW_OPENCLAW_UID);
|
|
4904
|
+
const snapshotGid = nonNegativeIntFromEnv(env.MLCLAW_OPENCLAW_GID);
|
|
4830
4905
|
return {
|
|
4831
4906
|
liveDir: env.OPENCLAW_LIVE_DIR?.trim() || DEFAULT_LIVE_DIR,
|
|
4832
4907
|
bucket: env.OPENCLAW_HF_STATE_BUCKET?.trim() || null,
|
|
4908
|
+
stateMountDir: env.MLCLAW_STATE_MOUNT_DIR?.trim() || null,
|
|
4833
4909
|
bucketPrefix: normalizeBucketPrefix(env.OPENCLAW_HF_STATE_PREFIX),
|
|
4834
4910
|
intervalSeconds: positiveIntFromEnv(env.HF_STATE_SYNC_INTERVAL_SECONDS, DEFAULT_INTERVAL_SECONDS),
|
|
4835
4911
|
handoffPollSeconds: positiveIntFromEnv(env.HF_STATE_SYNC_HANDOFF_POLL_SECONDS, DEFAULT_HANDOFF_POLL_SECONDS),
|
|
@@ -4838,9 +4914,16 @@ function resolveSyncConfig(env = process.env) {
|
|
|
4838
4914
|
runtimeId: env.MLCLAW_RUNTIME_ID?.trim() || runId,
|
|
4839
4915
|
agentName: env.OPENCLAW_AGENT_NAME?.trim() || "openclaw",
|
|
4840
4916
|
gatewayLocation: env.MLCLAW_GATEWAY_LOCATION === "local" || env.MLCLAW_GATEWAY_LOCATION === "space" ? env.MLCLAW_GATEWAY_LOCATION : "unknown",
|
|
4841
|
-
runtimeImage: env.MLCLAW_RUNTIME_IMAGE?.trim() || "unknown"
|
|
4917
|
+
runtimeImage: env.MLCLAW_RUNTIME_IMAGE?.trim() || "unknown",
|
|
4918
|
+
...snapshotUid !== void 0 ? { snapshotUid } : {},
|
|
4919
|
+
...snapshotGid !== void 0 ? { snapshotGid } : {},
|
|
4920
|
+
...env.MLCLAW_PROTECTED_STATE_DIR?.trim() ? { protectedStateDir: env.MLCLAW_PROTECTED_STATE_DIR.trim() } : {}
|
|
4842
4921
|
};
|
|
4843
4922
|
}
|
|
4923
|
+
function nonNegativeIntFromEnv(value) {
|
|
4924
|
+
const parsed = Number(value);
|
|
4925
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : void 0;
|
|
4926
|
+
}
|
|
4844
4927
|
function remotePath(config, name) {
|
|
4845
4928
|
return `${normalizeBucketPrefix(config.bucketPrefix)}/${name.replace(/^\/+/, "")}`;
|
|
4846
4929
|
}
|
|
@@ -4858,20 +4941,20 @@ function logError(message) {
|
|
|
4858
4941
|
// src/hf-state-sync/restore.ts
|
|
4859
4942
|
import fs4 from "node:fs/promises";
|
|
4860
4943
|
import os from "node:os";
|
|
4861
|
-
import
|
|
4944
|
+
import path4 from "node:path";
|
|
4862
4945
|
|
|
4863
4946
|
// src/hf-state-sync/archive.ts
|
|
4864
4947
|
import { execFile } from "node:child_process";
|
|
4865
4948
|
import { createHash } from "node:crypto";
|
|
4866
4949
|
import { createReadStream } from "node:fs";
|
|
4867
4950
|
import fs3 from "node:fs/promises";
|
|
4868
|
-
import
|
|
4951
|
+
import path3 from "node:path";
|
|
4869
4952
|
import { pipeline } from "node:stream/promises";
|
|
4870
4953
|
import { promisify } from "node:util";
|
|
4871
4954
|
|
|
4872
4955
|
// src/hf-state-sync/sqlite.ts
|
|
4873
4956
|
import fs2 from "node:fs/promises";
|
|
4874
|
-
import
|
|
4957
|
+
import path2 from "node:path";
|
|
4875
4958
|
import { DatabaseSync } from "node:sqlite";
|
|
4876
4959
|
async function findSqliteFiles(root) {
|
|
4877
4960
|
const found = [];
|
|
@@ -4883,7 +4966,7 @@ async function findSqliteFiles(root) {
|
|
|
4883
4966
|
}
|
|
4884
4967
|
for (const entry of entries) {
|
|
4885
4968
|
if (entry.isFile() && entry.name.endsWith(".sqlite")) {
|
|
4886
|
-
found.push(
|
|
4969
|
+
found.push(path2.join(entry.parentPath, entry.name));
|
|
4887
4970
|
}
|
|
4888
4971
|
}
|
|
4889
4972
|
return found.sort();
|
|
@@ -4919,6 +5002,7 @@ var STATE_EXCLUDED_NAMES = /* @__PURE__ */ new Set([".env", "credentials", "tmp"
|
|
|
4919
5002
|
var STATE_EXCLUDED_SUFFIXES = [".log"];
|
|
4920
5003
|
var SIDECAR_SUFFIXES = [".sqlite-wal", ".sqlite-shm"];
|
|
4921
5004
|
var STATE_DIR_NAME = ".openclaw";
|
|
5005
|
+
var PROTECTED_STATE_DIR_NAME = ".mlclaw-protected";
|
|
4922
5006
|
function isExcluded(name, inStateDir) {
|
|
4923
5007
|
if (SIDECAR_SUFFIXES.some((suffix) => name.endsWith(suffix))) {
|
|
4924
5008
|
return true;
|
|
@@ -4929,15 +5013,18 @@ function isExcluded(name, inStateDir) {
|
|
|
4929
5013
|
return STATE_EXCLUDED_NAMES.has(name) || STATE_EXCLUDED_SUFFIXES.some((suffix) => name.endsWith(suffix));
|
|
4930
5014
|
}
|
|
4931
5015
|
async function copyTreeFiltered(params) {
|
|
4932
|
-
const { sourceDir, destDir, databases, rootDir, inStateDir, depth } = params;
|
|
5016
|
+
const { sourceDir, destDir, databases, rootDir, inStateDir, depth, excludeProtectedState } = params;
|
|
4933
5017
|
await fs3.mkdir(destDir, { recursive: true });
|
|
4934
5018
|
const entries = await fs3.readdir(sourceDir, { withFileTypes: true });
|
|
4935
5019
|
for (const entry of entries) {
|
|
5020
|
+
if (excludeProtectedState && depth === 0 && entry.name === PROTECTED_STATE_DIR_NAME) {
|
|
5021
|
+
continue;
|
|
5022
|
+
}
|
|
4936
5023
|
if (isExcluded(entry.name, inStateDir)) {
|
|
4937
5024
|
continue;
|
|
4938
5025
|
}
|
|
4939
|
-
const source =
|
|
4940
|
-
const dest =
|
|
5026
|
+
const source = path3.join(sourceDir, entry.name);
|
|
5027
|
+
const dest = path3.join(destDir, entry.name);
|
|
4941
5028
|
if (entry.isDirectory()) {
|
|
4942
5029
|
await copyTreeFiltered({
|
|
4943
5030
|
sourceDir: source,
|
|
@@ -4947,11 +5034,12 @@ async function copyTreeFiltered(params) {
|
|
|
4947
5034
|
// Only the top-level .openclaw dir is OpenClaw state; a workspace
|
|
4948
5035
|
// project may legitimately contain its own .openclaw directory.
|
|
4949
5036
|
inStateDir: inStateDir || depth === 0 && entry.name === STATE_DIR_NAME,
|
|
4950
|
-
depth: depth + 1
|
|
5037
|
+
depth: depth + 1,
|
|
5038
|
+
excludeProtectedState
|
|
4951
5039
|
});
|
|
4952
5040
|
} else if (entry.isFile()) {
|
|
4953
5041
|
if (entry.name.endsWith(".sqlite")) {
|
|
4954
|
-
databases.push(
|
|
5042
|
+
databases.push(path3.relative(rootDir, source));
|
|
4955
5043
|
} else {
|
|
4956
5044
|
await fs3.copyFile(source, dest);
|
|
4957
5045
|
}
|
|
@@ -4960,7 +5048,7 @@ async function copyTreeFiltered(params) {
|
|
|
4960
5048
|
}
|
|
4961
5049
|
}
|
|
4962
5050
|
}
|
|
4963
|
-
async function stageLiveDir(liveDir, stagingDir) {
|
|
5051
|
+
async function stageLiveDir(liveDir, stagingDir, options = {}) {
|
|
4964
5052
|
const databases = [];
|
|
4965
5053
|
await copyTreeFiltered({
|
|
4966
5054
|
sourceDir: liveDir,
|
|
@@ -4968,12 +5056,13 @@ async function stageLiveDir(liveDir, stagingDir) {
|
|
|
4968
5056
|
databases,
|
|
4969
5057
|
rootDir: liveDir,
|
|
4970
5058
|
inStateDir: false,
|
|
4971
|
-
depth: 0
|
|
5059
|
+
depth: 0,
|
|
5060
|
+
excludeProtectedState: options.excludeProtectedState ?? false
|
|
4972
5061
|
});
|
|
4973
5062
|
for (const relative of databases) {
|
|
4974
|
-
const staged =
|
|
4975
|
-
await fs3.mkdir(
|
|
4976
|
-
vacuumInto(
|
|
5063
|
+
const staged = path3.join(stagingDir, relative);
|
|
5064
|
+
await fs3.mkdir(path3.dirname(staged), { recursive: true });
|
|
5065
|
+
vacuumInto(path3.join(liveDir, relative), staged);
|
|
4977
5066
|
const integrity = checkIntegrity(staged);
|
|
4978
5067
|
if (integrity.kind === "corrupt") {
|
|
4979
5068
|
return { kind: "corrupt-database", database: relative, detail: integrity.detail };
|
|
@@ -5480,8 +5569,8 @@ function getErrorMap() {
|
|
|
5480
5569
|
|
|
5481
5570
|
// node_modules/zod/v3/helpers/parseUtil.js
|
|
5482
5571
|
var makeIssue = (params) => {
|
|
5483
|
-
const { data, path:
|
|
5484
|
-
const fullPath = [...
|
|
5572
|
+
const { data, path: path9, errorMaps, issueData } = params;
|
|
5573
|
+
const fullPath = [...path9, ...issueData.path || []];
|
|
5485
5574
|
const fullIssue = {
|
|
5486
5575
|
...issueData,
|
|
5487
5576
|
path: fullPath
|
|
@@ -5597,11 +5686,11 @@ var errorUtil;
|
|
|
5597
5686
|
|
|
5598
5687
|
// node_modules/zod/v3/types.js
|
|
5599
5688
|
var ParseInputLazyPath = class {
|
|
5600
|
-
constructor(parent, value,
|
|
5689
|
+
constructor(parent, value, path9, key) {
|
|
5601
5690
|
this._cachedPath = [];
|
|
5602
5691
|
this.parent = parent;
|
|
5603
5692
|
this.data = value;
|
|
5604
|
-
this._path =
|
|
5693
|
+
this._path = path9;
|
|
5605
5694
|
this._key = key;
|
|
5606
5695
|
}
|
|
5607
5696
|
get path() {
|
|
@@ -9086,7 +9175,7 @@ function promoteSnapshot(params) {
|
|
|
9086
9175
|
// src/hf-state-sync/restore.ts
|
|
9087
9176
|
async function tryRestoreEntry(params) {
|
|
9088
9177
|
const { hub, entry, workDir, liveDir } = params;
|
|
9089
|
-
const archivePath =
|
|
9178
|
+
const archivePath = path4.join(workDir, `candidate-${entry.id}.tar.zst`);
|
|
9090
9179
|
const downloaded = await hub.download(entry.path, archivePath);
|
|
9091
9180
|
if (downloaded === "not-found") {
|
|
9092
9181
|
logError(`snapshot ${entry.id} missing from bucket`);
|
|
@@ -9110,11 +9199,11 @@ async function tryRestoreEntry(params) {
|
|
|
9110
9199
|
for (const database of await findSqliteFiles(extractDir)) {
|
|
9111
9200
|
const integrity = checkIntegrity(database);
|
|
9112
9201
|
if (integrity.kind === "corrupt") {
|
|
9113
|
-
logError(`snapshot ${entry.id} db ${
|
|
9202
|
+
logError(`snapshot ${entry.id} db ${path4.basename(database)} corrupt: ${integrity.detail}`);
|
|
9114
9203
|
return "failed";
|
|
9115
9204
|
}
|
|
9116
9205
|
}
|
|
9117
|
-
await fs4.mkdir(
|
|
9206
|
+
await fs4.mkdir(path4.dirname(liveDir), { recursive: true });
|
|
9118
9207
|
await fs4.rename(extractDir, liveDir);
|
|
9119
9208
|
return "restored";
|
|
9120
9209
|
} finally {
|
|
@@ -9131,9 +9220,9 @@ async function runRestore(params) {
|
|
|
9131
9220
|
return { kind: "fresh-start", reason: "live-dir-exists" };
|
|
9132
9221
|
} catch {
|
|
9133
9222
|
}
|
|
9134
|
-
const workDir = await fs4.mkdtemp(
|
|
9223
|
+
const workDir = await fs4.mkdtemp(path4.join(os.tmpdir(), "hf-state-restore-"));
|
|
9135
9224
|
try {
|
|
9136
|
-
const manifestPath =
|
|
9225
|
+
const manifestPath = path4.join(workDir, "manifest.json");
|
|
9137
9226
|
const downloaded = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), manifestPath);
|
|
9138
9227
|
if (downloaded === "not-found") {
|
|
9139
9228
|
return { kind: "fresh-start", reason: "no-manifest" };
|
|
@@ -9157,10 +9246,84 @@ async function runRestore(params) {
|
|
|
9157
9246
|
}
|
|
9158
9247
|
}
|
|
9159
9248
|
|
|
9160
|
-
// src/hf-state-sync/
|
|
9249
|
+
// src/hf-state-sync/prepare.ts
|
|
9161
9250
|
import fs5 from "node:fs/promises";
|
|
9251
|
+
import path5 from "node:path";
|
|
9252
|
+
async function prepareRestore(config) {
|
|
9253
|
+
if (config.snapshotUid === void 0 || config.snapshotGid === void 0) {
|
|
9254
|
+
throw new Error("restore preparation requires MLCLAW_OPENCLAW_UID and MLCLAW_OPENCLAW_GID");
|
|
9255
|
+
}
|
|
9256
|
+
const liveParent = path5.dirname(config.liveDir);
|
|
9257
|
+
await fs5.mkdir(liveParent, { recursive: true });
|
|
9258
|
+
await fs5.chown(liveParent, config.snapshotUid, config.snapshotGid);
|
|
9259
|
+
if (!config.stateMountDir) {
|
|
9260
|
+
return;
|
|
9261
|
+
}
|
|
9262
|
+
await makeTraversableDirectory(config.stateMountDir);
|
|
9263
|
+
const prefixRoot = confinedPath(config.stateMountDir, config.bucketPrefix);
|
|
9264
|
+
let prefixPart = config.stateMountDir;
|
|
9265
|
+
for (const part of config.bucketPrefix.split("/").filter(Boolean)) {
|
|
9266
|
+
prefixPart = path5.join(prefixPart, part);
|
|
9267
|
+
await makeTraversableDirectory(prefixPart);
|
|
9268
|
+
}
|
|
9269
|
+
await makeReadableIfFile(path5.join(prefixRoot, "manifest.json"));
|
|
9270
|
+
const snapshotsDir = path5.join(prefixRoot, "snapshots");
|
|
9271
|
+
await makeTraversableDirectory(snapshotsDir);
|
|
9272
|
+
let entries;
|
|
9273
|
+
try {
|
|
9274
|
+
entries = await fs5.readdir(snapshotsDir, { withFileTypes: true });
|
|
9275
|
+
} catch (err) {
|
|
9276
|
+
if (isNotFound2(err)) {
|
|
9277
|
+
return;
|
|
9278
|
+
}
|
|
9279
|
+
throw err;
|
|
9280
|
+
}
|
|
9281
|
+
for (const entry of entries) {
|
|
9282
|
+
if (entry.isFile()) {
|
|
9283
|
+
await fs5.chmod(path5.join(snapshotsDir, entry.name), 420);
|
|
9284
|
+
}
|
|
9285
|
+
}
|
|
9286
|
+
}
|
|
9287
|
+
async function makeTraversableDirectory(directory) {
|
|
9288
|
+
try {
|
|
9289
|
+
const stat = await fs5.lstat(directory);
|
|
9290
|
+
if (stat.isDirectory()) {
|
|
9291
|
+
await fs5.chmod(directory, 457);
|
|
9292
|
+
}
|
|
9293
|
+
} catch (err) {
|
|
9294
|
+
if (!isNotFound2(err)) {
|
|
9295
|
+
throw err;
|
|
9296
|
+
}
|
|
9297
|
+
}
|
|
9298
|
+
}
|
|
9299
|
+
function confinedPath(root, relative) {
|
|
9300
|
+
const resolvedRoot = path5.resolve(root);
|
|
9301
|
+
const resolved = path5.resolve(resolvedRoot, relative);
|
|
9302
|
+
if (resolved !== resolvedRoot && !resolved.startsWith(`${resolvedRoot}${path5.sep}`)) {
|
|
9303
|
+
throw new Error(`invalid state prefix outside mounted bucket: ${relative}`);
|
|
9304
|
+
}
|
|
9305
|
+
return resolved;
|
|
9306
|
+
}
|
|
9307
|
+
async function makeReadableIfFile(file) {
|
|
9308
|
+
try {
|
|
9309
|
+
const stat = await fs5.lstat(file);
|
|
9310
|
+
if (stat.isFile()) {
|
|
9311
|
+
await fs5.chmod(file, 420);
|
|
9312
|
+
}
|
|
9313
|
+
} catch (err) {
|
|
9314
|
+
if (!isNotFound2(err)) {
|
|
9315
|
+
throw err;
|
|
9316
|
+
}
|
|
9317
|
+
}
|
|
9318
|
+
}
|
|
9319
|
+
function isNotFound2(err) {
|
|
9320
|
+
return err instanceof Error && "code" in err && err.code === "ENOENT";
|
|
9321
|
+
}
|
|
9322
|
+
|
|
9323
|
+
// src/hf-state-sync/snapshot.ts
|
|
9324
|
+
import fs6 from "node:fs/promises";
|
|
9162
9325
|
import os2 from "node:os";
|
|
9163
|
-
import
|
|
9326
|
+
import path6 from "node:path";
|
|
9164
9327
|
var snapshotCounter = 0;
|
|
9165
9328
|
function snapshotId(now, runId) {
|
|
9166
9329
|
snapshotCounter += 1;
|
|
@@ -9168,12 +9331,12 @@ function snapshotId(now, runId) {
|
|
|
9168
9331
|
return `${stamp}-${runId.slice(0, 8)}-${snapshotCounter}`;
|
|
9169
9332
|
}
|
|
9170
9333
|
async function fetchManifest(config, hub, workDir) {
|
|
9171
|
-
const localPath =
|
|
9334
|
+
const localPath = path6.join(workDir, "manifest.remote.json");
|
|
9172
9335
|
const result = await hub.download(remotePath(config, MANIFEST_REMOTE_NAME), localPath);
|
|
9173
9336
|
if (result === "not-found") {
|
|
9174
9337
|
return { kind: "none" };
|
|
9175
9338
|
}
|
|
9176
|
-
const parsed = parseManifest(await
|
|
9339
|
+
const parsed = parseManifest(await fs6.readFile(localPath, "utf8"));
|
|
9177
9340
|
return parsed.kind === "ok" ? { kind: "ok", manifest: parsed.manifest } : { kind: "invalid", reason: parsed.reason };
|
|
9178
9341
|
}
|
|
9179
9342
|
async function runSnapshot(params) {
|
|
@@ -9182,31 +9345,29 @@ async function runSnapshot(params) {
|
|
|
9182
9345
|
return { kind: "skipped", reason: "no-bucket" };
|
|
9183
9346
|
}
|
|
9184
9347
|
try {
|
|
9185
|
-
await
|
|
9348
|
+
await fs6.access(config.liveDir);
|
|
9186
9349
|
} catch {
|
|
9187
9350
|
return { kind: "skipped", reason: "empty-state" };
|
|
9188
9351
|
}
|
|
9189
|
-
const workDir = await
|
|
9352
|
+
const workDir = await fs6.mkdtemp(path6.join(os2.tmpdir(), "hf-state-snapshot-"));
|
|
9190
9353
|
try {
|
|
9191
|
-
const
|
|
9192
|
-
const
|
|
9354
|
+
const now = (params.now ?? (() => /* @__PURE__ */ new Date()))();
|
|
9355
|
+
const id = snapshotId(now, config.runId);
|
|
9356
|
+
const archiveName = `state-${id}.tar.zst`;
|
|
9357
|
+
const archivePath = path6.join(workDir, archiveName);
|
|
9358
|
+
const staged = params.stageArchive ? await params.stageArchive({ liveDir: config.liveDir, archivePath }) : await stageArchiveInProcess(config.liveDir, path6.join(workDir, "stage"), archivePath);
|
|
9193
9359
|
if (staged.kind === "corrupt-database") {
|
|
9194
9360
|
return {
|
|
9195
9361
|
kind: "failed",
|
|
9196
9362
|
detail: `live database ${staged.database} failed integrity check: ${staged.detail}`
|
|
9197
9363
|
};
|
|
9198
9364
|
}
|
|
9199
|
-
const now = (params.now ?? (() => /* @__PURE__ */ new Date()))();
|
|
9200
|
-
const id = snapshotId(now, config.runId);
|
|
9201
|
-
const archiveName = `state-${id}.tar.zst`;
|
|
9202
|
-
const archivePath = path4.join(workDir, archiveName);
|
|
9203
|
-
await createTarZst(stagingDir, archivePath);
|
|
9204
9365
|
const entry = {
|
|
9205
9366
|
id,
|
|
9206
9367
|
path: remotePath(config, `snapshots/${archiveName}`),
|
|
9207
9368
|
createdAt: now.toISOString(),
|
|
9208
9369
|
sha256: await sha256File(archivePath),
|
|
9209
|
-
sizeBytes: (await
|
|
9370
|
+
sizeBytes: (await fs6.stat(archivePath)).size,
|
|
9210
9371
|
runId: config.runId,
|
|
9211
9372
|
bootTime: params.bootTime
|
|
9212
9373
|
};
|
|
@@ -9223,27 +9384,182 @@ async function runSnapshot(params) {
|
|
|
9223
9384
|
entry,
|
|
9224
9385
|
keep: config.keepSnapshots
|
|
9225
9386
|
});
|
|
9226
|
-
const manifestPath =
|
|
9227
|
-
await
|
|
9387
|
+
const manifestPath = path6.join(workDir, "manifest.json");
|
|
9388
|
+
await fs6.writeFile(manifestPath, serializeManifest(manifest));
|
|
9228
9389
|
await hub.upload(manifestPath, remotePath(config, MANIFEST_REMOTE_NAME));
|
|
9229
9390
|
if (expired.length > 0) {
|
|
9230
9391
|
await hub.delete(expired.map((e) => e.path));
|
|
9231
9392
|
}
|
|
9232
|
-
log(`snapshot ${entry.id} uploaded (${entry.sizeBytes} bytes, ${staged.
|
|
9393
|
+
log(`snapshot ${entry.id} uploaded (${entry.sizeBytes} bytes, ${staged.databaseCount} dbs)`);
|
|
9233
9394
|
return { kind: "uploaded", entry };
|
|
9234
9395
|
} catch (err) {
|
|
9235
9396
|
return { kind: "failed", detail: err instanceof Error ? err.message : String(err) };
|
|
9236
9397
|
} finally {
|
|
9237
|
-
await
|
|
9398
|
+
await fs6.rm(workDir, { recursive: true, force: true });
|
|
9399
|
+
}
|
|
9400
|
+
}
|
|
9401
|
+
async function stageArchiveInProcess(liveDir, stagingDir, archivePath) {
|
|
9402
|
+
const staged = await stageLiveDir(liveDir, stagingDir);
|
|
9403
|
+
if (staged.kind === "corrupt-database") {
|
|
9404
|
+
return staged;
|
|
9238
9405
|
}
|
|
9406
|
+
await createTarZst(stagingDir, archivePath);
|
|
9407
|
+
return { kind: "staged", databaseCount: staged.databases.length };
|
|
9239
9408
|
}
|
|
9240
9409
|
|
|
9241
9410
|
// src/hf-state-sync/supervise.ts
|
|
9411
|
+
import { spawn as spawn2 } from "node:child_process";
|
|
9412
|
+
import fs8 from "node:fs/promises";
|
|
9413
|
+
import os4 from "node:os";
|
|
9414
|
+
import path8 from "node:path";
|
|
9415
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
9416
|
+
|
|
9417
|
+
// src/hf-state-sync/stage-worker.ts
|
|
9242
9418
|
import { spawn } from "node:child_process";
|
|
9243
|
-
import
|
|
9419
|
+
import { createReadStream as createReadStream2, createWriteStream, writeFileSync } from "node:fs";
|
|
9420
|
+
import fs7 from "node:fs/promises";
|
|
9244
9421
|
import os3 from "node:os";
|
|
9245
|
-
import
|
|
9246
|
-
import {
|
|
9422
|
+
import path7 from "node:path";
|
|
9423
|
+
import { pipeline as pipeline2 } from "node:stream/promises";
|
|
9424
|
+
function unprivilegedStageArchive(params) {
|
|
9425
|
+
return async ({ liveDir, archivePath }) => {
|
|
9426
|
+
const child = spawn(process.execPath, [params.scriptPath, "stage-worker", liveDir], {
|
|
9427
|
+
uid: params.uid,
|
|
9428
|
+
gid: params.gid,
|
|
9429
|
+
env: snapshotWorkerEnvironment(process.env),
|
|
9430
|
+
stdio: ["ignore", "pipe", "inherit", "pipe"]
|
|
9431
|
+
});
|
|
9432
|
+
if (!child.stdout || !child.stdio[3]) {
|
|
9433
|
+
child.kill("SIGKILL");
|
|
9434
|
+
throw new Error("snapshot staging worker pipes are unavailable");
|
|
9435
|
+
}
|
|
9436
|
+
const archiveOutput = createWriteStream(archivePath, { flags: "wx", mode: 384 });
|
|
9437
|
+
const metadata = collect(child.stdio[3]);
|
|
9438
|
+
const archive = pipeline2(child.stdout, archiveOutput);
|
|
9439
|
+
const exitCode = await new Promise((resolve, reject) => {
|
|
9440
|
+
child.once("error", reject);
|
|
9441
|
+
child.once("exit", (code, signal) => resolve(code ?? (signal ? 128 : 1)));
|
|
9442
|
+
});
|
|
9443
|
+
await archive;
|
|
9444
|
+
const message = parseWorkerMessage(await metadata);
|
|
9445
|
+
if (exitCode !== 0 || message.kind === "failed") {
|
|
9446
|
+
throw new Error(
|
|
9447
|
+
message.kind === "failed" ? message.detail : `snapshot staging worker exited with code ${exitCode}`
|
|
9448
|
+
);
|
|
9449
|
+
}
|
|
9450
|
+
return message;
|
|
9451
|
+
};
|
|
9452
|
+
}
|
|
9453
|
+
function protectedStageArchive(params) {
|
|
9454
|
+
return async (request) => {
|
|
9455
|
+
const outcome = await params.base(request);
|
|
9456
|
+
if (outcome.kind !== "staged") {
|
|
9457
|
+
return outcome;
|
|
9458
|
+
}
|
|
9459
|
+
const workDir = await fs7.mkdtemp(path7.join(os3.tmpdir(), "hf-state-protected-stage-"));
|
|
9460
|
+
try {
|
|
9461
|
+
const stagingDir = path7.join(workDir, "stage");
|
|
9462
|
+
await extractTarZst(request.archivePath, stagingDir);
|
|
9463
|
+
const destination = path7.join(stagingDir, params.archiveName);
|
|
9464
|
+
await fs7.cp(params.sourceDir, destination, {
|
|
9465
|
+
recursive: true,
|
|
9466
|
+
force: false,
|
|
9467
|
+
preserveTimestamps: true,
|
|
9468
|
+
filter: (source) => includeProtectedSnapshotPath(params.sourceDir, source)
|
|
9469
|
+
});
|
|
9470
|
+
await fs7.chmod(destination, 448);
|
|
9471
|
+
await fs7.rm(request.archivePath, { force: true });
|
|
9472
|
+
await createTarZst(stagingDir, request.archivePath);
|
|
9473
|
+
await fs7.chmod(request.archivePath, 384);
|
|
9474
|
+
return outcome;
|
|
9475
|
+
} finally {
|
|
9476
|
+
await fs7.rm(workDir, { recursive: true, force: true });
|
|
9477
|
+
}
|
|
9478
|
+
};
|
|
9479
|
+
}
|
|
9480
|
+
function includeProtectedSnapshotPath(sourceDir, source) {
|
|
9481
|
+
const relative = path7.relative(sourceDir, source);
|
|
9482
|
+
return relative !== "hf-broker/mirrors" && !relative.startsWith(`hf-broker/mirrors${path7.sep}`);
|
|
9483
|
+
}
|
|
9484
|
+
function trustedStageArchive(config, scriptPath) {
|
|
9485
|
+
const canStageAsOpenClaw = process.getuid?.() === 0 && Boolean(scriptPath) && config.snapshotUid !== void 0 && config.snapshotGid !== void 0;
|
|
9486
|
+
if (!canStageAsOpenClaw) {
|
|
9487
|
+
if (config.protectedStateDir) {
|
|
9488
|
+
throw new Error("protected runtime state requires root snapshot staging with an OpenClaw UID and GID");
|
|
9489
|
+
}
|
|
9490
|
+
return void 0;
|
|
9491
|
+
}
|
|
9492
|
+
let stageArchive = unprivilegedStageArchive({
|
|
9493
|
+
uid: config.snapshotUid,
|
|
9494
|
+
gid: config.snapshotGid,
|
|
9495
|
+
scriptPath
|
|
9496
|
+
});
|
|
9497
|
+
if (config.protectedStateDir) {
|
|
9498
|
+
stageArchive = protectedStageArchive({
|
|
9499
|
+
base: stageArchive,
|
|
9500
|
+
sourceDir: config.protectedStateDir,
|
|
9501
|
+
archiveName: PROTECTED_STATE_DIR_NAME
|
|
9502
|
+
});
|
|
9503
|
+
}
|
|
9504
|
+
return stageArchive;
|
|
9505
|
+
}
|
|
9506
|
+
async function runStageWorker(liveDir) {
|
|
9507
|
+
const workDir = await fs7.mkdtemp(path7.join(os3.tmpdir(), "hf-state-stage-worker-"));
|
|
9508
|
+
try {
|
|
9509
|
+
const stagingDir = path7.join(workDir, "stage");
|
|
9510
|
+
const archivePath = path7.join(workDir, "snapshot.tar.zst");
|
|
9511
|
+
const staged = await stageLiveDir(liveDir, stagingDir, { excludeProtectedState: true });
|
|
9512
|
+
if (staged.kind === "corrupt-database") {
|
|
9513
|
+
writeWorkerMessage(staged);
|
|
9514
|
+
return 0;
|
|
9515
|
+
}
|
|
9516
|
+
await createTarZst(stagingDir, archivePath);
|
|
9517
|
+
writeWorkerMessage({ kind: "staged", databaseCount: staged.databases.length });
|
|
9518
|
+
await pipeline2(createReadStream2(archivePath), process.stdout);
|
|
9519
|
+
return 0;
|
|
9520
|
+
} catch (err) {
|
|
9521
|
+
writeWorkerMessage({ kind: "failed", detail: err instanceof Error ? err.message : String(err) });
|
|
9522
|
+
return 1;
|
|
9523
|
+
} finally {
|
|
9524
|
+
await fs7.rm(workDir, { recursive: true, force: true });
|
|
9525
|
+
}
|
|
9526
|
+
}
|
|
9527
|
+
function snapshotWorkerEnvironment(env) {
|
|
9528
|
+
return {
|
|
9529
|
+
HOME: "/home/node",
|
|
9530
|
+
PATH: env.PATH,
|
|
9531
|
+
TMPDIR: env.TMPDIR
|
|
9532
|
+
};
|
|
9533
|
+
}
|
|
9534
|
+
function writeWorkerMessage(message) {
|
|
9535
|
+
writeFileSync(3, `${JSON.stringify(message)}
|
|
9536
|
+
`);
|
|
9537
|
+
}
|
|
9538
|
+
async function collect(stream) {
|
|
9539
|
+
if (!stream) {
|
|
9540
|
+
throw new Error("snapshot staging worker metadata pipe is unavailable");
|
|
9541
|
+
}
|
|
9542
|
+
const chunks = [];
|
|
9543
|
+
for await (const chunk of stream) {
|
|
9544
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
9545
|
+
}
|
|
9546
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
9547
|
+
}
|
|
9548
|
+
function parseWorkerMessage(raw) {
|
|
9549
|
+
const parsed = JSON.parse(raw);
|
|
9550
|
+
if (parsed.kind === "staged" && Number.isInteger(parsed.databaseCount) && parsed.databaseCount >= 0) {
|
|
9551
|
+
return parsed;
|
|
9552
|
+
}
|
|
9553
|
+
if (parsed.kind === "corrupt-database" && parsed.database && parsed.detail) {
|
|
9554
|
+
return parsed;
|
|
9555
|
+
}
|
|
9556
|
+
if (parsed.kind === "failed" && parsed.detail) {
|
|
9557
|
+
return parsed;
|
|
9558
|
+
}
|
|
9559
|
+
throw new Error("snapshot staging worker returned invalid metadata");
|
|
9560
|
+
}
|
|
9561
|
+
|
|
9562
|
+
// src/hf-state-sync/supervise.ts
|
|
9247
9563
|
var LEASE_HEARTBEAT_MS = 6e4;
|
|
9248
9564
|
var HANDOFF_REQUEST_TTL_MS = 10 * 60 * 1e3;
|
|
9249
9565
|
async function supervise(params) {
|
|
@@ -9253,6 +9569,8 @@ async function supervise(params) {
|
|
|
9253
9569
|
throw new Error("supervise: missing child command");
|
|
9254
9570
|
}
|
|
9255
9571
|
const bootTime = (/* @__PURE__ */ new Date()).toISOString();
|
|
9572
|
+
const scriptPath = process.argv[1];
|
|
9573
|
+
const stageArchive = trustedStageArchive(config, scriptPath);
|
|
9256
9574
|
let lastSnapshotId;
|
|
9257
9575
|
const handoffState = { request: null };
|
|
9258
9576
|
const writeLease = async () => {
|
|
@@ -9266,24 +9584,24 @@ async function supervise(params) {
|
|
|
9266
9584
|
lastHeartbeatAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9267
9585
|
...lastSnapshotId ? { lastSnapshotId } : {}
|
|
9268
9586
|
};
|
|
9269
|
-
const tmpDir = await
|
|
9587
|
+
const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-lease-"));
|
|
9270
9588
|
try {
|
|
9271
|
-
const file =
|
|
9272
|
-
await
|
|
9589
|
+
const file = path8.join(tmpDir, "status.json");
|
|
9590
|
+
await fs8.writeFile(file, JSON.stringify(status, null, 2) + "\n");
|
|
9273
9591
|
await hub.upload(file, remotePath(config, "runtime/status.json"));
|
|
9274
9592
|
} finally {
|
|
9275
|
-
await
|
|
9593
|
+
await fs8.rm(tmpDir, { recursive: true, force: true });
|
|
9276
9594
|
}
|
|
9277
9595
|
};
|
|
9278
9596
|
const readHandoffRequest = async () => {
|
|
9279
|
-
const tmpDir = await
|
|
9597
|
+
const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-handoff-"));
|
|
9280
9598
|
try {
|
|
9281
|
-
const file =
|
|
9599
|
+
const file = path8.join(tmpDir, "request.json");
|
|
9282
9600
|
const result = await hub.download(remotePath(config, "runtime/handoff-request.json"), file);
|
|
9283
9601
|
if (result === "not-found") {
|
|
9284
9602
|
return null;
|
|
9285
9603
|
}
|
|
9286
|
-
const parsed = JSON.parse(await
|
|
9604
|
+
const parsed = JSON.parse(await fs8.readFile(file, "utf8"));
|
|
9287
9605
|
if (parsed?.schemaVersion !== 1 || parsed.agent !== config.agentName || parsed.runtimeId !== config.runtimeId || typeof parsed.requestedAt !== "string" || typeof parsed.requestId !== "string" || !parsed.requestId) {
|
|
9288
9606
|
return null;
|
|
9289
9607
|
}
|
|
@@ -9297,7 +9615,7 @@ async function supervise(params) {
|
|
|
9297
9615
|
}
|
|
9298
9616
|
return parsed;
|
|
9299
9617
|
} finally {
|
|
9300
|
-
await
|
|
9618
|
+
await fs8.rm(tmpDir, { recursive: true, force: true });
|
|
9301
9619
|
}
|
|
9302
9620
|
};
|
|
9303
9621
|
const writeHandoffAck = async (request) => {
|
|
@@ -9310,17 +9628,20 @@ async function supervise(params) {
|
|
|
9310
9628
|
completedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9311
9629
|
...lastSnapshotId ? { lastSnapshotId } : {}
|
|
9312
9630
|
};
|
|
9313
|
-
const tmpDir = await
|
|
9631
|
+
const tmpDir = await fs8.mkdtemp(path8.join(os4.tmpdir(), "hf-state-handoff-ack-"));
|
|
9314
9632
|
try {
|
|
9315
|
-
const file =
|
|
9316
|
-
await
|
|
9633
|
+
const file = path8.join(tmpDir, "ack.json");
|
|
9634
|
+
await fs8.writeFile(file, JSON.stringify(ack, null, 2) + "\n");
|
|
9317
9635
|
await hub.upload(file, remotePath(config, "runtime/handoff-ack.json"));
|
|
9318
9636
|
await hub.delete([remotePath(config, "runtime/handoff-request.json")]);
|
|
9319
9637
|
} finally {
|
|
9320
|
-
await
|
|
9638
|
+
await fs8.rm(tmpDir, { recursive: true, force: true });
|
|
9321
9639
|
}
|
|
9322
9640
|
};
|
|
9323
|
-
const child =
|
|
9641
|
+
const child = spawn2(binary, args, {
|
|
9642
|
+
stdio: "inherit",
|
|
9643
|
+
env: supervisedChildEnvironment(process.env)
|
|
9644
|
+
});
|
|
9324
9645
|
const childExit = new Promise((resolve) => {
|
|
9325
9646
|
child.on("exit", (code, signal) => resolve(code ?? (signal ? 128 : 1)));
|
|
9326
9647
|
child.on("error", (err) => {
|
|
@@ -9332,7 +9653,12 @@ async function supervise(params) {
|
|
|
9332
9653
|
let inFlight = null;
|
|
9333
9654
|
const runOnce = async (label) => {
|
|
9334
9655
|
try {
|
|
9335
|
-
const outcome = await runSnapshot({
|
|
9656
|
+
const outcome = await runSnapshot({
|
|
9657
|
+
config,
|
|
9658
|
+
hub,
|
|
9659
|
+
bootTime,
|
|
9660
|
+
...stageArchive ? { stageArchive } : {}
|
|
9661
|
+
});
|
|
9336
9662
|
if (outcome.kind === "failed") {
|
|
9337
9663
|
logError(`${label}: snapshot failed: ${outcome.detail}`);
|
|
9338
9664
|
} else if (outcome.kind === "uploaded") {
|
|
@@ -9372,7 +9698,9 @@ async function supervise(params) {
|
|
|
9372
9698
|
})();
|
|
9373
9699
|
void snapshotLoop;
|
|
9374
9700
|
const heartbeatLoop = (async () => {
|
|
9375
|
-
await writeLease().catch(
|
|
9701
|
+
await writeLease().catch(
|
|
9702
|
+
(err) => logError(`initial lease failed: ${err instanceof Error ? err.message : String(err)}`)
|
|
9703
|
+
);
|
|
9376
9704
|
while (!stopping) {
|
|
9377
9705
|
await delay(LEASE_HEARTBEAT_MS);
|
|
9378
9706
|
if (stopping) {
|
|
@@ -9430,16 +9758,25 @@ async function supervise(params) {
|
|
|
9430
9758
|
if (handoffState.request) {
|
|
9431
9759
|
const request = handoffState.request;
|
|
9432
9760
|
if (finalOutcome.kind !== "uploaded") {
|
|
9433
|
-
throw new Error(
|
|
9761
|
+
throw new Error(
|
|
9762
|
+
`handoff ${request.requestId} final snapshot did not upload: ${snapshotFailureDetail(finalOutcome)}`
|
|
9763
|
+
);
|
|
9434
9764
|
}
|
|
9435
9765
|
await writeHandoffAck(request).catch((err) => {
|
|
9436
|
-
throw new Error(
|
|
9766
|
+
throw new Error(
|
|
9767
|
+
`handoff ${request.requestId} snapshot completed but ack failed: ${err instanceof Error ? err.message : String(err)}`
|
|
9768
|
+
);
|
|
9437
9769
|
});
|
|
9438
9770
|
log(`handoff ${request.requestId} acknowledged`);
|
|
9439
9771
|
return 0;
|
|
9440
9772
|
}
|
|
9441
9773
|
return exitCode;
|
|
9442
9774
|
}
|
|
9775
|
+
function supervisedChildEnvironment(source) {
|
|
9776
|
+
const env = { ...source };
|
|
9777
|
+
delete env.MLCLAW_STATE_HF_TOKEN;
|
|
9778
|
+
return env;
|
|
9779
|
+
}
|
|
9443
9780
|
function snapshotFailureDetail(outcome) {
|
|
9444
9781
|
switch (outcome.kind) {
|
|
9445
9782
|
case "uploaded":
|
|
@@ -9456,17 +9793,36 @@ var USAGE = `usage:
|
|
|
9456
9793
|
hf-state-sync restore
|
|
9457
9794
|
hf-state-sync snapshot
|
|
9458
9795
|
hf-state-sync supervise -- <command> [args...]`;
|
|
9459
|
-
function makeHub(
|
|
9460
|
-
|
|
9796
|
+
function makeHub(config) {
|
|
9797
|
+
if (!config.bucket) {
|
|
9798
|
+
return null;
|
|
9799
|
+
}
|
|
9800
|
+
if (config.stateMountDir) {
|
|
9801
|
+
log(`using mounted state bucket at ${config.stateMountDir}`);
|
|
9802
|
+
return createMountedBucketHub({ mountDir: config.stateMountDir });
|
|
9803
|
+
}
|
|
9804
|
+
const token = process.env.MLCLAW_STATE_HF_TOKEN ?? process.env.HF_TOKEN;
|
|
9805
|
+
return createHfBucketHub({ bucket: config.bucket, ...token ? { token } : {} });
|
|
9461
9806
|
}
|
|
9462
9807
|
async function main(argv) {
|
|
9808
|
+
if (argv[0] === "stage-worker") {
|
|
9809
|
+
const liveDir = argv[1];
|
|
9810
|
+
if (!liveDir) {
|
|
9811
|
+
logError("stage-worker: missing live directory");
|
|
9812
|
+
return 2;
|
|
9813
|
+
}
|
|
9814
|
+
return runStageWorker(liveDir);
|
|
9815
|
+
}
|
|
9463
9816
|
const config = resolveSyncConfig();
|
|
9464
|
-
const hub = makeHub(config
|
|
9817
|
+
const hub = makeHub(config);
|
|
9465
9818
|
if (!hub) {
|
|
9466
9819
|
logError("OPENCLAW_HF_STATE_BUCKET is not set; state will NOT survive restarts");
|
|
9467
9820
|
}
|
|
9468
9821
|
const mode = argv[0];
|
|
9469
9822
|
switch (mode) {
|
|
9823
|
+
case "prepare-restore":
|
|
9824
|
+
await prepareRestore(config);
|
|
9825
|
+
return 0;
|
|
9470
9826
|
case "restore": {
|
|
9471
9827
|
if (!hub) {
|
|
9472
9828
|
return 0;
|
|
@@ -9491,7 +9847,13 @@ async function main(argv) {
|
|
|
9491
9847
|
if (!hub) {
|
|
9492
9848
|
return 1;
|
|
9493
9849
|
}
|
|
9494
|
-
const
|
|
9850
|
+
const stageArchive = trustedStageArchive(config, process.argv[1]);
|
|
9851
|
+
const outcome = await runSnapshot({
|
|
9852
|
+
config,
|
|
9853
|
+
hub,
|
|
9854
|
+
bootTime: (/* @__PURE__ */ new Date()).toISOString(),
|
|
9855
|
+
...stageArchive ? { stageArchive } : {}
|
|
9856
|
+
});
|
|
9495
9857
|
if (outcome.kind === "failed") {
|
|
9496
9858
|
logError(outcome.detail);
|
|
9497
9859
|
return 1;
|
|
@@ -9507,8 +9869,8 @@ async function main(argv) {
|
|
|
9507
9869
|
return 2;
|
|
9508
9870
|
}
|
|
9509
9871
|
if (!hub) {
|
|
9510
|
-
const { spawn:
|
|
9511
|
-
const child =
|
|
9872
|
+
const { spawn: spawn3 } = await import("node:child_process");
|
|
9873
|
+
const child = spawn3(command[0], command.slice(1), { stdio: "inherit" });
|
|
9512
9874
|
return await new Promise((resolve) => {
|
|
9513
9875
|
process.on("SIGTERM", () => child.kill("SIGTERM"));
|
|
9514
9876
|
process.on("SIGINT", () => child.kill("SIGINT"));
|