@alfe.ai/openclaw-sync 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cli/index.cjs +17 -25
- package/dist/cli/index.js +18 -26
- package/dist/cli/index.js.map +1 -1
- package/dist/defaults/common.alfesyncignore +10 -0
- package/dist/index.cjs +3 -0
- package/dist/index.d.cts +132 -56
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +132 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/plugin.d.cts +0 -2
- package/dist/plugin.d.cts.map +1 -1
- package/dist/plugin.d.ts +0 -2
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin2.cjs +392 -150
- package/dist/plugin2.js +396 -154
- package/dist/plugin2.js.map +1 -1
- package/dist/sync-engine.cjs +262 -66
- package/dist/sync-engine.js +247 -69
- package/dist/sync-engine.js.map +1 -1
- package/package.json +4 -4
package/dist/sync-engine.cjs
CHANGED
|
@@ -29,6 +29,53 @@ let node_url = require("node:url");
|
|
|
29
29
|
let micromatch = require("micromatch");
|
|
30
30
|
micromatch = __toESM(micromatch);
|
|
31
31
|
let _auriclabs_logger = require("@auriclabs/logger");
|
|
32
|
+
//#region src/path-contract.ts
|
|
33
|
+
/**
|
|
34
|
+
* Canonical path contract for the private workspace sync data plane.
|
|
35
|
+
*
|
|
36
|
+
* Paths received from manifests, relays, CLI arguments, and watcher events
|
|
37
|
+
* are identifiers relative to the configured workspace. Keep validation in
|
|
38
|
+
* one place so a malformed remote path can never become an arbitrary local
|
|
39
|
+
* read, write, or delete.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_SYNC_PATH_LENGTH = 1024;
|
|
42
|
+
function validatePrivateRelativePath(relativePath) {
|
|
43
|
+
if (relativePath.length === 0 || relativePath.length > MAX_SYNC_PATH_LENGTH || relativePath.includes("\0") || relativePath.includes("\\") || (0, node_path.isAbsolute)(relativePath)) throw new Error("Invalid private sync path");
|
|
44
|
+
const segments = relativePath.split("/");
|
|
45
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) throw new Error("Invalid private sync path");
|
|
46
|
+
if (segments[0] === "shared") throw new Error("The shared workspace tree is not part of private sync");
|
|
47
|
+
return segments.join("/");
|
|
48
|
+
}
|
|
49
|
+
function resolvePrivateWorkspacePath(workspacePath, relativePath) {
|
|
50
|
+
const canonical = validatePrivateRelativePath(relativePath);
|
|
51
|
+
const workspace = (0, node_path.resolve)(workspacePath);
|
|
52
|
+
const absolutePath = (0, node_path.resolve)(workspace, canonical);
|
|
53
|
+
const fromWorkspace = (0, node_path.relative)(workspace, absolutePath);
|
|
54
|
+
if (fromWorkspace === "" || fromWorkspace === ".." || fromWorkspace.startsWith(`..${node_path.sep}`) || (0, node_path.isAbsolute)(fromWorkspace)) throw new Error("Private sync path escapes the workspace");
|
|
55
|
+
return absolutePath;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Reject an existing symlink in any path component. Lexical containment alone
|
|
59
|
+
* is insufficient: `workspace/link/file` can escape when `link` points out of
|
|
60
|
+
* the workspace. This check covers reads, writes, and parent-directory walks.
|
|
61
|
+
*/
|
|
62
|
+
async function assertNoSymlinkTraversal(workspacePath, relativePath) {
|
|
63
|
+
const canonical = validatePrivateRelativePath(relativePath);
|
|
64
|
+
const absolutePath = resolvePrivateWorkspacePath(workspacePath, canonical);
|
|
65
|
+
const segments = canonical.split("/");
|
|
66
|
+
let current = (0, node_path.resolve)(workspacePath);
|
|
67
|
+
for (const segment of segments) {
|
|
68
|
+
current = (0, node_path.resolve)(current, segment);
|
|
69
|
+
try {
|
|
70
|
+
if ((await (0, node_fs_promises.lstat)(current)).isSymbolicLink()) throw new Error("Private sync path crosses a symbolic link");
|
|
71
|
+
} catch (error) {
|
|
72
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") break;
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return absolutePath;
|
|
77
|
+
}
|
|
78
|
+
//#endregion
|
|
32
79
|
//#region src/manifest.ts
|
|
33
80
|
/**
|
|
34
81
|
* AlfeSync manifest — local file manifest at `~/.alfe/sync/manifest.json`.
|
|
@@ -48,6 +95,9 @@ let _auriclabs_logger = require("@auriclabs/logger");
|
|
|
48
95
|
*/
|
|
49
96
|
const SYNC_STATE_DIR = (0, node_path.join)((0, node_os.homedir)(), ".alfe", "sync");
|
|
50
97
|
const MANIFEST_FILE = "manifest.json";
|
|
98
|
+
const MANIFEST_LOCK_FILE = "manifest.lock";
|
|
99
|
+
const MANIFEST_LOCK_TIMEOUT_MS = 1e4;
|
|
100
|
+
const MANIFEST_LOCK_STALE_MS = 6e4;
|
|
51
101
|
/**
|
|
52
102
|
* Resolve the manifest file path. Lives under `~/.alfe/sync/`, independent
|
|
53
103
|
* of the workspace path — one agent has one manifest.
|
|
@@ -55,6 +105,82 @@ const MANIFEST_FILE = "manifest.json";
|
|
|
55
105
|
function manifestPath() {
|
|
56
106
|
return (0, node_path.join)(SYNC_STATE_DIR, MANIFEST_FILE);
|
|
57
107
|
}
|
|
108
|
+
function manifestLockPath() {
|
|
109
|
+
return (0, node_path.join)(SYNC_STATE_DIR, MANIFEST_LOCK_FILE);
|
|
110
|
+
}
|
|
111
|
+
async function readManifestUnlocked() {
|
|
112
|
+
try {
|
|
113
|
+
const raw = await (0, node_fs_promises.readFile)(manifestPath(), "utf-8");
|
|
114
|
+
const parsed = JSON.parse(raw);
|
|
115
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { files: {} };
|
|
116
|
+
const candidate = parsed;
|
|
117
|
+
const files = candidate.files;
|
|
118
|
+
if (typeof files !== "object" || files === null || Array.isArray(files)) return { files: {} };
|
|
119
|
+
const validFiles = {};
|
|
120
|
+
for (const [relativePath, value] of Object.entries(files)) {
|
|
121
|
+
try {
|
|
122
|
+
validatePrivateRelativePath(relativePath);
|
|
123
|
+
} catch {
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
|
|
127
|
+
const entry = value;
|
|
128
|
+
if (typeof entry.hash !== "string" || typeof entry.size !== "number" || !Number.isFinite(entry.size) || entry.size < 0 || typeof entry.lastSynced !== "string" || entry.storageClass !== "STANDARD" && entry.storageClass !== "GLACIER_IR") continue;
|
|
129
|
+
validFiles[relativePath] = {
|
|
130
|
+
hash: entry.hash,
|
|
131
|
+
size: entry.size,
|
|
132
|
+
lastSynced: entry.lastSynced,
|
|
133
|
+
storageClass: entry.storageClass
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
files: validFiles,
|
|
138
|
+
...typeof candidate.agentId === "string" ? { agentId: candidate.agentId } : {}
|
|
139
|
+
};
|
|
140
|
+
} catch {
|
|
141
|
+
return { files: {} };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
async function writeManifestUnlocked(manifest) {
|
|
145
|
+
await (0, node_fs_promises.mkdir)(SYNC_STATE_DIR, { recursive: true });
|
|
146
|
+
const target = manifestPath();
|
|
147
|
+
const temporary = `${target}.${String(process.pid)}.${String(Date.now())}.tmp`;
|
|
148
|
+
try {
|
|
149
|
+
await (0, node_fs_promises.writeFile)(temporary, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
|
|
150
|
+
await (0, node_fs_promises.rename)(temporary, target);
|
|
151
|
+
} catch (error) {
|
|
152
|
+
await (0, node_fs_promises.unlink)(temporary).catch(() => void 0);
|
|
153
|
+
throw error;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
async function withManifestLock(operation) {
|
|
157
|
+
await (0, node_fs_promises.mkdir)(SYNC_STATE_DIR, { recursive: true });
|
|
158
|
+
const lockPath = manifestLockPath();
|
|
159
|
+
const deadline = Date.now() + MANIFEST_LOCK_TIMEOUT_MS;
|
|
160
|
+
for (;;) {
|
|
161
|
+
let handle;
|
|
162
|
+
try {
|
|
163
|
+
handle = await (0, node_fs_promises.open)(lockPath, "wx");
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if ((error instanceof Error && "code" in error ? error.code : void 0) !== "EEXIST") throw error;
|
|
166
|
+
const lockStat = await (0, node_fs_promises.stat)(lockPath).catch(() => null);
|
|
167
|
+
if (lockStat && Date.now() - lockStat.mtimeMs > MANIFEST_LOCK_STALE_MS) {
|
|
168
|
+
await (0, node_fs_promises.unlink)(lockPath).catch(() => void 0);
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
if (Date.now() >= deadline) throw new Error("Timed out waiting for the local sync manifest lock");
|
|
172
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
try {
|
|
176
|
+
await handle.writeFile(`${String(process.pid)}\n`, "utf-8");
|
|
177
|
+
return await operation();
|
|
178
|
+
} finally {
|
|
179
|
+
await handle.close().catch(() => void 0);
|
|
180
|
+
await (0, node_fs_promises.unlink)(lockPath).catch(() => void 0);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
}
|
|
58
184
|
/**
|
|
59
185
|
* Read the local manifest. Returns empty manifest if not found.
|
|
60
186
|
*
|
|
@@ -63,31 +189,25 @@ function manifestPath() {
|
|
|
63
189
|
* `~/.alfe/sync/` regardless of which workspace the call comes from.
|
|
64
190
|
*/
|
|
65
191
|
async function readManifest(workspacePath) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
try {
|
|
69
|
-
const raw = await (0, node_fs_promises.readFile)(path, "utf-8");
|
|
70
|
-
return JSON.parse(raw);
|
|
71
|
-
} catch {
|
|
72
|
-
return { files: {} };
|
|
73
|
-
}
|
|
192
|
+
if (!(0, node_fs.existsSync)(manifestPath())) return { files: {} };
|
|
193
|
+
return readManifestUnlocked();
|
|
74
194
|
}
|
|
75
195
|
/**
|
|
76
196
|
* Write the local manifest. `workspacePath` is accepted for call-site
|
|
77
197
|
* symmetry but unused (see `readManifest`).
|
|
78
198
|
*/
|
|
79
199
|
async function writeManifest(workspacePath, manifest) {
|
|
80
|
-
|
|
81
|
-
await (0, node_fs_promises.mkdir)(SYNC_STATE_DIR, { recursive: true });
|
|
82
|
-
await (0, node_fs_promises.writeFile)(path, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
|
|
200
|
+
await withManifestLock(() => writeManifestUnlocked(manifest));
|
|
83
201
|
}
|
|
84
202
|
/**
|
|
85
203
|
* Update a single file entry in the local manifest.
|
|
86
204
|
*/
|
|
87
205
|
async function updateManifestEntry(workspacePath, relativePath, entry) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
206
|
+
await withManifestLock(async () => {
|
|
207
|
+
const manifest = await readManifestUnlocked();
|
|
208
|
+
manifest.files[relativePath] = entry;
|
|
209
|
+
await writeManifestUnlocked(manifest);
|
|
210
|
+
});
|
|
91
211
|
}
|
|
92
212
|
/**
|
|
93
213
|
* Update many file entries in one read-modify-write pass.
|
|
@@ -98,18 +218,22 @@ async function updateManifestEntry(workspacePath, relativePath, entry) {
|
|
|
98
218
|
* "identical → heal" pass). Optionally stamps the owning agentId.
|
|
99
219
|
*/
|
|
100
220
|
async function updateManifestEntries(workspacePath, entries, options = {}) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
221
|
+
await withManifestLock(async () => {
|
|
222
|
+
const manifest = await readManifestUnlocked();
|
|
223
|
+
Object.assign(manifest.files, entries);
|
|
224
|
+
if (options.agentId) manifest.agentId = options.agentId;
|
|
225
|
+
await writeManifestUnlocked(manifest);
|
|
226
|
+
});
|
|
105
227
|
}
|
|
106
228
|
/**
|
|
107
229
|
* Remove a file entry from the local manifest.
|
|
108
230
|
*/
|
|
109
231
|
async function removeManifestEntry(workspacePath, relativePath) {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
232
|
+
await withManifestLock(async () => {
|
|
233
|
+
const manifest = await readManifestUnlocked();
|
|
234
|
+
manifest.files = Object.fromEntries(Object.entries(manifest.files).filter(([key]) => key !== relativePath));
|
|
235
|
+
await writeManifestUnlocked(manifest);
|
|
236
|
+
});
|
|
113
237
|
}
|
|
114
238
|
/**
|
|
115
239
|
* Compute SHA-256 hash of a file using streaming (memory-efficient).
|
|
@@ -290,6 +414,8 @@ async function withRetry(fn, options = {}) {
|
|
|
290
414
|
* Each step retries with exponential backoff via `withRetry`.
|
|
291
415
|
*/
|
|
292
416
|
const log$2 = (0, _auriclabs_logger.createLogger)("SyncUploader");
|
|
417
|
+
const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
|
|
418
|
+
const MAX_TRANSFER_CONCURRENCY$1 = 100;
|
|
293
419
|
/**
|
|
294
420
|
* Prefixes stored as GLACIER_IR on the sync bucket. Shared with the recovery
|
|
295
421
|
* classifier in sync-engine.ts, which must never cut conflict sidecars for
|
|
@@ -309,20 +435,26 @@ function getContentType(relativePath) {
|
|
|
309
435
|
return "application/octet-stream";
|
|
310
436
|
}
|
|
311
437
|
async function uploadOne(workspacePath, relativePath, client) {
|
|
312
|
-
|
|
438
|
+
let canonicalPath;
|
|
313
439
|
try {
|
|
440
|
+
canonicalPath = validatePrivateRelativePath(relativePath);
|
|
441
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
442
|
+
const fileStat = await (0, node_fs_promises.stat)(absolutePath);
|
|
443
|
+
if (!fileStat.isFile()) throw new Error("Sync upload path is not a file");
|
|
444
|
+
const size = fileStat.size;
|
|
445
|
+
if (size > MAX_UPLOAD_BYTES) throw new Error("File exceeds the sync upload limit");
|
|
446
|
+
const fileContent = await (0, node_fs_promises.readFile)(absolutePath);
|
|
447
|
+
if (fileContent.length > MAX_UPLOAD_BYTES) throw new Error("File exceeds the sync upload limit");
|
|
448
|
+
const hash = `sha256:${(0, node_crypto.createHash)("sha256").update(fileContent).digest("hex")}`;
|
|
449
|
+
const storageClass = getStorageClass(canonicalPath);
|
|
450
|
+
const contentType = getContentType(canonicalPath);
|
|
314
451
|
return await withRetry(async () => {
|
|
315
|
-
const [hash, fileStat] = await Promise.all([computeFileHash(absolutePath), (0, node_fs_promises.stat)(absolutePath)]);
|
|
316
|
-
const size = fileStat.size;
|
|
317
|
-
const storageClass = getStorageClass(relativePath);
|
|
318
|
-
const contentType = getContentType(relativePath);
|
|
319
452
|
const url = (await client.syncPresign({ files: [{
|
|
320
|
-
path:
|
|
453
|
+
path: canonicalPath,
|
|
321
454
|
operation: "put",
|
|
322
455
|
contentType
|
|
323
456
|
}] })).urls[0]?.url;
|
|
324
457
|
if (!url) throw new Error("No presigned URL returned");
|
|
325
|
-
const fileContent = await (0, node_fs_promises.readFile)(absolutePath);
|
|
326
458
|
const putResponse = await fetch(url, {
|
|
327
459
|
method: "PUT",
|
|
328
460
|
headers: { "Content-Type": contentType },
|
|
@@ -330,22 +462,23 @@ async function uploadOne(workspacePath, relativePath, client) {
|
|
|
330
462
|
});
|
|
331
463
|
if (!putResponse.ok) throw new Error(`S3 PUT failed (${String(putResponse.status)}): ${await putResponse.text()}`);
|
|
332
464
|
await client.syncConfirmUpload({
|
|
333
|
-
filePath:
|
|
465
|
+
filePath: canonicalPath,
|
|
334
466
|
hash,
|
|
335
467
|
size,
|
|
336
468
|
storageClass
|
|
337
469
|
});
|
|
470
|
+
const entry = {
|
|
471
|
+
hash,
|
|
472
|
+
size,
|
|
473
|
+
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
474
|
+
storageClass
|
|
475
|
+
};
|
|
338
476
|
return {
|
|
339
|
-
path:
|
|
477
|
+
path: canonicalPath,
|
|
340
478
|
success: true,
|
|
341
479
|
hash,
|
|
342
480
|
size,
|
|
343
|
-
entry
|
|
344
|
-
hash,
|
|
345
|
-
size,
|
|
346
|
-
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
347
|
-
storageClass
|
|
348
|
-
}
|
|
481
|
+
entry
|
|
349
482
|
};
|
|
350
483
|
});
|
|
351
484
|
} catch (err) {
|
|
@@ -361,6 +494,7 @@ async function uploadOne(workspacePath, relativePath, client) {
|
|
|
361
494
|
*/
|
|
362
495
|
async function uploadFiles(workspacePath, relativePaths, client, options = {}) {
|
|
363
496
|
const { concurrency = 5, quiet = false } = options;
|
|
497
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > MAX_TRANSFER_CONCURRENCY$1) throw new Error("Upload concurrency must be an integer between 1 and 100");
|
|
364
498
|
const results = [];
|
|
365
499
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
366
500
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
@@ -399,28 +533,35 @@ function formatBytes$1(bytes) {
|
|
|
399
533
|
* 4. Update the local manifest
|
|
400
534
|
*/
|
|
401
535
|
const log$1 = (0, _auriclabs_logger.createLogger)("SyncDownloader");
|
|
536
|
+
const MAX_DOWNLOAD_BYTES = 100 * 1024 * 1024;
|
|
537
|
+
const MAX_TRANSFER_CONCURRENCY = 100;
|
|
402
538
|
async function downloadOne(workspacePath, relativePath, client, remoteEntry) {
|
|
539
|
+
let canonicalPath;
|
|
403
540
|
try {
|
|
541
|
+
canonicalPath = validatePrivateRelativePath(relativePath);
|
|
404
542
|
return await withRetry(async () => {
|
|
405
543
|
const url = (await client.syncPresign({ files: [{
|
|
406
|
-
path:
|
|
544
|
+
path: canonicalPath,
|
|
407
545
|
operation: "get"
|
|
408
546
|
}] })).urls[0]?.url;
|
|
409
547
|
if (!url) throw new Error("No presigned URL returned");
|
|
410
548
|
const response = await fetch(url);
|
|
411
549
|
if (!response.ok) throw new Error(`S3 GET failed (${String(response.status)}): ${await response.text()}`);
|
|
550
|
+
const contentLength = Number(response.headers.get("content-length") ?? "0");
|
|
551
|
+
if (Number.isFinite(contentLength) && contentLength > MAX_DOWNLOAD_BYTES) throw new Error("S3 object exceeds the local sync download limit");
|
|
412
552
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
413
|
-
|
|
553
|
+
if (buffer.length > MAX_DOWNLOAD_BYTES) throw new Error("S3 object exceeds the local sync download limit");
|
|
554
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
414
555
|
await (0, node_fs_promises.mkdir)((0, node_path.dirname)(absolutePath), { recursive: true });
|
|
415
556
|
await (0, node_fs_promises.writeFile)(absolutePath, buffer);
|
|
416
557
|
const entry = {
|
|
417
558
|
hash: remoteEntry?.hash ?? "",
|
|
418
559
|
size: buffer.length,
|
|
419
560
|
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
420
|
-
storageClass: remoteEntry?.storageClass
|
|
561
|
+
storageClass: remoteEntry?.storageClass === "GLACIER_IR" ? "GLACIER_IR" : "STANDARD"
|
|
421
562
|
};
|
|
422
563
|
return {
|
|
423
|
-
path:
|
|
564
|
+
path: canonicalPath,
|
|
424
565
|
success: true,
|
|
425
566
|
size: buffer.length,
|
|
426
567
|
entry
|
|
@@ -436,6 +577,7 @@ async function downloadOne(workspacePath, relativePath, client, remoteEntry) {
|
|
|
436
577
|
}
|
|
437
578
|
async function downloadFiles(workspacePath, relativePaths, client, remoteManifest, options = {}) {
|
|
438
579
|
const { concurrency = 5, quiet = false } = options;
|
|
580
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > MAX_TRANSFER_CONCURRENCY) throw new Error("Download concurrency must be an integer between 1 and 100");
|
|
439
581
|
const results = [];
|
|
440
582
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
441
583
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
@@ -485,6 +627,15 @@ const log = (0, _auriclabs_logger.createLogger)("SyncEngine");
|
|
|
485
627
|
const MAX_PRESERVE_BYTES = 100 * 1024 * 1024;
|
|
486
628
|
/** Marker that keeps the heartbeat recovery nudge idempotent across runs. */
|
|
487
629
|
const RECOVERY_NUDGE_MARKER = "<!-- alfe-sync:recovery-nudge -->";
|
|
630
|
+
function filterSafePrivatePaths(paths) {
|
|
631
|
+
const safe = [];
|
|
632
|
+
for (const candidate of paths) try {
|
|
633
|
+
safe.push(validatePrivateRelativePath(candidate));
|
|
634
|
+
} catch {
|
|
635
|
+
log.warn("Ignored an invalid path from sync state");
|
|
636
|
+
}
|
|
637
|
+
return [...new Set(safe)];
|
|
638
|
+
}
|
|
488
639
|
/**
|
|
489
640
|
* Recovery artifacts: `.conflict-<ts>` sidecars and `RECOVERY-<ts>.md`
|
|
490
641
|
* reports. Anchored to the timestamp shape so legitimate user files like
|
|
@@ -522,7 +673,8 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
522
673
|
async push(paths, options = {}) {
|
|
523
674
|
const { quiet = false, filter } = options;
|
|
524
675
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
525
|
-
let filesToPush = paths && paths.length > 0 ? filterIgnored(paths, ignorePatterns) : await detectLocalChanges(workspacePath, ignorePatterns);
|
|
676
|
+
let filesToPush = paths && paths.length > 0 ? filterIgnored(filterSafePrivatePaths(paths), ignorePatterns) : await detectLocalChanges(workspacePath, ignorePatterns);
|
|
677
|
+
if (paths && paths.length > 0) filesToPush = await filterChangedExplicitPaths(workspacePath, filesToPush);
|
|
526
678
|
if (filter) filesToPush = filesToPush.filter((p) => p.startsWith(filter));
|
|
527
679
|
if (filesToPush.length === 0) {
|
|
528
680
|
if (!quiet) log.info("Nothing to push");
|
|
@@ -556,7 +708,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
556
708
|
if (!quiet) log.info(`Deleting ${String(paths.length)} file(s) from cloud`);
|
|
557
709
|
let deleted = 0;
|
|
558
710
|
let errors = 0;
|
|
559
|
-
for (const path of paths) try {
|
|
711
|
+
for (const path of filterSafePrivatePaths(paths)) try {
|
|
560
712
|
await client.syncDeleteFile(path);
|
|
561
713
|
await removeManifestEntry(workspacePath, path);
|
|
562
714
|
deleted++;
|
|
@@ -587,7 +739,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
587
739
|
};
|
|
588
740
|
}
|
|
589
741
|
if (!quiet) log.info(`Pulling ${String(diff.toPull.length)} file(s)`);
|
|
590
|
-
const results = await downloadFiles(workspacePath, [...diff.toPull], client, remoteManifest, { quiet });
|
|
742
|
+
const results = await downloadFiles(workspacePath, filterSafePrivatePaths([...diff.toPull]), client, remoteManifest, { quiet });
|
|
591
743
|
const pulled = results.filter((r) => r.success).length;
|
|
592
744
|
const errors = results.filter((r) => !r.success).length;
|
|
593
745
|
if (!quiet) log.info(`Pull complete: ${String(pulled)} downloaded, ${String(errors)} failed`);
|
|
@@ -604,8 +756,8 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
604
756
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
605
757
|
const localChanges = await detectLocalChanges(workspacePath, ignorePatterns);
|
|
606
758
|
const diff = diffManifests(localManifest, remoteManifest);
|
|
607
|
-
const trueConflicts = diff.conflicts.filter((p) => localChanges.includes(p));
|
|
608
|
-
const remoteOnlyChanges = diff.conflicts.filter((p) => !localChanges.includes(p));
|
|
759
|
+
const trueConflicts = filterSafePrivatePaths(diff.conflicts.filter((p) => localChanges.includes(p)));
|
|
760
|
+
const remoteOnlyChanges = filterSafePrivatePaths(diff.conflicts.filter((p) => !localChanges.includes(p)));
|
|
609
761
|
let conflictCount = 0;
|
|
610
762
|
const conflictTimestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
611
763
|
for (const conflictPath of trueConflicts) {
|
|
@@ -614,12 +766,12 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
614
766
|
if (outcome.status !== "too-large" && !quiet) log.warn(`Conflict: ${conflictPath} — saved as ${outcome.conflictPath}`);
|
|
615
767
|
conflictCount++;
|
|
616
768
|
}
|
|
617
|
-
const filesToPush = filterIgnored([...diff.toPush, ...localChanges.filter((p) => !diff.conflicts.includes(p))], ignorePatterns);
|
|
618
|
-
const filesToPull = [
|
|
769
|
+
const filesToPush = filterIgnored(filterSafePrivatePaths([...diff.toPush, ...localChanges.filter((p) => !diff.conflicts.includes(p))]), ignorePatterns);
|
|
770
|
+
const filesToPull = filterSafePrivatePaths([
|
|
619
771
|
...diff.toPull,
|
|
620
772
|
...remoteOnlyChanges,
|
|
621
773
|
...trueConflicts
|
|
622
|
-
];
|
|
774
|
+
]);
|
|
623
775
|
const pushResult = {
|
|
624
776
|
pushed: 0,
|
|
625
777
|
errors: 0
|
|
@@ -652,7 +804,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
652
804
|
async firstRunReconcile(options = {}) {
|
|
653
805
|
const { quiet = false } = options;
|
|
654
806
|
const remoteManifest = await client.syncGetManifest();
|
|
655
|
-
const remotePaths = Object.keys(remoteManifest.files);
|
|
807
|
+
const remotePaths = filterSafePrivatePaths(Object.keys(remoteManifest.files));
|
|
656
808
|
if (remotePaths.length === 0) {
|
|
657
809
|
if (!quiet) log.info("First-run sync: no cloud state — seeding workspace to cloud");
|
|
658
810
|
const seedResult = await this.push(void 0, options);
|
|
@@ -703,7 +855,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
703
855
|
let recoveryReportPath = null;
|
|
704
856
|
if (preserved.length > 0 && (newlyPreserved > 0 || !await recoveryReportExists(workspacePath))) {
|
|
705
857
|
recoveryReportPath = `${agentDirPrefix(workspacePath)}RECOVERY-${timestamp}.md`;
|
|
706
|
-
const reportAbsolute =
|
|
858
|
+
const reportAbsolute = await assertNoSymlinkTraversal(workspacePath, recoveryReportPath);
|
|
707
859
|
await (0, node_fs_promises.mkdir)((0, node_path.dirname)(reportAbsolute), { recursive: true });
|
|
708
860
|
await (0, node_fs_promises.writeFile)(reportAbsolute, buildRecoveryReport(preserved, workspacePath), "utf-8");
|
|
709
861
|
log.warn(`Recovery: ${String(preserved.length)} diverged local file(s) — see ${recoveryReportPath}`);
|
|
@@ -765,7 +917,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
765
917
|
};
|
|
766
918
|
const remoteManifest = await client.syncGetManifest();
|
|
767
919
|
const localManifest = await readManifest(workspacePath);
|
|
768
|
-
const filesToPull = paths.filter((p) => {
|
|
920
|
+
const filesToPull = filterSafePrivatePaths(paths).filter((p) => {
|
|
769
921
|
if (!(p in remoteManifest.files)) return false;
|
|
770
922
|
if (!(p in localManifest.files)) return true;
|
|
771
923
|
return localManifest.files[p].hash !== remoteManifest.files[p].hash;
|
|
@@ -795,7 +947,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
795
947
|
const { quiet = false, limit = 1e3 } = options;
|
|
796
948
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
797
949
|
const remoteManifest = await client.syncGetManifest();
|
|
798
|
-
const ignoredPaths = Object.keys(remoteManifest.files).filter((p) => shouldIgnore(p, ignorePatterns));
|
|
950
|
+
const ignoredPaths = filterSafePrivatePaths(Object.keys(remoteManifest.files)).filter((p) => shouldIgnore(p, ignorePatterns));
|
|
799
951
|
if (ignoredPaths.length === 0) return {
|
|
800
952
|
pushed: 0,
|
|
801
953
|
pulled: 0,
|
|
@@ -811,20 +963,38 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
811
963
|
},
|
|
812
964
|
async removeLocalFile(filePath, options = {}) {
|
|
813
965
|
const { quiet = false } = options;
|
|
814
|
-
const
|
|
966
|
+
const canonicalPath = validatePrivateRelativePath(filePath);
|
|
967
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
815
968
|
try {
|
|
816
969
|
const { unlink } = await import("node:fs/promises");
|
|
817
970
|
if ((0, node_fs.existsSync)(absolutePath)) {
|
|
818
971
|
await unlink(absolutePath);
|
|
819
|
-
if (!quiet) log.info(`Deleted: ${
|
|
972
|
+
if (!quiet) log.info(`Deleted: ${canonicalPath}`);
|
|
820
973
|
}
|
|
821
974
|
} catch (err) {
|
|
822
|
-
if (!quiet) log.error({ err }, `Failed to delete ${
|
|
975
|
+
if (!quiet) log.error({ err }, `Failed to delete ${canonicalPath}`);
|
|
823
976
|
}
|
|
824
|
-
await removeManifestEntry(workspacePath,
|
|
977
|
+
await removeManifestEntry(workspacePath, canonicalPath);
|
|
825
978
|
}
|
|
826
979
|
};
|
|
827
980
|
}
|
|
981
|
+
async function filterChangedExplicitPaths(workspacePath, paths) {
|
|
982
|
+
const manifest = await readManifest(workspacePath);
|
|
983
|
+
const changed = [];
|
|
984
|
+
for (const relativePath of paths) {
|
|
985
|
+
if (!(relativePath in manifest.files)) {
|
|
986
|
+
changed.push(relativePath);
|
|
987
|
+
continue;
|
|
988
|
+
}
|
|
989
|
+
const entry = manifest.files[relativePath];
|
|
990
|
+
try {
|
|
991
|
+
if (await computeFileHash(await assertNoSymlinkTraversal(workspacePath, relativePath)) !== entry.hash) changed.push(relativePath);
|
|
992
|
+
} catch {
|
|
993
|
+
changed.push(relativePath);
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
return changed;
|
|
997
|
+
}
|
|
828
998
|
/**
|
|
829
999
|
* Walk the workspace and return paths whose hash differs from the manifest
|
|
830
1000
|
* (or that are missing from it entirely).
|
|
@@ -880,10 +1050,11 @@ async function recoveryReportExists(workspacePath) {
|
|
|
880
1050
|
* `sourceHash` when the caller already hashed the file.
|
|
881
1051
|
*/
|
|
882
1052
|
async function preserveConflictCopy(workspacePath, relativePath, timestamp, options = {}) {
|
|
883
|
-
const
|
|
884
|
-
const
|
|
885
|
-
const
|
|
886
|
-
const
|
|
1053
|
+
const canonicalPath = validatePrivateRelativePath(relativePath);
|
|
1054
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
1055
|
+
const ext = (0, node_path.extname)(canonicalPath);
|
|
1056
|
+
const base = (0, node_path.basename)(canonicalPath, ext);
|
|
1057
|
+
const dir = (0, node_path.dirname)(canonicalPath);
|
|
887
1058
|
try {
|
|
888
1059
|
const fileStat = await (0, node_fs_promises.stat)(absolutePath);
|
|
889
1060
|
if (options.maxBytes !== void 0 && fileStat.size > options.maxBytes) return {
|
|
@@ -895,16 +1066,17 @@ async function preserveConflictCopy(workspacePath, relativePath, timestamp, opti
|
|
|
895
1066
|
for (const name of siblings) {
|
|
896
1067
|
if (!name.startsWith(`${base}.conflict-`)) continue;
|
|
897
1068
|
if (ext !== "" && !name.endsWith(ext)) continue;
|
|
898
|
-
if (await
|
|
1069
|
+
if (await assertNoSymlinkTraversal(workspacePath, dir === "." ? name : `${dir}/${name}`).then((path) => computeFileHash(path)).catch(() => null) === sourceHash) return {
|
|
899
1070
|
status: "already-preserved",
|
|
900
1071
|
conflictPath: (0, node_path.join)(dir, name).replace(/\\/g, "/")
|
|
901
1072
|
};
|
|
902
1073
|
}
|
|
903
1074
|
const conflictName = `${base}.conflict-${timestamp}${ext}`;
|
|
904
|
-
|
|
1075
|
+
const conflictPath = dir === "." ? conflictName : `${dir}/${conflictName}`;
|
|
1076
|
+
await (0, node_fs_promises.copyFile)(absolutePath, await assertNoSymlinkTraversal(workspacePath, conflictPath));
|
|
905
1077
|
return {
|
|
906
1078
|
status: "preserved",
|
|
907
|
-
conflictPath
|
|
1079
|
+
conflictPath
|
|
908
1080
|
};
|
|
909
1081
|
} catch {
|
|
910
1082
|
return { status: "vanished" };
|
|
@@ -931,7 +1103,13 @@ async function classifyRestorePaths(workspacePath, restorePaths, remoteFiles, lo
|
|
|
931
1103
|
async function classifyOne(path) {
|
|
932
1104
|
const remote = remoteFiles[path];
|
|
933
1105
|
const entry = manifestTrusted ? localManifest.files[path] : void 0;
|
|
934
|
-
|
|
1106
|
+
let absolutePath;
|
|
1107
|
+
try {
|
|
1108
|
+
absolutePath = await assertNoSymlinkTraversal(workspacePath, path);
|
|
1109
|
+
} catch {
|
|
1110
|
+
log.warn("Skipped a cloud sync path that crosses a local symbolic link");
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
935
1113
|
let fileStat = null;
|
|
936
1114
|
try {
|
|
937
1115
|
fileStat = await (0, node_fs_promises.stat)(absolutePath);
|
|
@@ -960,7 +1138,7 @@ async function classifyRestorePaths(workspacePath, restorePaths, remoteFiles, lo
|
|
|
960
1138
|
hash: remote.hash,
|
|
961
1139
|
size: fileStat.size,
|
|
962
1140
|
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
963
|
-
storageClass: remote.storageClass
|
|
1141
|
+
storageClass: remote.storageClass === "GLACIER_IR" ? "GLACIER_IR" : "STANDARD"
|
|
964
1142
|
};
|
|
965
1143
|
return;
|
|
966
1144
|
}
|
|
@@ -1020,7 +1198,7 @@ function buildRecoveryReport(records, workspacePath) {
|
|
|
1020
1198
|
async function appendRecoveryNudge(workspacePath, recoveryReportPath, preservedCount) {
|
|
1021
1199
|
const prefix = agentDirPrefix(workspacePath);
|
|
1022
1200
|
const heartbeatPath = `${prefix}HEARTBEAT.md`;
|
|
1023
|
-
const absolutePath =
|
|
1201
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, heartbeatPath);
|
|
1024
1202
|
if (((0, node_fs.existsSync)(absolutePath) ? await (0, node_fs_promises.readFile)(absolutePath, "utf-8") : "").includes(RECOVERY_NUDGE_MARKER)) return null;
|
|
1025
1203
|
const reportForAgent = recoveryReportPath.startsWith(prefix) ? recoveryReportPath.slice(prefix.length) : recoveryReportPath;
|
|
1026
1204
|
const block = [
|
|
@@ -1052,6 +1230,12 @@ Object.defineProperty(exports, "__toESM", {
|
|
|
1052
1230
|
return __toESM;
|
|
1053
1231
|
}
|
|
1054
1232
|
});
|
|
1233
|
+
Object.defineProperty(exports, "assertNoSymlinkTraversal", {
|
|
1234
|
+
enumerable: true,
|
|
1235
|
+
get: function() {
|
|
1236
|
+
return assertNoSymlinkTraversal;
|
|
1237
|
+
}
|
|
1238
|
+
});
|
|
1055
1239
|
Object.defineProperty(exports, "computeFileHash", {
|
|
1056
1240
|
enumerable: true,
|
|
1057
1241
|
get: function() {
|
|
@@ -1106,6 +1290,12 @@ Object.defineProperty(exports, "removeManifestEntry", {
|
|
|
1106
1290
|
return removeManifestEntry;
|
|
1107
1291
|
}
|
|
1108
1292
|
});
|
|
1293
|
+
Object.defineProperty(exports, "resolvePrivateWorkspacePath", {
|
|
1294
|
+
enumerable: true,
|
|
1295
|
+
get: function() {
|
|
1296
|
+
return resolvePrivateWorkspacePath;
|
|
1297
|
+
}
|
|
1298
|
+
});
|
|
1109
1299
|
Object.defineProperty(exports, "shouldIgnore", {
|
|
1110
1300
|
enumerable: true,
|
|
1111
1301
|
get: function() {
|
|
@@ -1130,6 +1320,12 @@ Object.defineProperty(exports, "uploadFiles", {
|
|
|
1130
1320
|
return uploadFiles;
|
|
1131
1321
|
}
|
|
1132
1322
|
});
|
|
1323
|
+
Object.defineProperty(exports, "validatePrivateRelativePath", {
|
|
1324
|
+
enumerable: true,
|
|
1325
|
+
get: function() {
|
|
1326
|
+
return validatePrivateRelativePath;
|
|
1327
|
+
}
|
|
1328
|
+
});
|
|
1133
1329
|
Object.defineProperty(exports, "withRetry", {
|
|
1134
1330
|
enumerable: true,
|
|
1135
1331
|
get: function() {
|