@alfe.ai/openclaw-sync 0.3.6 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cli/index.cjs +17 -25
- package/dist/cli/index.js +18 -26
- package/dist/cli/index.js.map +1 -1
- package/dist/index.cjs +3 -0
- package/dist/index.d.cts +132 -56
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +132 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/plugin.d.cts.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin2.cjs +392 -138
- package/dist/plugin2.js +396 -142
- package/dist/plugin2.js.map +1 -1
- package/dist/sync-engine.cjs +262 -66
- package/dist/sync-engine.js +247 -69
- package/dist/sync-engine.js.map +1 -1
- package/package.json +4 -4
package/dist/sync-engine.js
CHANGED
|
@@ -1,11 +1,58 @@
|
|
|
1
|
-
import { appendFile, copyFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
|
|
1
|
+
import { appendFile, copyFile, lstat, mkdir, open, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
|
|
2
2
|
import { createReadStream, existsSync, readFileSync } from "node:fs";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import { basename, dirname, extname, join } from "node:path";
|
|
4
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { homedir } from "node:os";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import micromatch from "micromatch";
|
|
8
8
|
import { createLogger } from "@auriclabs/logger";
|
|
9
|
+
//#region src/path-contract.ts
|
|
10
|
+
/**
|
|
11
|
+
* Canonical path contract for the private workspace sync data plane.
|
|
12
|
+
*
|
|
13
|
+
* Paths received from manifests, relays, CLI arguments, and watcher events
|
|
14
|
+
* are identifiers relative to the configured workspace. Keep validation in
|
|
15
|
+
* one place so a malformed remote path can never become an arbitrary local
|
|
16
|
+
* read, write, or delete.
|
|
17
|
+
*/
|
|
18
|
+
const MAX_SYNC_PATH_LENGTH = 1024;
|
|
19
|
+
function validatePrivateRelativePath(relativePath) {
|
|
20
|
+
if (relativePath.length === 0 || relativePath.length > MAX_SYNC_PATH_LENGTH || relativePath.includes("\0") || relativePath.includes("\\") || isAbsolute(relativePath)) throw new Error("Invalid private sync path");
|
|
21
|
+
const segments = relativePath.split("/");
|
|
22
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) throw new Error("Invalid private sync path");
|
|
23
|
+
if (segments[0] === "shared") throw new Error("The shared workspace tree is not part of private sync");
|
|
24
|
+
return segments.join("/");
|
|
25
|
+
}
|
|
26
|
+
function resolvePrivateWorkspacePath(workspacePath, relativePath) {
|
|
27
|
+
const canonical = validatePrivateRelativePath(relativePath);
|
|
28
|
+
const workspace = resolve(workspacePath);
|
|
29
|
+
const absolutePath = resolve(workspace, canonical);
|
|
30
|
+
const fromWorkspace = relative(workspace, absolutePath);
|
|
31
|
+
if (fromWorkspace === "" || fromWorkspace === ".." || fromWorkspace.startsWith(`..${sep}`) || isAbsolute(fromWorkspace)) throw new Error("Private sync path escapes the workspace");
|
|
32
|
+
return absolutePath;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Reject an existing symlink in any path component. Lexical containment alone
|
|
36
|
+
* is insufficient: `workspace/link/file` can escape when `link` points out of
|
|
37
|
+
* the workspace. This check covers reads, writes, and parent-directory walks.
|
|
38
|
+
*/
|
|
39
|
+
async function assertNoSymlinkTraversal(workspacePath, relativePath) {
|
|
40
|
+
const canonical = validatePrivateRelativePath(relativePath);
|
|
41
|
+
const absolutePath = resolvePrivateWorkspacePath(workspacePath, canonical);
|
|
42
|
+
const segments = canonical.split("/");
|
|
43
|
+
let current = resolve(workspacePath);
|
|
44
|
+
for (const segment of segments) {
|
|
45
|
+
current = resolve(current, segment);
|
|
46
|
+
try {
|
|
47
|
+
if ((await lstat(current)).isSymbolicLink()) throw new Error("Private sync path crosses a symbolic link");
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") break;
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return absolutePath;
|
|
54
|
+
}
|
|
55
|
+
//#endregion
|
|
9
56
|
//#region src/manifest.ts
|
|
10
57
|
/**
|
|
11
58
|
* AlfeSync manifest — local file manifest at `~/.alfe/sync/manifest.json`.
|
|
@@ -25,6 +72,9 @@ import { createLogger } from "@auriclabs/logger";
|
|
|
25
72
|
*/
|
|
26
73
|
const SYNC_STATE_DIR = join(homedir(), ".alfe", "sync");
|
|
27
74
|
const MANIFEST_FILE = "manifest.json";
|
|
75
|
+
const MANIFEST_LOCK_FILE = "manifest.lock";
|
|
76
|
+
const MANIFEST_LOCK_TIMEOUT_MS = 1e4;
|
|
77
|
+
const MANIFEST_LOCK_STALE_MS = 6e4;
|
|
28
78
|
/**
|
|
29
79
|
* Resolve the manifest file path. Lives under `~/.alfe/sync/`, independent
|
|
30
80
|
* of the workspace path — one agent has one manifest.
|
|
@@ -32,6 +82,82 @@ const MANIFEST_FILE = "manifest.json";
|
|
|
32
82
|
function manifestPath() {
|
|
33
83
|
return join(SYNC_STATE_DIR, MANIFEST_FILE);
|
|
34
84
|
}
|
|
85
|
+
function manifestLockPath() {
|
|
86
|
+
return join(SYNC_STATE_DIR, MANIFEST_LOCK_FILE);
|
|
87
|
+
}
|
|
88
|
+
async function readManifestUnlocked() {
|
|
89
|
+
try {
|
|
90
|
+
const raw = await readFile(manifestPath(), "utf-8");
|
|
91
|
+
const parsed = JSON.parse(raw);
|
|
92
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return { files: {} };
|
|
93
|
+
const candidate = parsed;
|
|
94
|
+
const files = candidate.files;
|
|
95
|
+
if (typeof files !== "object" || files === null || Array.isArray(files)) return { files: {} };
|
|
96
|
+
const validFiles = {};
|
|
97
|
+
for (const [relativePath, value] of Object.entries(files)) {
|
|
98
|
+
try {
|
|
99
|
+
validatePrivateRelativePath(relativePath);
|
|
100
|
+
} catch {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) continue;
|
|
104
|
+
const entry = value;
|
|
105
|
+
if (typeof entry.hash !== "string" || typeof entry.size !== "number" || !Number.isFinite(entry.size) || entry.size < 0 || typeof entry.lastSynced !== "string" || entry.storageClass !== "STANDARD" && entry.storageClass !== "GLACIER_IR") continue;
|
|
106
|
+
validFiles[relativePath] = {
|
|
107
|
+
hash: entry.hash,
|
|
108
|
+
size: entry.size,
|
|
109
|
+
lastSynced: entry.lastSynced,
|
|
110
|
+
storageClass: entry.storageClass
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
files: validFiles,
|
|
115
|
+
...typeof candidate.agentId === "string" ? { agentId: candidate.agentId } : {}
|
|
116
|
+
};
|
|
117
|
+
} catch {
|
|
118
|
+
return { files: {} };
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async function writeManifestUnlocked(manifest) {
|
|
122
|
+
await mkdir(SYNC_STATE_DIR, { recursive: true });
|
|
123
|
+
const target = manifestPath();
|
|
124
|
+
const temporary = `${target}.${String(process.pid)}.${String(Date.now())}.tmp`;
|
|
125
|
+
try {
|
|
126
|
+
await writeFile(temporary, `${JSON.stringify(manifest, null, 2)}\n`, "utf-8");
|
|
127
|
+
await rename(temporary, target);
|
|
128
|
+
} catch (error) {
|
|
129
|
+
await unlink(temporary).catch(() => void 0);
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
async function withManifestLock(operation) {
|
|
134
|
+
await mkdir(SYNC_STATE_DIR, { recursive: true });
|
|
135
|
+
const lockPath = manifestLockPath();
|
|
136
|
+
const deadline = Date.now() + MANIFEST_LOCK_TIMEOUT_MS;
|
|
137
|
+
for (;;) {
|
|
138
|
+
let handle;
|
|
139
|
+
try {
|
|
140
|
+
handle = await open(lockPath, "wx");
|
|
141
|
+
} catch (error) {
|
|
142
|
+
if ((error instanceof Error && "code" in error ? error.code : void 0) !== "EEXIST") throw error;
|
|
143
|
+
const lockStat = await stat(lockPath).catch(() => null);
|
|
144
|
+
if (lockStat && Date.now() - lockStat.mtimeMs > MANIFEST_LOCK_STALE_MS) {
|
|
145
|
+
await unlink(lockPath).catch(() => void 0);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (Date.now() >= deadline) throw new Error("Timed out waiting for the local sync manifest lock");
|
|
149
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
await handle.writeFile(`${String(process.pid)}\n`, "utf-8");
|
|
154
|
+
return await operation();
|
|
155
|
+
} finally {
|
|
156
|
+
await handle.close().catch(() => void 0);
|
|
157
|
+
await unlink(lockPath).catch(() => void 0);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
35
161
|
/**
|
|
36
162
|
* Read the local manifest. Returns empty manifest if not found.
|
|
37
163
|
*
|
|
@@ -40,31 +166,25 @@ function manifestPath() {
|
|
|
40
166
|
* `~/.alfe/sync/` regardless of which workspace the call comes from.
|
|
41
167
|
*/
|
|
42
168
|
async function readManifest(workspacePath) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
try {
|
|
46
|
-
const raw = await readFile(path, "utf-8");
|
|
47
|
-
return JSON.parse(raw);
|
|
48
|
-
} catch {
|
|
49
|
-
return { files: {} };
|
|
50
|
-
}
|
|
169
|
+
if (!existsSync(manifestPath())) return { files: {} };
|
|
170
|
+
return readManifestUnlocked();
|
|
51
171
|
}
|
|
52
172
|
/**
|
|
53
173
|
* Write the local manifest. `workspacePath` is accepted for call-site
|
|
54
174
|
* symmetry but unused (see `readManifest`).
|
|
55
175
|
*/
|
|
56
176
|
async function writeManifest(workspacePath, manifest) {
|
|
57
|
-
|
|
58
|
-
await mkdir(SYNC_STATE_DIR, { recursive: true });
|
|
59
|
-
await writeFile(path, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
|
|
177
|
+
await withManifestLock(() => writeManifestUnlocked(manifest));
|
|
60
178
|
}
|
|
61
179
|
/**
|
|
62
180
|
* Update a single file entry in the local manifest.
|
|
63
181
|
*/
|
|
64
182
|
async function updateManifestEntry(workspacePath, relativePath, entry) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
183
|
+
await withManifestLock(async () => {
|
|
184
|
+
const manifest = await readManifestUnlocked();
|
|
185
|
+
manifest.files[relativePath] = entry;
|
|
186
|
+
await writeManifestUnlocked(manifest);
|
|
187
|
+
});
|
|
68
188
|
}
|
|
69
189
|
/**
|
|
70
190
|
* Update many file entries in one read-modify-write pass.
|
|
@@ -75,18 +195,22 @@ async function updateManifestEntry(workspacePath, relativePath, entry) {
|
|
|
75
195
|
* "identical → heal" pass). Optionally stamps the owning agentId.
|
|
76
196
|
*/
|
|
77
197
|
async function updateManifestEntries(workspacePath, entries, options = {}) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
198
|
+
await withManifestLock(async () => {
|
|
199
|
+
const manifest = await readManifestUnlocked();
|
|
200
|
+
Object.assign(manifest.files, entries);
|
|
201
|
+
if (options.agentId) manifest.agentId = options.agentId;
|
|
202
|
+
await writeManifestUnlocked(manifest);
|
|
203
|
+
});
|
|
82
204
|
}
|
|
83
205
|
/**
|
|
84
206
|
* Remove a file entry from the local manifest.
|
|
85
207
|
*/
|
|
86
208
|
async function removeManifestEntry(workspacePath, relativePath) {
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
209
|
+
await withManifestLock(async () => {
|
|
210
|
+
const manifest = await readManifestUnlocked();
|
|
211
|
+
manifest.files = Object.fromEntries(Object.entries(manifest.files).filter(([key]) => key !== relativePath));
|
|
212
|
+
await writeManifestUnlocked(manifest);
|
|
213
|
+
});
|
|
90
214
|
}
|
|
91
215
|
/**
|
|
92
216
|
* Compute SHA-256 hash of a file using streaming (memory-efficient).
|
|
@@ -267,6 +391,8 @@ async function withRetry(fn, options = {}) {
|
|
|
267
391
|
* Each step retries with exponential backoff via `withRetry`.
|
|
268
392
|
*/
|
|
269
393
|
const log$2 = createLogger("SyncUploader");
|
|
394
|
+
const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
|
|
395
|
+
const MAX_TRANSFER_CONCURRENCY$1 = 100;
|
|
270
396
|
/**
|
|
271
397
|
* Prefixes stored as GLACIER_IR on the sync bucket. Shared with the recovery
|
|
272
398
|
* classifier in sync-engine.ts, which must never cut conflict sidecars for
|
|
@@ -286,20 +412,26 @@ function getContentType(relativePath) {
|
|
|
286
412
|
return "application/octet-stream";
|
|
287
413
|
}
|
|
288
414
|
async function uploadOne(workspacePath, relativePath, client) {
|
|
289
|
-
|
|
415
|
+
let canonicalPath;
|
|
290
416
|
try {
|
|
417
|
+
canonicalPath = validatePrivateRelativePath(relativePath);
|
|
418
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
419
|
+
const fileStat = await stat(absolutePath);
|
|
420
|
+
if (!fileStat.isFile()) throw new Error("Sync upload path is not a file");
|
|
421
|
+
const size = fileStat.size;
|
|
422
|
+
if (size > MAX_UPLOAD_BYTES) throw new Error("File exceeds the sync upload limit");
|
|
423
|
+
const fileContent = await readFile(absolutePath);
|
|
424
|
+
if (fileContent.length > MAX_UPLOAD_BYTES) throw new Error("File exceeds the sync upload limit");
|
|
425
|
+
const hash = `sha256:${createHash("sha256").update(fileContent).digest("hex")}`;
|
|
426
|
+
const storageClass = getStorageClass(canonicalPath);
|
|
427
|
+
const contentType = getContentType(canonicalPath);
|
|
291
428
|
return await withRetry(async () => {
|
|
292
|
-
const [hash, fileStat] = await Promise.all([computeFileHash(absolutePath), stat(absolutePath)]);
|
|
293
|
-
const size = fileStat.size;
|
|
294
|
-
const storageClass = getStorageClass(relativePath);
|
|
295
|
-
const contentType = getContentType(relativePath);
|
|
296
429
|
const url = (await client.syncPresign({ files: [{
|
|
297
|
-
path:
|
|
430
|
+
path: canonicalPath,
|
|
298
431
|
operation: "put",
|
|
299
432
|
contentType
|
|
300
433
|
}] })).urls[0]?.url;
|
|
301
434
|
if (!url) throw new Error("No presigned URL returned");
|
|
302
|
-
const fileContent = await readFile(absolutePath);
|
|
303
435
|
const putResponse = await fetch(url, {
|
|
304
436
|
method: "PUT",
|
|
305
437
|
headers: { "Content-Type": contentType },
|
|
@@ -307,22 +439,23 @@ async function uploadOne(workspacePath, relativePath, client) {
|
|
|
307
439
|
});
|
|
308
440
|
if (!putResponse.ok) throw new Error(`S3 PUT failed (${String(putResponse.status)}): ${await putResponse.text()}`);
|
|
309
441
|
await client.syncConfirmUpload({
|
|
310
|
-
filePath:
|
|
442
|
+
filePath: canonicalPath,
|
|
311
443
|
hash,
|
|
312
444
|
size,
|
|
313
445
|
storageClass
|
|
314
446
|
});
|
|
447
|
+
const entry = {
|
|
448
|
+
hash,
|
|
449
|
+
size,
|
|
450
|
+
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
451
|
+
storageClass
|
|
452
|
+
};
|
|
315
453
|
return {
|
|
316
|
-
path:
|
|
454
|
+
path: canonicalPath,
|
|
317
455
|
success: true,
|
|
318
456
|
hash,
|
|
319
457
|
size,
|
|
320
|
-
entry
|
|
321
|
-
hash,
|
|
322
|
-
size,
|
|
323
|
-
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
324
|
-
storageClass
|
|
325
|
-
}
|
|
458
|
+
entry
|
|
326
459
|
};
|
|
327
460
|
});
|
|
328
461
|
} catch (err) {
|
|
@@ -338,6 +471,7 @@ async function uploadOne(workspacePath, relativePath, client) {
|
|
|
338
471
|
*/
|
|
339
472
|
async function uploadFiles(workspacePath, relativePaths, client, options = {}) {
|
|
340
473
|
const { concurrency = 5, quiet = false } = options;
|
|
474
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > MAX_TRANSFER_CONCURRENCY$1) throw new Error("Upload concurrency must be an integer between 1 and 100");
|
|
341
475
|
const results = [];
|
|
342
476
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
343
477
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
@@ -376,28 +510,35 @@ function formatBytes$1(bytes) {
|
|
|
376
510
|
* 4. Update the local manifest
|
|
377
511
|
*/
|
|
378
512
|
const log$1 = createLogger("SyncDownloader");
|
|
513
|
+
const MAX_DOWNLOAD_BYTES = 100 * 1024 * 1024;
|
|
514
|
+
const MAX_TRANSFER_CONCURRENCY = 100;
|
|
379
515
|
async function downloadOne(workspacePath, relativePath, client, remoteEntry) {
|
|
516
|
+
let canonicalPath;
|
|
380
517
|
try {
|
|
518
|
+
canonicalPath = validatePrivateRelativePath(relativePath);
|
|
381
519
|
return await withRetry(async () => {
|
|
382
520
|
const url = (await client.syncPresign({ files: [{
|
|
383
|
-
path:
|
|
521
|
+
path: canonicalPath,
|
|
384
522
|
operation: "get"
|
|
385
523
|
}] })).urls[0]?.url;
|
|
386
524
|
if (!url) throw new Error("No presigned URL returned");
|
|
387
525
|
const response = await fetch(url);
|
|
388
526
|
if (!response.ok) throw new Error(`S3 GET failed (${String(response.status)}): ${await response.text()}`);
|
|
527
|
+
const contentLength = Number(response.headers.get("content-length") ?? "0");
|
|
528
|
+
if (Number.isFinite(contentLength) && contentLength > MAX_DOWNLOAD_BYTES) throw new Error("S3 object exceeds the local sync download limit");
|
|
389
529
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
390
|
-
|
|
530
|
+
if (buffer.length > MAX_DOWNLOAD_BYTES) throw new Error("S3 object exceeds the local sync download limit");
|
|
531
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
391
532
|
await mkdir(dirname(absolutePath), { recursive: true });
|
|
392
533
|
await writeFile(absolutePath, buffer);
|
|
393
534
|
const entry = {
|
|
394
535
|
hash: remoteEntry?.hash ?? "",
|
|
395
536
|
size: buffer.length,
|
|
396
537
|
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
397
|
-
storageClass: remoteEntry?.storageClass
|
|
538
|
+
storageClass: remoteEntry?.storageClass === "GLACIER_IR" ? "GLACIER_IR" : "STANDARD"
|
|
398
539
|
};
|
|
399
540
|
return {
|
|
400
|
-
path:
|
|
541
|
+
path: canonicalPath,
|
|
401
542
|
success: true,
|
|
402
543
|
size: buffer.length,
|
|
403
544
|
entry
|
|
@@ -413,6 +554,7 @@ async function downloadOne(workspacePath, relativePath, client, remoteEntry) {
|
|
|
413
554
|
}
|
|
414
555
|
async function downloadFiles(workspacePath, relativePaths, client, remoteManifest, options = {}) {
|
|
415
556
|
const { concurrency = 5, quiet = false } = options;
|
|
557
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > MAX_TRANSFER_CONCURRENCY) throw new Error("Download concurrency must be an integer between 1 and 100");
|
|
416
558
|
const results = [];
|
|
417
559
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
418
560
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
@@ -462,6 +604,15 @@ const log = createLogger("SyncEngine");
|
|
|
462
604
|
const MAX_PRESERVE_BYTES = 100 * 1024 * 1024;
|
|
463
605
|
/** Marker that keeps the heartbeat recovery nudge idempotent across runs. */
|
|
464
606
|
const RECOVERY_NUDGE_MARKER = "<!-- alfe-sync:recovery-nudge -->";
|
|
607
|
+
function filterSafePrivatePaths(paths) {
|
|
608
|
+
const safe = [];
|
|
609
|
+
for (const candidate of paths) try {
|
|
610
|
+
safe.push(validatePrivateRelativePath(candidate));
|
|
611
|
+
} catch {
|
|
612
|
+
log.warn("Ignored an invalid path from sync state");
|
|
613
|
+
}
|
|
614
|
+
return [...new Set(safe)];
|
|
615
|
+
}
|
|
465
616
|
/**
|
|
466
617
|
* Recovery artifacts: `.conflict-<ts>` sidecars and `RECOVERY-<ts>.md`
|
|
467
618
|
* reports. Anchored to the timestamp shape so legitimate user files like
|
|
@@ -499,7 +650,8 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
499
650
|
async push(paths, options = {}) {
|
|
500
651
|
const { quiet = false, filter } = options;
|
|
501
652
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
502
|
-
let filesToPush = paths && paths.length > 0 ? filterIgnored(paths, ignorePatterns) : await detectLocalChanges(workspacePath, ignorePatterns);
|
|
653
|
+
let filesToPush = paths && paths.length > 0 ? filterIgnored(filterSafePrivatePaths(paths), ignorePatterns) : await detectLocalChanges(workspacePath, ignorePatterns);
|
|
654
|
+
if (paths && paths.length > 0) filesToPush = await filterChangedExplicitPaths(workspacePath, filesToPush);
|
|
503
655
|
if (filter) filesToPush = filesToPush.filter((p) => p.startsWith(filter));
|
|
504
656
|
if (filesToPush.length === 0) {
|
|
505
657
|
if (!quiet) log.info("Nothing to push");
|
|
@@ -533,7 +685,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
533
685
|
if (!quiet) log.info(`Deleting ${String(paths.length)} file(s) from cloud`);
|
|
534
686
|
let deleted = 0;
|
|
535
687
|
let errors = 0;
|
|
536
|
-
for (const path of paths) try {
|
|
688
|
+
for (const path of filterSafePrivatePaths(paths)) try {
|
|
537
689
|
await client.syncDeleteFile(path);
|
|
538
690
|
await removeManifestEntry(workspacePath, path);
|
|
539
691
|
deleted++;
|
|
@@ -564,7 +716,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
564
716
|
};
|
|
565
717
|
}
|
|
566
718
|
if (!quiet) log.info(`Pulling ${String(diff.toPull.length)} file(s)`);
|
|
567
|
-
const results = await downloadFiles(workspacePath, [...diff.toPull], client, remoteManifest, { quiet });
|
|
719
|
+
const results = await downloadFiles(workspacePath, filterSafePrivatePaths([...diff.toPull]), client, remoteManifest, { quiet });
|
|
568
720
|
const pulled = results.filter((r) => r.success).length;
|
|
569
721
|
const errors = results.filter((r) => !r.success).length;
|
|
570
722
|
if (!quiet) log.info(`Pull complete: ${String(pulled)} downloaded, ${String(errors)} failed`);
|
|
@@ -581,8 +733,8 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
581
733
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
582
734
|
const localChanges = await detectLocalChanges(workspacePath, ignorePatterns);
|
|
583
735
|
const diff = diffManifests(localManifest, remoteManifest);
|
|
584
|
-
const trueConflicts = diff.conflicts.filter((p) => localChanges.includes(p));
|
|
585
|
-
const remoteOnlyChanges = diff.conflicts.filter((p) => !localChanges.includes(p));
|
|
736
|
+
const trueConflicts = filterSafePrivatePaths(diff.conflicts.filter((p) => localChanges.includes(p)));
|
|
737
|
+
const remoteOnlyChanges = filterSafePrivatePaths(diff.conflicts.filter((p) => !localChanges.includes(p)));
|
|
586
738
|
let conflictCount = 0;
|
|
587
739
|
const conflictTimestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
588
740
|
for (const conflictPath of trueConflicts) {
|
|
@@ -591,12 +743,12 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
591
743
|
if (outcome.status !== "too-large" && !quiet) log.warn(`Conflict: ${conflictPath} — saved as ${outcome.conflictPath}`);
|
|
592
744
|
conflictCount++;
|
|
593
745
|
}
|
|
594
|
-
const filesToPush = filterIgnored([...diff.toPush, ...localChanges.filter((p) => !diff.conflicts.includes(p))], ignorePatterns);
|
|
595
|
-
const filesToPull = [
|
|
746
|
+
const filesToPush = filterIgnored(filterSafePrivatePaths([...diff.toPush, ...localChanges.filter((p) => !diff.conflicts.includes(p))]), ignorePatterns);
|
|
747
|
+
const filesToPull = filterSafePrivatePaths([
|
|
596
748
|
...diff.toPull,
|
|
597
749
|
...remoteOnlyChanges,
|
|
598
750
|
...trueConflicts
|
|
599
|
-
];
|
|
751
|
+
]);
|
|
600
752
|
const pushResult = {
|
|
601
753
|
pushed: 0,
|
|
602
754
|
errors: 0
|
|
@@ -629,7 +781,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
629
781
|
async firstRunReconcile(options = {}) {
|
|
630
782
|
const { quiet = false } = options;
|
|
631
783
|
const remoteManifest = await client.syncGetManifest();
|
|
632
|
-
const remotePaths = Object.keys(remoteManifest.files);
|
|
784
|
+
const remotePaths = filterSafePrivatePaths(Object.keys(remoteManifest.files));
|
|
633
785
|
if (remotePaths.length === 0) {
|
|
634
786
|
if (!quiet) log.info("First-run sync: no cloud state — seeding workspace to cloud");
|
|
635
787
|
const seedResult = await this.push(void 0, options);
|
|
@@ -680,7 +832,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
680
832
|
let recoveryReportPath = null;
|
|
681
833
|
if (preserved.length > 0 && (newlyPreserved > 0 || !await recoveryReportExists(workspacePath))) {
|
|
682
834
|
recoveryReportPath = `${agentDirPrefix(workspacePath)}RECOVERY-${timestamp}.md`;
|
|
683
|
-
const reportAbsolute =
|
|
835
|
+
const reportAbsolute = await assertNoSymlinkTraversal(workspacePath, recoveryReportPath);
|
|
684
836
|
await mkdir(dirname(reportAbsolute), { recursive: true });
|
|
685
837
|
await writeFile(reportAbsolute, buildRecoveryReport(preserved, workspacePath), "utf-8");
|
|
686
838
|
log.warn(`Recovery: ${String(preserved.length)} diverged local file(s) — see ${recoveryReportPath}`);
|
|
@@ -742,7 +894,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
742
894
|
};
|
|
743
895
|
const remoteManifest = await client.syncGetManifest();
|
|
744
896
|
const localManifest = await readManifest(workspacePath);
|
|
745
|
-
const filesToPull = paths.filter((p) => {
|
|
897
|
+
const filesToPull = filterSafePrivatePaths(paths).filter((p) => {
|
|
746
898
|
if (!(p in remoteManifest.files)) return false;
|
|
747
899
|
if (!(p in localManifest.files)) return true;
|
|
748
900
|
return localManifest.files[p].hash !== remoteManifest.files[p].hash;
|
|
@@ -772,7 +924,7 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
772
924
|
const { quiet = false, limit = 1e3 } = options;
|
|
773
925
|
const ignorePatterns = await loadIgnorePatterns(workspacePath, runtime);
|
|
774
926
|
const remoteManifest = await client.syncGetManifest();
|
|
775
|
-
const ignoredPaths = Object.keys(remoteManifest.files).filter((p) => shouldIgnore(p, ignorePatterns));
|
|
927
|
+
const ignoredPaths = filterSafePrivatePaths(Object.keys(remoteManifest.files)).filter((p) => shouldIgnore(p, ignorePatterns));
|
|
776
928
|
if (ignoredPaths.length === 0) return {
|
|
777
929
|
pushed: 0,
|
|
778
930
|
pulled: 0,
|
|
@@ -788,20 +940,38 @@ function createSyncEngine({ workspacePath, client, runtime = "openclaw", maxPres
|
|
|
788
940
|
},
|
|
789
941
|
async removeLocalFile(filePath, options = {}) {
|
|
790
942
|
const { quiet = false } = options;
|
|
791
|
-
const
|
|
943
|
+
const canonicalPath = validatePrivateRelativePath(filePath);
|
|
944
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
792
945
|
try {
|
|
793
946
|
const { unlink } = await import("node:fs/promises");
|
|
794
947
|
if (existsSync(absolutePath)) {
|
|
795
948
|
await unlink(absolutePath);
|
|
796
|
-
if (!quiet) log.info(`Deleted: ${
|
|
949
|
+
if (!quiet) log.info(`Deleted: ${canonicalPath}`);
|
|
797
950
|
}
|
|
798
951
|
} catch (err) {
|
|
799
|
-
if (!quiet) log.error({ err }, `Failed to delete ${
|
|
952
|
+
if (!quiet) log.error({ err }, `Failed to delete ${canonicalPath}`);
|
|
800
953
|
}
|
|
801
|
-
await removeManifestEntry(workspacePath,
|
|
954
|
+
await removeManifestEntry(workspacePath, canonicalPath);
|
|
802
955
|
}
|
|
803
956
|
};
|
|
804
957
|
}
|
|
958
|
+
async function filterChangedExplicitPaths(workspacePath, paths) {
|
|
959
|
+
const manifest = await readManifest(workspacePath);
|
|
960
|
+
const changed = [];
|
|
961
|
+
for (const relativePath of paths) {
|
|
962
|
+
if (!(relativePath in manifest.files)) {
|
|
963
|
+
changed.push(relativePath);
|
|
964
|
+
continue;
|
|
965
|
+
}
|
|
966
|
+
const entry = manifest.files[relativePath];
|
|
967
|
+
try {
|
|
968
|
+
if (await computeFileHash(await assertNoSymlinkTraversal(workspacePath, relativePath)) !== entry.hash) changed.push(relativePath);
|
|
969
|
+
} catch {
|
|
970
|
+
changed.push(relativePath);
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
return changed;
|
|
974
|
+
}
|
|
805
975
|
/**
|
|
806
976
|
* Walk the workspace and return paths whose hash differs from the manifest
|
|
807
977
|
* (or that are missing from it entirely).
|
|
@@ -857,10 +1027,11 @@ async function recoveryReportExists(workspacePath) {
|
|
|
857
1027
|
* `sourceHash` when the caller already hashed the file.
|
|
858
1028
|
*/
|
|
859
1029
|
async function preserveConflictCopy(workspacePath, relativePath, timestamp, options = {}) {
|
|
860
|
-
const
|
|
861
|
-
const
|
|
862
|
-
const
|
|
863
|
-
const
|
|
1030
|
+
const canonicalPath = validatePrivateRelativePath(relativePath);
|
|
1031
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, canonicalPath);
|
|
1032
|
+
const ext = extname(canonicalPath);
|
|
1033
|
+
const base = basename(canonicalPath, ext);
|
|
1034
|
+
const dir = dirname(canonicalPath);
|
|
864
1035
|
try {
|
|
865
1036
|
const fileStat = await stat(absolutePath);
|
|
866
1037
|
if (options.maxBytes !== void 0 && fileStat.size > options.maxBytes) return {
|
|
@@ -872,16 +1043,17 @@ async function preserveConflictCopy(workspacePath, relativePath, timestamp, opti
|
|
|
872
1043
|
for (const name of siblings) {
|
|
873
1044
|
if (!name.startsWith(`${base}.conflict-`)) continue;
|
|
874
1045
|
if (ext !== "" && !name.endsWith(ext)) continue;
|
|
875
|
-
if (await
|
|
1046
|
+
if (await assertNoSymlinkTraversal(workspacePath, dir === "." ? name : `${dir}/${name}`).then((path) => computeFileHash(path)).catch(() => null) === sourceHash) return {
|
|
876
1047
|
status: "already-preserved",
|
|
877
1048
|
conflictPath: join(dir, name).replace(/\\/g, "/")
|
|
878
1049
|
};
|
|
879
1050
|
}
|
|
880
1051
|
const conflictName = `${base}.conflict-${timestamp}${ext}`;
|
|
881
|
-
|
|
1052
|
+
const conflictPath = dir === "." ? conflictName : `${dir}/${conflictName}`;
|
|
1053
|
+
await copyFile(absolutePath, await assertNoSymlinkTraversal(workspacePath, conflictPath));
|
|
882
1054
|
return {
|
|
883
1055
|
status: "preserved",
|
|
884
|
-
conflictPath
|
|
1056
|
+
conflictPath
|
|
885
1057
|
};
|
|
886
1058
|
} catch {
|
|
887
1059
|
return { status: "vanished" };
|
|
@@ -908,7 +1080,13 @@ async function classifyRestorePaths(workspacePath, restorePaths, remoteFiles, lo
|
|
|
908
1080
|
async function classifyOne(path) {
|
|
909
1081
|
const remote = remoteFiles[path];
|
|
910
1082
|
const entry = manifestTrusted ? localManifest.files[path] : void 0;
|
|
911
|
-
|
|
1083
|
+
let absolutePath;
|
|
1084
|
+
try {
|
|
1085
|
+
absolutePath = await assertNoSymlinkTraversal(workspacePath, path);
|
|
1086
|
+
} catch {
|
|
1087
|
+
log.warn("Skipped a cloud sync path that crosses a local symbolic link");
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
912
1090
|
let fileStat = null;
|
|
913
1091
|
try {
|
|
914
1092
|
fileStat = await stat(absolutePath);
|
|
@@ -937,7 +1115,7 @@ async function classifyRestorePaths(workspacePath, restorePaths, remoteFiles, lo
|
|
|
937
1115
|
hash: remote.hash,
|
|
938
1116
|
size: fileStat.size,
|
|
939
1117
|
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
940
|
-
storageClass: remote.storageClass
|
|
1118
|
+
storageClass: remote.storageClass === "GLACIER_IR" ? "GLACIER_IR" : "STANDARD"
|
|
941
1119
|
};
|
|
942
1120
|
return;
|
|
943
1121
|
}
|
|
@@ -997,7 +1175,7 @@ function buildRecoveryReport(records, workspacePath) {
|
|
|
997
1175
|
async function appendRecoveryNudge(workspacePath, recoveryReportPath, preservedCount) {
|
|
998
1176
|
const prefix = agentDirPrefix(workspacePath);
|
|
999
1177
|
const heartbeatPath = `${prefix}HEARTBEAT.md`;
|
|
1000
|
-
const absolutePath =
|
|
1178
|
+
const absolutePath = await assertNoSymlinkTraversal(workspacePath, heartbeatPath);
|
|
1001
1179
|
if ((existsSync(absolutePath) ? await readFile(absolutePath, "utf-8") : "").includes(RECOVERY_NUDGE_MARKER)) return null;
|
|
1002
1180
|
const reportForAgent = recoveryReportPath.startsWith(prefix) ? recoveryReportPath.slice(prefix.length) : recoveryReportPath;
|
|
1003
1181
|
const block = [
|
|
@@ -1017,6 +1195,6 @@ async function appendRecoveryNudge(workspacePath, recoveryReportPath, preservedC
|
|
|
1017
1195
|
return heartbeatPath;
|
|
1018
1196
|
}
|
|
1019
1197
|
//#endregion
|
|
1020
|
-
export { withRetry as a, loadIgnorePatterns as c, computeFileHash as d, diffManifests as f, writeManifest as g, updateManifestEntry as h, uploadFiles as i, shouldIgnore as l, removeManifestEntry as m, isRecoveryArtifact as n, DEFAULT_IGNORES as o, readManifest as p, downloadFiles as r, filterIgnored as s, createSyncEngine as t, shouldIgnoreDir as u };
|
|
1198
|
+
export { assertNoSymlinkTraversal as _, withRetry as a, loadIgnorePatterns as c, computeFileHash as d, diffManifests as f, writeManifest as g, updateManifestEntry as h, uploadFiles as i, shouldIgnore as l, removeManifestEntry as m, isRecoveryArtifact as n, DEFAULT_IGNORES as o, readManifest as p, downloadFiles as r, filterIgnored as s, createSyncEngine as t, shouldIgnoreDir as u, resolvePrivateWorkspacePath as v, validatePrivateRelativePath as y };
|
|
1021
1199
|
|
|
1022
1200
|
//# sourceMappingURL=sync-engine.js.map
|