@alfe.ai/openclaw-sync 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.cjs +45 -63
- package/dist/cli/index.js +44 -62
- package/dist/cli/index.js.map +1 -1
- package/dist/index.cjs +4 -10
- package/dist/index.d.cts +785 -204
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +785 -204
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -3
- package/dist/plugin.d.cts +16 -18
- package/dist/plugin.d.cts.map +1 -1
- package/dist/plugin.d.ts +16 -18
- package/dist/plugin.d.ts.map +1 -1
- package/dist/plugin2.cjs +118 -109
- package/dist/plugin2.js +117 -108
- package/dist/plugin2.js.map +1 -1
- package/dist/sync-engine.cjs +248 -372
- package/dist/sync-engine.js +207 -348
- package/dist/sync-engine.js.map +1 -1
- package/package.json +2 -1
- package/dist/ignore.cjs +0 -120
- package/dist/ignore.js +0 -74
- package/dist/ignore.js.map +0 -1
package/dist/sync-engine.js
CHANGED
|
@@ -1,112 +1,10 @@
|
|
|
1
|
-
import { r as loadIgnorePatterns, t as filterIgnored } from "./ignore.js";
|
|
2
|
-
import { basename, dirname, extname, join } from "node:path";
|
|
3
|
-
import { homedir } from "node:os";
|
|
4
|
-
import { createReadStream, existsSync } from "node:fs";
|
|
5
|
-
import { configExists, resolveConfig } from "@alfe.ai/config";
|
|
6
1
|
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
2
|
+
import { createReadStream, existsSync } from "node:fs";
|
|
7
3
|
import { createHash } from "node:crypto";
|
|
4
|
+
import { basename, dirname, extname, join } from "node:path";
|
|
5
|
+
import { homedir } from "node:os";
|
|
6
|
+
import micromatch from "micromatch";
|
|
8
7
|
import { createLogger } from "@auriclabs/logger";
|
|
9
|
-
//#region src/config.ts
|
|
10
|
-
/**
|
|
11
|
-
* AlfeSync configuration.
|
|
12
|
-
*
|
|
13
|
-
* Derives credentials from `@alfe.ai/config` (~/.alfe/config.toml) so the
|
|
14
|
-
* sync plugin works on any agent that's already logged in — no separate
|
|
15
|
-
* `.alfesync/config.json` or manual `alfesync init` required.
|
|
16
|
-
*
|
|
17
|
-
* The only field not directly in @alfe.ai/config is `agentId`, which is
|
|
18
|
-
* resolved once at startup by calling POST /auth/validate (same pattern
|
|
19
|
-
* the gateway daemon uses to bootstrap its own identity).
|
|
20
|
-
*/
|
|
21
|
-
const SYNC_STATE_DIR = join(join(homedir(), ".alfe"), "sync");
|
|
22
|
-
/**
|
|
23
|
-
* In-memory cache — agentId is stable for process lifetime.
|
|
24
|
-
*
|
|
25
|
-
* We cache the *Promise* (not just the resolved value) so that concurrent
|
|
26
|
-
* callers during plugin activation share a single `/auth/validate` round-trip
|
|
27
|
-
* instead of each firing their own.
|
|
28
|
-
*/
|
|
29
|
-
let cachedSyncConfigPromise = null;
|
|
30
|
-
/**
|
|
31
|
-
* Local on-disk state directory (manifest, etc.) — separate from credentials.
|
|
32
|
-
* Lives under ~/.alfe/sync/ so sync state stays alongside the rest of Alfe's
|
|
33
|
-
* agent state, not scattered across the workspace.
|
|
34
|
-
*/
|
|
35
|
-
function syncStateDir() {
|
|
36
|
-
return SYNC_STATE_DIR;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* True if the agent has logged in (~/.alfe/config.toml exists).
|
|
40
|
-
* Sync needs no separate initialization — login is enough.
|
|
41
|
-
*/
|
|
42
|
-
function isInitialized() {
|
|
43
|
-
return configExists();
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Invalidate the in-memory config cache. Call after login changes.
|
|
47
|
-
*/
|
|
48
|
-
function invalidateSyncConfigCache() {
|
|
49
|
-
cachedSyncConfigPromise = null;
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Resolve the full sync config — apiKey + apiUrl from @alfe.ai/config,
|
|
53
|
-
* agentId/orgId fetched once from /auth/validate.
|
|
54
|
-
*
|
|
55
|
-
* Returns null if the agent isn't logged in or validation fails.
|
|
56
|
-
* Callers that require config should use requireConfig().
|
|
57
|
-
*
|
|
58
|
-
* Concurrent callers share a single in-flight promise; failures are
|
|
59
|
-
* not cached so a transient API blip doesn't poison the process.
|
|
60
|
-
*/
|
|
61
|
-
async function resolveSyncConfig() {
|
|
62
|
-
if (cachedSyncConfigPromise) return cachedSyncConfigPromise;
|
|
63
|
-
const promise = doResolveSyncConfig();
|
|
64
|
-
cachedSyncConfigPromise = promise;
|
|
65
|
-
const result = await promise;
|
|
66
|
-
if (result === null) cachedSyncConfigPromise = null;
|
|
67
|
-
return result;
|
|
68
|
-
}
|
|
69
|
-
async function doResolveSyncConfig() {
|
|
70
|
-
if (!configExists()) return null;
|
|
71
|
-
let alfe;
|
|
72
|
-
try {
|
|
73
|
-
alfe = resolveConfig();
|
|
74
|
-
} catch {
|
|
75
|
-
return null;
|
|
76
|
-
}
|
|
77
|
-
const validated = await validateToken(alfe.apiUrl, alfe.apiKey);
|
|
78
|
-
if (!validated?.agentId || !validated.tenantId) return null;
|
|
79
|
-
return {
|
|
80
|
-
agentId: validated.agentId,
|
|
81
|
-
orgId: validated.tenantId,
|
|
82
|
-
token: alfe.apiKey,
|
|
83
|
-
workspacePath: alfe.workspacePath,
|
|
84
|
-
apiUrl: alfe.apiUrl
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
/**
|
|
88
|
-
* Resolve config or throw with a clear message. Use in CLI entry points.
|
|
89
|
-
*/
|
|
90
|
-
async function requireConfig() {
|
|
91
|
-
if (!configExists()) throw new Error("Alfe not configured — run `alfe login` first.");
|
|
92
|
-
const config = await resolveSyncConfig();
|
|
93
|
-
if (!config) throw new Error("Failed to resolve sync config — token may be invalid or the API is unreachable.");
|
|
94
|
-
return config;
|
|
95
|
-
}
|
|
96
|
-
async function validateToken(apiUrl, token) {
|
|
97
|
-
try {
|
|
98
|
-
const res = await fetch(`${apiUrl}/auth/validate`, {
|
|
99
|
-
method: "POST",
|
|
100
|
-
headers: { "Content-Type": "application/json" },
|
|
101
|
-
body: JSON.stringify({ token })
|
|
102
|
-
});
|
|
103
|
-
if (!res.ok) return null;
|
|
104
|
-
return await res.json();
|
|
105
|
-
} catch {
|
|
106
|
-
return null;
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
//#endregion
|
|
110
8
|
//#region src/manifest.ts
|
|
111
9
|
/**
|
|
112
10
|
* AlfeSync manifest — local file manifest at `~/.alfe/sync/manifest.json`.
|
|
@@ -119,13 +17,19 @@ async function validateToken(apiUrl, token) {
|
|
|
119
17
|
* rest of the package, but the manifest itself is workspace-independent
|
|
120
18
|
* (one agent, one workspace, one manifest).
|
|
121
19
|
*/
|
|
20
|
+
/**
|
|
21
|
+
* Local state directory for sync — manifest, etc. Lives under `~/.alfe/sync/`
|
|
22
|
+
* so it stays alongside the rest of Alfe's agent state instead of polluting
|
|
23
|
+
* the workspace.
|
|
24
|
+
*/
|
|
25
|
+
const SYNC_STATE_DIR = join(homedir(), ".alfe", "sync");
|
|
122
26
|
const MANIFEST_FILE = "manifest.json";
|
|
123
27
|
/**
|
|
124
28
|
* Resolve the manifest file path. Lives under `~/.alfe/sync/`, independent
|
|
125
29
|
* of the workspace path — one agent has one manifest.
|
|
126
30
|
*/
|
|
127
31
|
function manifestPath() {
|
|
128
|
-
return join(
|
|
32
|
+
return join(SYNC_STATE_DIR, MANIFEST_FILE);
|
|
129
33
|
}
|
|
130
34
|
/**
|
|
131
35
|
* Read the local manifest. Returns empty manifest if not found.
|
|
@@ -150,7 +54,7 @@ async function readManifest(workspacePath) {
|
|
|
150
54
|
*/
|
|
151
55
|
async function writeManifest(workspacePath, manifest) {
|
|
152
56
|
const path = manifestPath();
|
|
153
|
-
await mkdir(
|
|
57
|
+
await mkdir(SYNC_STATE_DIR, { recursive: true });
|
|
154
58
|
await writeFile(path, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
|
|
155
59
|
}
|
|
156
60
|
/**
|
|
@@ -216,118 +120,100 @@ function diffManifests(local, remote) {
|
|
|
216
120
|
};
|
|
217
121
|
}
|
|
218
122
|
//#endregion
|
|
219
|
-
//#region src/
|
|
123
|
+
//#region src/ignore.ts
|
|
220
124
|
/**
|
|
221
|
-
*
|
|
125
|
+
* AlfeSync ignore — parse `.alfesyncignore` (gitignore-style glob matching).
|
|
126
|
+
*
|
|
127
|
+
* Uses micromatch for glob pattern matching, compatible with .gitignore syntax.
|
|
222
128
|
*/
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
throw new Error(`AlfeSync API error (${String(response.status)}): ${errorMsg}`);
|
|
129
|
+
/** Default ignore patterns — always applied */
|
|
130
|
+
const DEFAULT_IGNORES = [
|
|
131
|
+
".alfesync/**",
|
|
132
|
+
"node_modules/**",
|
|
133
|
+
"*.tmp",
|
|
134
|
+
".DS_Store",
|
|
135
|
+
".git/**",
|
|
136
|
+
".sst/**",
|
|
137
|
+
".build/**",
|
|
138
|
+
"dist/**"
|
|
139
|
+
];
|
|
140
|
+
/**
|
|
141
|
+
* Load ignore patterns from `.alfesyncignore` file + defaults.
|
|
142
|
+
*/
|
|
143
|
+
async function loadIgnorePatterns(workspacePath) {
|
|
144
|
+
const ignoreFile = join(workspacePath, ".alfesyncignore");
|
|
145
|
+
const patterns = [...DEFAULT_IGNORES];
|
|
146
|
+
if (existsSync(ignoreFile)) try {
|
|
147
|
+
const lines = (await readFile(ignoreFile, "utf-8")).split("\n");
|
|
148
|
+
for (const line of lines) {
|
|
149
|
+
const trimmed = line.trim();
|
|
150
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
151
|
+
patterns.push(trimmed);
|
|
247
152
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
return request("GET", `/sync/agents/${agentId}/stats`);
|
|
291
|
-
},
|
|
292
|
-
async registerAgent(displayName) {
|
|
293
|
-
return request("POST", "/sync/agents", {
|
|
294
|
-
agentId,
|
|
295
|
-
displayName
|
|
296
|
-
});
|
|
297
|
-
},
|
|
298
|
-
async getFileHistory(filePath) {
|
|
299
|
-
return (await request("GET", `/sync/agents/${agentId}/files/${filePath}/versions`)).versions;
|
|
300
|
-
},
|
|
301
|
-
async reconstruct(mode = "full") {
|
|
302
|
-
return request("POST", `/sync/agents/${agentId}/reconstruct`, { mode });
|
|
153
|
+
} catch {}
|
|
154
|
+
return patterns;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Check if a relative path should be ignored.
|
|
158
|
+
*/
|
|
159
|
+
function shouldIgnore(relativePath, patterns) {
|
|
160
|
+
return micromatch.isMatch(relativePath, patterns, {
|
|
161
|
+
dot: true,
|
|
162
|
+
matchBase: true
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Filter a list of relative paths, removing ignored ones.
|
|
167
|
+
*/
|
|
168
|
+
function filterIgnored(paths, patterns) {
|
|
169
|
+
return paths.filter((p) => !shouldIgnore(p, patterns));
|
|
170
|
+
}
|
|
171
|
+
//#endregion
|
|
172
|
+
//#region src/retry.ts
|
|
173
|
+
/**
|
|
174
|
+
* Tiny retry helper used by uploader/downloader. Extracted so both
|
|
175
|
+
* use the same timing and surface the same final error shape.
|
|
176
|
+
*/
|
|
177
|
+
const DEFAULT_MAX_RETRIES = 3;
|
|
178
|
+
const DEFAULT_BASE_DELAY_MS = 1e3;
|
|
179
|
+
/**
|
|
180
|
+
* Run `fn` up to `maxRetries + 1` times with exponential backoff
|
|
181
|
+
* (`base * 2^attempt`). Returns the first successful value, or rethrows
|
|
182
|
+
* the last error.
|
|
183
|
+
*/
|
|
184
|
+
async function withRetry(fn, options = {}) {
|
|
185
|
+
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
186
|
+
const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;
|
|
187
|
+
let lastError;
|
|
188
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) try {
|
|
189
|
+
return await fn();
|
|
190
|
+
} catch (err) {
|
|
191
|
+
lastError = err;
|
|
192
|
+
if (attempt < maxRetries) {
|
|
193
|
+
const delay = baseDelayMs * Math.pow(2, attempt);
|
|
194
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
303
195
|
}
|
|
304
|
-
}
|
|
196
|
+
}
|
|
197
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
305
198
|
}
|
|
306
199
|
//#endregion
|
|
307
200
|
//#region src/uploader.ts
|
|
308
201
|
/**
|
|
309
|
-
* AlfeSync uploader —
|
|
202
|
+
* AlfeSync uploader — push changed files to S3.
|
|
310
203
|
*
|
|
311
|
-
*
|
|
312
|
-
* 1.
|
|
313
|
-
* 2. PUT
|
|
314
|
-
* 3.
|
|
315
|
-
* 4. Update local manifest
|
|
204
|
+
* Per file:
|
|
205
|
+
* 1. Ask the agent API for a presigned PUT URL
|
|
206
|
+
* 2. PUT bytes directly to S3 (raw fetch — S3 isn't an Alfe API)
|
|
207
|
+
* 3. Tell the agent API the upload completed (`syncConfirmUpload`)
|
|
208
|
+
* 4. Update the local manifest
|
|
316
209
|
*
|
|
317
|
-
*
|
|
210
|
+
* Each step retries with exponential backoff via `withRetry`.
|
|
318
211
|
*/
|
|
319
212
|
const log$2 = createLogger("SyncUploader");
|
|
320
|
-
const
|
|
321
|
-
const BASE_DELAY_MS$1 = 1e3;
|
|
322
|
-
/**
|
|
323
|
-
* Determine the storage class based on file path.
|
|
324
|
-
*/
|
|
213
|
+
const ARCHIVE_PREFIXES = ["sessions/archive/", "context/archive/"];
|
|
325
214
|
function getStorageClass(relativePath) {
|
|
326
|
-
return
|
|
215
|
+
return ARCHIVE_PREFIXES.some((p) => relativePath.startsWith(p)) ? "GLACIER_IR" : "STANDARD";
|
|
327
216
|
}
|
|
328
|
-
/**
|
|
329
|
-
* Determine MIME type from file path.
|
|
330
|
-
*/
|
|
331
217
|
function getContentType(relativePath) {
|
|
332
218
|
if (relativePath.endsWith(".json")) return "application/json";
|
|
333
219
|
if (relativePath.endsWith(".md")) return "text/markdown";
|
|
@@ -336,62 +222,63 @@ function getContentType(relativePath) {
|
|
|
336
222
|
if (relativePath.endsWith(".yaml") || relativePath.endsWith(".yml")) return "text/yaml";
|
|
337
223
|
return "application/octet-stream";
|
|
338
224
|
}
|
|
339
|
-
|
|
340
|
-
* Upload a single file with retry logic.
|
|
341
|
-
*/
|
|
342
|
-
async function uploadFileWithRetry(workspacePath, relativePath, client) {
|
|
225
|
+
async function uploadOne(workspacePath, relativePath, client) {
|
|
343
226
|
const absolutePath = join(workspacePath, relativePath);
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
227
|
+
try {
|
|
228
|
+
return await withRetry(async () => {
|
|
229
|
+
const [hash, fileStat] = await Promise.all([computeFileHash(absolutePath), stat(absolutePath)]);
|
|
230
|
+
const size = fileStat.size;
|
|
231
|
+
const storageClass = getStorageClass(relativePath);
|
|
232
|
+
const contentType = getContentType(relativePath);
|
|
233
|
+
const url = (await client.syncPresign({ files: [{
|
|
234
|
+
path: relativePath,
|
|
235
|
+
operation: "put",
|
|
236
|
+
contentType
|
|
237
|
+
}] })).urls[0]?.url;
|
|
238
|
+
if (!url) throw new Error("No presigned URL returned");
|
|
239
|
+
const fileContent = await readFile(absolutePath);
|
|
240
|
+
const putResponse = await fetch(url, {
|
|
241
|
+
method: "PUT",
|
|
242
|
+
headers: { "Content-Type": contentType },
|
|
243
|
+
body: fileContent
|
|
244
|
+
});
|
|
245
|
+
if (!putResponse.ok) throw new Error(`S3 PUT failed (${String(putResponse.status)}): ${await putResponse.text()}`);
|
|
246
|
+
await client.syncConfirmUpload({
|
|
247
|
+
filePath: relativePath,
|
|
248
|
+
hash,
|
|
249
|
+
size,
|
|
250
|
+
storageClass
|
|
251
|
+
});
|
|
252
|
+
await updateManifestEntry(workspacePath, relativePath, {
|
|
253
|
+
hash,
|
|
254
|
+
size,
|
|
255
|
+
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
256
|
+
storageClass
|
|
257
|
+
});
|
|
258
|
+
return {
|
|
259
|
+
path: relativePath,
|
|
260
|
+
success: true,
|
|
261
|
+
hash,
|
|
262
|
+
size
|
|
263
|
+
};
|
|
364
264
|
});
|
|
265
|
+
} catch (err) {
|
|
365
266
|
return {
|
|
366
267
|
path: relativePath,
|
|
367
|
-
success:
|
|
368
|
-
|
|
369
|
-
size
|
|
268
|
+
success: false,
|
|
269
|
+
error: err instanceof Error ? err.message : String(err)
|
|
370
270
|
};
|
|
371
|
-
} catch (err) {
|
|
372
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
373
|
-
if (attempt < MAX_RETRIES$1) {
|
|
374
|
-
const delay = BASE_DELAY_MS$1 * Math.pow(2, attempt);
|
|
375
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
376
|
-
}
|
|
377
271
|
}
|
|
378
|
-
return {
|
|
379
|
-
path: relativePath,
|
|
380
|
-
success: false,
|
|
381
|
-
error: lastError?.message ?? "Unknown error"
|
|
382
|
-
};
|
|
383
272
|
}
|
|
384
273
|
/**
|
|
385
|
-
* Upload
|
|
386
|
-
*
|
|
387
|
-
* Uploads are performed in parallel with a concurrency limit.
|
|
274
|
+
* Upload many files, batched by `concurrency`.
|
|
388
275
|
*/
|
|
389
276
|
async function uploadFiles(workspacePath, relativePaths, client, options = {}) {
|
|
390
277
|
const { concurrency = 5, quiet = false } = options;
|
|
391
278
|
const results = [];
|
|
392
279
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
393
280
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
394
|
-
const batchResults = await Promise.all(batch.map((path) =>
|
|
281
|
+
const batchResults = await Promise.all(batch.map((path) => uploadOne(workspacePath, path, client)));
|
|
395
282
|
for (const result of batchResults) {
|
|
396
283
|
results.push(result);
|
|
397
284
|
if (!quiet) if (result.success) log$2.info(`Uploaded ${result.path} (${formatBytes$1(result.size ?? 0)})`);
|
|
@@ -408,65 +295,55 @@ function formatBytes$1(bytes) {
|
|
|
408
295
|
//#endregion
|
|
409
296
|
//#region src/downloader.ts
|
|
410
297
|
/**
|
|
411
|
-
* AlfeSync downloader —
|
|
298
|
+
* AlfeSync downloader — pull files from S3 to disk.
|
|
412
299
|
*
|
|
413
|
-
*
|
|
414
|
-
* 1.
|
|
415
|
-
* 2. GET
|
|
416
|
-
* 3. Write to
|
|
417
|
-
* 4. Update local manifest
|
|
300
|
+
* Per file:
|
|
301
|
+
* 1. Ask the agent API for a presigned GET URL
|
|
302
|
+
* 2. GET bytes directly from S3 (raw fetch — S3 isn't an Alfe API)
|
|
303
|
+
* 3. Write to disk
|
|
304
|
+
* 4. Update the local manifest
|
|
418
305
|
*/
|
|
419
306
|
const log$1 = createLogger("SyncDownloader");
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
307
|
+
async function downloadOne(workspacePath, relativePath, client, remoteEntry) {
|
|
308
|
+
try {
|
|
309
|
+
return await withRetry(async () => {
|
|
310
|
+
const url = (await client.syncPresign({ files: [{
|
|
311
|
+
path: relativePath,
|
|
312
|
+
operation: "get"
|
|
313
|
+
}] })).urls[0]?.url;
|
|
314
|
+
if (!url) throw new Error("No presigned URL returned");
|
|
315
|
+
const response = await fetch(url);
|
|
316
|
+
if (!response.ok) throw new Error(`S3 GET failed (${String(response.status)}): ${await response.text()}`);
|
|
317
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
318
|
+
const absolutePath = join(workspacePath, relativePath);
|
|
319
|
+
await mkdir(dirname(absolutePath), { recursive: true });
|
|
320
|
+
await writeFile(absolutePath, buffer);
|
|
321
|
+
await updateManifestEntry(workspacePath, relativePath, {
|
|
322
|
+
hash: remoteEntry?.hash ?? "",
|
|
323
|
+
size: buffer.length,
|
|
324
|
+
lastSynced: (/* @__PURE__ */ new Date()).toISOString(),
|
|
325
|
+
storageClass: remoteEntry?.storageClass ?? "STANDARD"
|
|
326
|
+
});
|
|
327
|
+
return {
|
|
328
|
+
path: relativePath,
|
|
329
|
+
success: true,
|
|
330
|
+
size: buffer.length
|
|
331
|
+
};
|
|
440
332
|
});
|
|
333
|
+
} catch (err) {
|
|
441
334
|
return {
|
|
442
335
|
path: relativePath,
|
|
443
|
-
success:
|
|
444
|
-
|
|
336
|
+
success: false,
|
|
337
|
+
error: err instanceof Error ? err.message : String(err)
|
|
445
338
|
};
|
|
446
|
-
} catch (err) {
|
|
447
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
448
|
-
if (attempt < MAX_RETRIES) {
|
|
449
|
-
const delay = BASE_DELAY_MS * Math.pow(2, attempt);
|
|
450
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
451
|
-
}
|
|
452
339
|
}
|
|
453
|
-
return {
|
|
454
|
-
path: relativePath,
|
|
455
|
-
success: false,
|
|
456
|
-
error: lastError?.message ?? "Unknown error"
|
|
457
|
-
};
|
|
458
340
|
}
|
|
459
|
-
/**
|
|
460
|
-
* Download multiple files from S3.
|
|
461
|
-
*
|
|
462
|
-
* Downloads are performed in parallel with a concurrency limit.
|
|
463
|
-
*/
|
|
464
341
|
async function downloadFiles(workspacePath, relativePaths, client, remoteManifest, options = {}) {
|
|
465
342
|
const { concurrency = 5, quiet = false } = options;
|
|
466
343
|
const results = [];
|
|
467
344
|
for (let i = 0; i < relativePaths.length; i += concurrency) {
|
|
468
345
|
const batch = relativePaths.slice(i, i + concurrency);
|
|
469
|
-
const batchResults = await Promise.all(batch.map((path) =>
|
|
346
|
+
const batchResults = await Promise.all(batch.map((path) => downloadOne(workspacePath, path, client, remoteManifest?.files[path])));
|
|
470
347
|
for (const result of batchResults) {
|
|
471
348
|
results.push(result);
|
|
472
349
|
if (!quiet) if (result.success) log$1.info(`Downloaded ${result.path} (${formatBytes(result.size ?? 0)})`);
|
|
@@ -483,38 +360,31 @@ function formatBytes(bytes) {
|
|
|
483
360
|
//#endregion
|
|
484
361
|
//#region src/sync-engine.ts
|
|
485
362
|
/**
|
|
486
|
-
* AlfeSync engine — orchestrates push, pull, and full sync
|
|
363
|
+
* AlfeSync engine — orchestrates push, pull, and full sync.
|
|
487
364
|
*
|
|
488
|
-
* Handles:
|
|
489
365
|
* - push(paths[]): upload changed files to S3
|
|
490
366
|
* - pull(): download files newer on remote
|
|
491
367
|
* - fullSync(): bidirectional sync with conflict detection
|
|
492
368
|
*
|
|
493
|
-
* Conflict resolution:
|
|
494
|
-
* AND local file has changed
|
|
369
|
+
* Conflict resolution: when a remote file is newer than the local manifest
|
|
370
|
+
* entry AND the local file has also changed, the local copy is renamed to
|
|
371
|
+
* `<name>.conflict-<timestamp>.<ext>` and the remote version wins.
|
|
495
372
|
*/
|
|
496
373
|
const log = createLogger("SyncEngine");
|
|
497
374
|
/**
|
|
498
|
-
*
|
|
499
|
-
*
|
|
375
|
+
* Construct a sync engine bound to a workspace + agent API client.
|
|
376
|
+
*
|
|
377
|
+
* The client is constructed once in plugin.ts (or the CLI) and passed in,
|
|
378
|
+
* so credentials never leak into multiple places.
|
|
500
379
|
*/
|
|
501
|
-
|
|
502
|
-
const config = await requireConfig();
|
|
503
|
-
const workspacePath = workspacePathOverride ?? config.workspacePath;
|
|
504
|
-
const client = createApiClient({
|
|
505
|
-
apiUrl: config.apiUrl,
|
|
506
|
-
token: config.token,
|
|
507
|
-
agentId: config.agentId
|
|
508
|
-
});
|
|
380
|
+
function createSyncEngine({ workspacePath, client }) {
|
|
509
381
|
return {
|
|
510
|
-
|
|
382
|
+
workspacePath,
|
|
511
383
|
client,
|
|
512
384
|
async push(paths, options = {}) {
|
|
513
385
|
const { quiet = false, filter } = options;
|
|
514
386
|
const ignorePatterns = await loadIgnorePatterns(workspacePath);
|
|
515
|
-
let filesToPush;
|
|
516
|
-
if (paths && paths.length > 0) filesToPush = filterIgnored(paths, ignorePatterns);
|
|
517
|
-
else filesToPush = await detectLocalChanges(workspacePath, ignorePatterns);
|
|
387
|
+
let filesToPush = paths && paths.length > 0 ? filterIgnored(paths, ignorePatterns) : await detectLocalChanges(workspacePath, ignorePatterns);
|
|
518
388
|
if (filter) filesToPush = filesToPush.filter((p) => p.startsWith(filter));
|
|
519
389
|
if (filesToPush.length === 0) {
|
|
520
390
|
if (!quiet) log.info("Nothing to push");
|
|
@@ -539,10 +409,9 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
539
409
|
},
|
|
540
410
|
async pull(options = {}) {
|
|
541
411
|
const { quiet = false } = options;
|
|
542
|
-
const [localManifest, remoteManifest] = await Promise.all([readManifest(workspacePath), client.
|
|
412
|
+
const [localManifest, remoteManifest] = await Promise.all([readManifest(workspacePath), client.syncGetManifest()]);
|
|
543
413
|
const diff = diffManifests(localManifest, remoteManifest);
|
|
544
|
-
|
|
545
|
-
if (filesToPull.length === 0) {
|
|
414
|
+
if (diff.toPull.length === 0) {
|
|
546
415
|
if (!quiet) log.info("Nothing to pull");
|
|
547
416
|
return {
|
|
548
417
|
pushed: 0,
|
|
@@ -551,8 +420,8 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
551
420
|
errors: 0
|
|
552
421
|
};
|
|
553
422
|
}
|
|
554
|
-
if (!quiet) log.info(`Pulling ${String(
|
|
555
|
-
const results = await downloadFiles(workspacePath,
|
|
423
|
+
if (!quiet) log.info(`Pulling ${String(diff.toPull.length)} file(s)`);
|
|
424
|
+
const results = await downloadFiles(workspacePath, [...diff.toPull], client, remoteManifest, { quiet });
|
|
556
425
|
const pulled = results.filter((r) => r.success).length;
|
|
557
426
|
const errors = results.filter((r) => !r.success).length;
|
|
558
427
|
if (!quiet) log.info(`Pull complete: ${String(pulled)} downloaded, ${String(errors)} failed`);
|
|
@@ -565,7 +434,7 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
565
434
|
},
|
|
566
435
|
async fullSync(options = {}) {
|
|
567
436
|
const { quiet = false } = options;
|
|
568
|
-
const [localManifest, remoteManifest] = await Promise.all([readManifest(workspacePath), client.
|
|
437
|
+
const [localManifest, remoteManifest] = await Promise.all([readManifest(workspacePath), client.syncGetManifest()]);
|
|
569
438
|
const ignorePatterns = await loadIgnorePatterns(workspacePath);
|
|
570
439
|
const localChanges = await detectLocalChanges(workspacePath, ignorePatterns);
|
|
571
440
|
const diff = diffManifests(localManifest, remoteManifest);
|
|
@@ -574,15 +443,14 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
574
443
|
let conflictCount = 0;
|
|
575
444
|
for (const conflictPath of trueConflicts) {
|
|
576
445
|
const absolutePath = join(workspacePath, conflictPath);
|
|
577
|
-
if (existsSync(absolutePath))
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
}
|
|
446
|
+
if (!existsSync(absolutePath)) continue;
|
|
447
|
+
const ext = extname(conflictPath);
|
|
448
|
+
const base = basename(conflictPath, ext);
|
|
449
|
+
const dir = dirname(conflictPath);
|
|
450
|
+
const conflictName = `${base}.conflict-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}${ext}`;
|
|
451
|
+
await writeFile(join(workspacePath, dir, conflictName), await readFile(absolutePath));
|
|
452
|
+
if (!quiet) log.warn(`Conflict: ${conflictPath} — saved as ${conflictName}`);
|
|
453
|
+
conflictCount++;
|
|
586
454
|
}
|
|
587
455
|
const filesToPush = filterIgnored([...diff.toPush, ...localChanges.filter((p) => !diff.conflicts.includes(p))], ignorePatterns);
|
|
588
456
|
const filesToPull = [
|
|
@@ -627,13 +495,12 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
627
495
|
conflicts: 0,
|
|
628
496
|
errors: 0
|
|
629
497
|
};
|
|
630
|
-
const remoteManifest = await client.
|
|
498
|
+
const remoteManifest = await client.syncGetManifest();
|
|
631
499
|
const localManifest = await readManifest(workspacePath);
|
|
632
500
|
const filesToPull = paths.filter((p) => {
|
|
633
501
|
if (!(p in remoteManifest.files)) return false;
|
|
634
|
-
const remoteEntry = remoteManifest.files[p];
|
|
635
502
|
if (!(p in localManifest.files)) return true;
|
|
636
|
-
return localManifest.files[p].hash !==
|
|
503
|
+
return localManifest.files[p].hash !== remoteManifest.files[p].hash;
|
|
637
504
|
});
|
|
638
505
|
if (filesToPull.length === 0) {
|
|
639
506
|
if (!quiet) log.info("All notified files already in sync");
|
|
@@ -673,10 +540,8 @@ async function createSyncEngine(workspacePathOverride) {
|
|
|
673
540
|
};
|
|
674
541
|
}
|
|
675
542
|
/**
|
|
676
|
-
*
|
|
677
|
-
*
|
|
678
|
-
* Compares current file hashes against the local manifest.
|
|
679
|
-
* Returns list of relative paths that need pushing.
|
|
543
|
+
* Walk the workspace and return paths whose hash differs from the manifest
|
|
544
|
+
* (or that are missing from it entirely).
|
|
680
545
|
*/
|
|
681
546
|
async function detectLocalChanges(workspacePath, ignorePatterns) {
|
|
682
547
|
const manifest = await readManifest(workspacePath);
|
|
@@ -689,33 +554,27 @@ async function detectLocalChanges(workspacePath, ignorePatterns) {
|
|
|
689
554
|
const relativePath = fullPath.slice(workspacePath.length + 1).replace(/\\/g, "/");
|
|
690
555
|
if (shouldSkipDir(entry.name)) continue;
|
|
691
556
|
if (entry.isDirectory()) {
|
|
692
|
-
if (!ignorePatterns.some((p) =>
|
|
693
|
-
if (p.endsWith("/**")) return relativePath.startsWith(p.slice(0, -3));
|
|
694
|
-
return false;
|
|
695
|
-
})) await walk(fullPath);
|
|
557
|
+
if (!ignorePatterns.some((p) => p.endsWith("/**") && relativePath.startsWith(p.slice(0, -3)))) await walk(fullPath);
|
|
696
558
|
} else if (entry.isFile()) {
|
|
697
|
-
const { shouldIgnore } = await import("./ignore.js").then((n) => n.n);
|
|
698
559
|
if (shouldIgnore(relativePath, ignorePatterns)) continue;
|
|
699
|
-
if (!(relativePath in manifest.files))
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
try {
|
|
703
|
-
if (await computeFileHash(fullPath) !== manifestEntry.hash) changed.push(relativePath);
|
|
704
|
-
} catch {}
|
|
560
|
+
if (!(relativePath in manifest.files)) {
|
|
561
|
+
changed.push(relativePath);
|
|
562
|
+
continue;
|
|
705
563
|
}
|
|
564
|
+
try {
|
|
565
|
+
if (await computeFileHash(fullPath) !== manifest.files[relativePath].hash) changed.push(relativePath);
|
|
566
|
+
} catch {}
|
|
706
567
|
}
|
|
707
568
|
}
|
|
708
569
|
}
|
|
709
570
|
await walk(workspacePath);
|
|
710
571
|
return changed;
|
|
711
572
|
}
|
|
712
|
-
/**
|
|
713
|
-
* Directories to always skip during walks.
|
|
714
|
-
*/
|
|
573
|
+
/** Directories the engine never descends into. */
|
|
715
574
|
function shouldSkipDir(name) {
|
|
716
575
|
return name === "node_modules" || name === ".git" || name === ".sst" || name === ".build" || name === "dist";
|
|
717
576
|
}
|
|
718
577
|
//#endregion
|
|
719
|
-
export {
|
|
578
|
+
export { filterIgnored as a, computeFileHash as c, removeManifestEntry as d, updateManifestEntry as f, withRetry as i, diffManifests as l, downloadFiles as n, loadIgnorePatterns as o, writeManifest as p, uploadFiles as r, shouldIgnore as s, createSyncEngine as t, readManifest as u };
|
|
720
579
|
|
|
721
580
|
//# sourceMappingURL=sync-engine.js.map
|