@remnic/cli 9.42.0 → 9.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +814 -313
- package/package.json +29 -29
package/dist/index.js
CHANGED
|
@@ -21,7 +21,7 @@ async function persistEnrichmentCandidate(storage, entityName, candidate) {
|
|
|
21
21
|
import fs12 from "fs";
|
|
22
22
|
import os from "os";
|
|
23
23
|
import path15 from "path";
|
|
24
|
-
import { createHash as
|
|
24
|
+
import { createHash as createHash3 } from "crypto";
|
|
25
25
|
import * as childProcess2 from "child_process";
|
|
26
26
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
27
27
|
import { gzipSync } from "zlib";
|
|
@@ -110,10 +110,10 @@ import {
|
|
|
110
110
|
parseXrayCliOptions,
|
|
111
111
|
renderXray,
|
|
112
112
|
OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
|
|
113
|
-
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
114
|
-
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as
|
|
113
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
|
|
114
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3,
|
|
115
115
|
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
116
|
-
applyOfflineSyncFileContentChunk,
|
|
116
|
+
applyOfflineSyncFileContentChunk as applyOfflineSyncFileContentChunk2,
|
|
117
117
|
applyOfflineSyncSnapshot,
|
|
118
118
|
buildOfflineSyncChangesetFromSnapshot,
|
|
119
119
|
drainPendingLifecycleForOfflineSync,
|
|
@@ -239,27 +239,297 @@ async function loadWecloneExportModule() {
|
|
|
239
239
|
}
|
|
240
240
|
|
|
241
241
|
// src/converge.ts
|
|
242
|
-
import * as
|
|
243
|
-
import
|
|
242
|
+
import * as fs3 from "fs";
|
|
243
|
+
import { createHash as createHash2 } from "crypto";
|
|
244
|
+
import * as path2 from "path";
|
|
244
245
|
import {
|
|
245
246
|
parseConfig as parseConfig2,
|
|
246
|
-
buildOfflineSyncSnapshotFromBase
|
|
247
|
+
buildOfflineSyncSnapshotFromBase,
|
|
248
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
249
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2,
|
|
250
|
+
applyOfflineSyncFileContentChunk
|
|
247
251
|
} from "@remnic/core";
|
|
248
252
|
import { resolveCorpusNamespaceRoots } from "@remnic/core/corpus-watermark.js";
|
|
249
253
|
import { listNamespaces } from "@remnic/core/namespaces/migrate.js";
|
|
250
254
|
import {
|
|
251
255
|
planReconciliation
|
|
252
256
|
} from "@remnic/core/reconcile/plan.js";
|
|
257
|
+
import {
|
|
258
|
+
defaultConvergeCursorPath,
|
|
259
|
+
readConvergeCursor,
|
|
260
|
+
writeConvergeCursor
|
|
261
|
+
} from "@remnic/core/reconcile/cursor.js";
|
|
262
|
+
|
|
263
|
+
// src/offline-storage-io.ts
|
|
264
|
+
import { mkdtemp, readdir, lstat, rm } from "fs/promises";
|
|
265
|
+
import fs2 from "fs";
|
|
266
|
+
import path from "path";
|
|
267
|
+
import { createHash, createDecipheriv } from "crypto";
|
|
268
|
+
import {
|
|
269
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
270
|
+
StorageManager
|
|
271
|
+
} from "@remnic/core";
|
|
272
|
+
import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
|
|
273
|
+
import {
|
|
274
|
+
AUTH_TAG_LENGTH,
|
|
275
|
+
ENVELOPE_HEADER_SIZE,
|
|
276
|
+
ENVELOPE_LAYOUT,
|
|
277
|
+
ENVELOPE_SALT_LENGTH,
|
|
278
|
+
ENVELOPE_VERSION,
|
|
279
|
+
FILE_FORMAT_FLAGS,
|
|
280
|
+
FILE_FORMAT_VERSION,
|
|
281
|
+
IV_LENGTH,
|
|
282
|
+
MAGIC_BYTES,
|
|
283
|
+
MAGIC_HEADER_SIZE,
|
|
284
|
+
SecureStoreLockedError,
|
|
285
|
+
filePathAad,
|
|
286
|
+
isEncryptedFile,
|
|
287
|
+
keyring,
|
|
288
|
+
readHeader,
|
|
289
|
+
secureStoreDir
|
|
290
|
+
} from "@remnic/core/secure-store";
|
|
291
|
+
async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
|
|
292
|
+
const storage = new StorageManager(memoryDir);
|
|
293
|
+
const header = await readHeader(memoryDir);
|
|
294
|
+
let secureStoreKey = null;
|
|
295
|
+
let secureStoreRequired = false;
|
|
296
|
+
if (header) {
|
|
297
|
+
secureStoreRequired = true;
|
|
298
|
+
storage.setSecureStoreRequired(true);
|
|
299
|
+
const key = keyring.getKey(secureStoreDir(memoryDir));
|
|
300
|
+
if (key) {
|
|
301
|
+
await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
|
|
302
|
+
secureStoreKey = key;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return { storage, secureStoreKey, secureStoreRequired };
|
|
306
|
+
}
|
|
307
|
+
async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
|
|
308
|
+
const memoryRoot = path.resolve(memoryDir);
|
|
309
|
+
const stateDir = path.dirname(filePath);
|
|
310
|
+
if (path.basename(stateDir) !== "state" || path.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
|
|
311
|
+
throw new Error(`invalid lifecycle ledger path: ${filePath}`);
|
|
312
|
+
}
|
|
313
|
+
const storageRoot = path.resolve(path.dirname(stateDir));
|
|
314
|
+
if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path.sep}`)) {
|
|
315
|
+
throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
|
|
316
|
+
}
|
|
317
|
+
const storage = new StorageManager(storageRoot);
|
|
318
|
+
if (configured.secureStoreRequired) {
|
|
319
|
+
storage.setSecureStoreRequired(true);
|
|
320
|
+
}
|
|
321
|
+
if (configured.secureStoreKey) {
|
|
322
|
+
await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
|
|
323
|
+
}
|
|
324
|
+
return storage;
|
|
325
|
+
}
|
|
326
|
+
async function createOfflineStorageIo(memoryDir, configuredStorage) {
|
|
327
|
+
await cleanupOrphanedOfflineDecryptStaging(memoryDir);
|
|
328
|
+
const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
|
|
329
|
+
return {
|
|
330
|
+
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
331
|
+
readFileDigest: async ({ filePath }) => {
|
|
332
|
+
const hash = createHash("sha256");
|
|
333
|
+
let bytes = 0;
|
|
334
|
+
for await (const rawChunk of readOfflineSyncFileChunks({
|
|
335
|
+
filePath,
|
|
336
|
+
memoryDir,
|
|
337
|
+
secureStoreKey,
|
|
338
|
+
chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
|
|
339
|
+
})) {
|
|
340
|
+
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
|
|
341
|
+
hash.update(chunk);
|
|
342
|
+
bytes += chunk.length;
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
sha256: hash.digest("hex"),
|
|
346
|
+
bytes
|
|
347
|
+
};
|
|
348
|
+
},
|
|
349
|
+
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
350
|
+
filePath,
|
|
351
|
+
memoryDir,
|
|
352
|
+
secureStoreKey,
|
|
353
|
+
chunkSize
|
|
354
|
+
}),
|
|
355
|
+
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
356
|
+
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
357
|
+
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
358
|
+
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
|
|
362
|
+
async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
|
|
363
|
+
let entries;
|
|
364
|
+
try {
|
|
365
|
+
entries = await readdir(memoryDir);
|
|
366
|
+
} catch {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
const now = Date.now();
|
|
370
|
+
for (const name of entries) {
|
|
371
|
+
if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
|
|
372
|
+
const dir = path.join(memoryDir, name);
|
|
373
|
+
try {
|
|
374
|
+
const info = await lstat(dir);
|
|
375
|
+
if (!info.isDirectory() || info.isSymbolicLink()) continue;
|
|
376
|
+
if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
|
|
377
|
+
await rm(dir, { recursive: true, force: true });
|
|
378
|
+
} catch {
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
async function* readOfflineSyncFileChunks(options) {
|
|
383
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
|
|
384
|
+
if (!isEncryptedFile(header)) {
|
|
385
|
+
yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (!options.secureStoreKey) {
|
|
389
|
+
throw new SecureStoreLockedError(
|
|
390
|
+
`secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
yield* readEncryptedOfflineFileChunks({
|
|
394
|
+
filePath: options.filePath,
|
|
395
|
+
memoryDir: options.memoryDir,
|
|
396
|
+
key: options.secureStoreKey,
|
|
397
|
+
chunkSize: options.chunkSize
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
async function readFilePrefix(filePath, length) {
|
|
401
|
+
const handle = await fs2.promises.open(filePath, "r");
|
|
402
|
+
try {
|
|
403
|
+
const out = Buffer.alloc(length);
|
|
404
|
+
const { bytesRead } = await handle.read(out, 0, length, 0);
|
|
405
|
+
return out.subarray(0, bytesRead);
|
|
406
|
+
} finally {
|
|
407
|
+
await handle.close();
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
async function* readPlainOfflineFileChunks(filePath, chunkSize) {
|
|
411
|
+
const stream = fs2.createReadStream(filePath, { highWaterMark: chunkSize });
|
|
412
|
+
for await (const chunk of stream) {
|
|
413
|
+
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
async function* readEncryptedOfflineFileChunks(options) {
|
|
417
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
418
|
+
if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
|
|
419
|
+
throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
|
|
420
|
+
}
|
|
421
|
+
const version = header.readUInt8(MAGIC_BYTES.length);
|
|
422
|
+
const flags = header.readUInt8(MAGIC_BYTES.length + 1);
|
|
423
|
+
if (version !== FILE_FORMAT_VERSION) {
|
|
424
|
+
throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
|
|
425
|
+
}
|
|
426
|
+
if (flags !== FILE_FORMAT_FLAGS) {
|
|
427
|
+
throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
|
|
428
|
+
}
|
|
429
|
+
const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
|
|
430
|
+
const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
|
|
431
|
+
if (envelopeVersion !== ENVELOPE_VERSION) {
|
|
432
|
+
throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
|
|
433
|
+
}
|
|
434
|
+
const salt = envelopeHeader.subarray(
|
|
435
|
+
ENVELOPE_LAYOUT.salt,
|
|
436
|
+
ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
|
|
437
|
+
);
|
|
438
|
+
const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
|
|
439
|
+
const authTag = envelopeHeader.subarray(
|
|
440
|
+
ENVELOPE_LAYOUT.authTag,
|
|
441
|
+
ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
|
|
442
|
+
);
|
|
443
|
+
const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
|
|
444
|
+
let lastError;
|
|
445
|
+
for (const aad of aadCandidates) {
|
|
446
|
+
const tempDir = await mkdtemp(path.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
|
|
447
|
+
const tempPath = path.join(tempDir, "content");
|
|
448
|
+
try {
|
|
449
|
+
const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
|
|
450
|
+
authTagLength: AUTH_TAG_LENGTH
|
|
451
|
+
});
|
|
452
|
+
decipher.setAuthTag(authTag);
|
|
453
|
+
decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
|
|
454
|
+
const output = fs2.createWriteStream(tempPath, { mode: 384 });
|
|
455
|
+
try {
|
|
456
|
+
const stream = fs2.createReadStream(options.filePath, {
|
|
457
|
+
start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
|
|
458
|
+
highWaterMark: options.chunkSize
|
|
459
|
+
});
|
|
460
|
+
for await (const encryptedChunk of stream) {
|
|
461
|
+
const plain = decipher.update(
|
|
462
|
+
Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
|
|
463
|
+
);
|
|
464
|
+
if (plain.length > 0 && !output.write(plain)) {
|
|
465
|
+
await new Promise((resolve, reject) => {
|
|
466
|
+
output.once("drain", resolve);
|
|
467
|
+
output.once("error", reject);
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
const finalPlain = decipher.final();
|
|
472
|
+
if (finalPlain.length > 0 && !output.write(finalPlain)) {
|
|
473
|
+
await new Promise((resolve, reject) => {
|
|
474
|
+
output.once("drain", resolve);
|
|
475
|
+
output.once("error", reject);
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
} finally {
|
|
479
|
+
await closeWriteStream(output);
|
|
480
|
+
}
|
|
481
|
+
yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
|
|
482
|
+
return;
|
|
483
|
+
} catch (error) {
|
|
484
|
+
lastError = error;
|
|
485
|
+
} finally {
|
|
486
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
|
|
490
|
+
}
|
|
491
|
+
function offlineFileAadCandidates(filePath, memoryDir) {
|
|
492
|
+
const candidates = [filePathAad(filePath, memoryDir)];
|
|
493
|
+
const relative = path.relative(memoryDir, filePath);
|
|
494
|
+
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return candidates;
|
|
495
|
+
const parts = relative.split(path.sep);
|
|
496
|
+
if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
|
|
497
|
+
candidates.push(filePathAad(filePath, path.join(memoryDir, "namespaces", parts[1])));
|
|
498
|
+
}
|
|
499
|
+
const memoryParts = path.resolve(memoryDir).split(path.sep);
|
|
500
|
+
if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
|
|
501
|
+
const topLevelRoot = memoryParts.slice(0, -2).join(path.sep) || path.sep;
|
|
502
|
+
const topRelative = path.relative(topLevelRoot, filePath);
|
|
503
|
+
if (topRelative && !topRelative.startsWith("..") && !path.isAbsolute(topRelative) && topRelative.split(path.sep)[0] === "namespaces" && topRelative.split(path.sep)[1] === memoryParts.at(-1)) {
|
|
504
|
+
candidates.push(filePathAad(filePath, topLevelRoot));
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return candidates;
|
|
508
|
+
}
|
|
509
|
+
async function closeWriteStream(stream) {
|
|
510
|
+
await new Promise((resolve, reject) => {
|
|
511
|
+
stream.once("error", reject);
|
|
512
|
+
stream.end(() => resolve());
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
function secureStoreEnvelopeHeaderAad(salt) {
|
|
516
|
+
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
517
|
+
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
518
|
+
Buffer.from(salt).copy(out, 1);
|
|
519
|
+
return out;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// src/converge.ts
|
|
253
523
|
import { resolveAgentAccessAuthToken } from "@remnic/core/resolve-auth-token.js";
|
|
254
524
|
async function readLocalTombstones(rootDir) {
|
|
255
525
|
const shaSet = /* @__PURE__ */ new Set();
|
|
256
526
|
const candidates = [
|
|
257
|
-
|
|
258
|
-
|
|
527
|
+
path2.join(rootDir, "state", "tombstones.jsonl"),
|
|
528
|
+
path2.join(rootDir, "tombstones.jsonl")
|
|
259
529
|
];
|
|
260
530
|
for (const tombPath of candidates) {
|
|
261
531
|
try {
|
|
262
|
-
const content = await
|
|
532
|
+
const content = await fs3.promises.readFile(tombPath, "utf-8");
|
|
263
533
|
for (const line of content.split("\n")) {
|
|
264
534
|
const trimmed = line.trim();
|
|
265
535
|
if (!trimmed) continue;
|
|
@@ -289,44 +559,190 @@ async function fetchPeerSnapshot(peerUrl, namespace, token, fetchImpl = globalTh
|
|
|
289
559
|
`/engram/v1/offline-sync/snapshot?namespace=${encodeURIComponent(namespace)}&content=false`
|
|
290
560
|
];
|
|
291
561
|
const headers = token ? { authorization: `Bearer ${token}` } : {};
|
|
562
|
+
let lastFailure = "no snapshot route responded";
|
|
292
563
|
for (const route of routes) {
|
|
564
|
+
let response;
|
|
293
565
|
try {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
566
|
+
response = await fetchImpl(`${base}${route}`, { headers });
|
|
567
|
+
} catch (error) {
|
|
568
|
+
lastFailure = error instanceof Error ? error.message : String(error);
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
if (!response.ok) {
|
|
572
|
+
lastFailure = `HTTP ${response.status}`;
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
let data;
|
|
576
|
+
try {
|
|
577
|
+
data = await response.json();
|
|
578
|
+
} catch {
|
|
579
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: response was not JSON`);
|
|
580
|
+
}
|
|
581
|
+
if (!data || typeof data !== "object" || !("files" in data) || !Array.isArray(data.files)) {
|
|
582
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: files must be an array`);
|
|
583
|
+
}
|
|
584
|
+
const files = data.files.map((item, index) => {
|
|
585
|
+
if (!item || typeof item !== "object" || !("path" in item) || typeof item.path !== "string" || !("sha256" in item) || typeof item.sha256 !== "string") {
|
|
586
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed file at index ${index}`);
|
|
309
587
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
588
|
+
return {
|
|
589
|
+
path: item.path,
|
|
590
|
+
sha256: item.sha256,
|
|
591
|
+
mtimeMs: "mtimeMs" in item && typeof item.mtimeMs === "number" ? item.mtimeMs : void 0,
|
|
592
|
+
bytes: "bytes" in item && typeof item.bytes === "number" ? item.bytes : void 0
|
|
593
|
+
};
|
|
594
|
+
});
|
|
595
|
+
const rawTombstones = "tombstones" in data ? data.tombstones : void 0;
|
|
596
|
+
if (rawTombstones !== void 0 && !Array.isArray(rawTombstones)) {
|
|
597
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: tombstones must be an array`);
|
|
598
|
+
}
|
|
599
|
+
const tombstones = /* @__PURE__ */ new Set();
|
|
600
|
+
for (const tombstone of rawTombstones ?? []) {
|
|
601
|
+
if (typeof tombstone !== "string" || !/^[0-9a-f]{64}$/i.test(tombstone)) {
|
|
602
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed tombstone`);
|
|
603
|
+
}
|
|
604
|
+
tombstones.add(tombstone.toLowerCase());
|
|
605
|
+
}
|
|
606
|
+
return { files, tombstones };
|
|
607
|
+
}
|
|
608
|
+
throw new Error(`failed to fetch peer snapshot for namespace ${namespace}: ${lastFailure}`);
|
|
609
|
+
}
|
|
610
|
+
function requiredResponseNumber(response, name) {
|
|
611
|
+
const raw = response.headers.get(name);
|
|
612
|
+
const value = raw === null ? Number.NaN : Number(raw);
|
|
613
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
614
|
+
throw new Error(`offline file content response had invalid ${name}`);
|
|
615
|
+
}
|
|
616
|
+
return value;
|
|
617
|
+
}
|
|
618
|
+
async function fetchPeerFileContent(peerUrl, namespace, filePath, token, fetchImpl = globalThis.fetch) {
|
|
619
|
+
const base = peerUrl.replace(/\/+$/, "");
|
|
620
|
+
const routes = [
|
|
621
|
+
"/remnic/v1/offline-sync/file-content",
|
|
622
|
+
"/engram/v1/offline-sync/file-content"
|
|
623
|
+
];
|
|
624
|
+
const headers = {
|
|
625
|
+
"content-type": "application/json",
|
|
626
|
+
...token ? { authorization: `Bearer ${token}` } : {}
|
|
627
|
+
};
|
|
628
|
+
for (const route of routes) {
|
|
629
|
+
try {
|
|
630
|
+
const chunks = [];
|
|
631
|
+
const hash = createHash2("sha256");
|
|
632
|
+
let offset = 0;
|
|
633
|
+
let expectedBytes;
|
|
634
|
+
let expectedSha256;
|
|
635
|
+
let mtimeMs;
|
|
636
|
+
do {
|
|
637
|
+
const response = await fetchImpl(`${base}${route}`, {
|
|
638
|
+
method: "POST",
|
|
639
|
+
headers,
|
|
640
|
+
body: JSON.stringify({
|
|
641
|
+
namespace,
|
|
642
|
+
includeTranscripts: false,
|
|
643
|
+
path: filePath,
|
|
644
|
+
offset,
|
|
645
|
+
length: OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES
|
|
646
|
+
})
|
|
647
|
+
});
|
|
648
|
+
if (!response.ok) throw new Error(`offline file content request failed: ${response.status}`);
|
|
649
|
+
const content = Buffer.from(await response.arrayBuffer());
|
|
650
|
+
const chunkOffset = requiredResponseNumber(response, "x-remnic-chunk-offset");
|
|
651
|
+
const chunkBytes = requiredResponseNumber(response, "x-remnic-chunk-bytes");
|
|
652
|
+
const totalBytes = requiredResponseNumber(response, "x-remnic-file-bytes");
|
|
653
|
+
const responseMtimeMs = requiredResponseNumber(response, "x-remnic-file-mtime-ms");
|
|
654
|
+
const sha256 = response.headers.get("x-remnic-file-sha256");
|
|
655
|
+
const encodedPath = response.headers.get("x-remnic-file-path");
|
|
656
|
+
if (!sha256 || chunkOffset !== offset || chunkBytes !== content.length || encodedPath !== null && decodeURIComponent(encodedPath) !== filePath || expectedBytes !== void 0 && expectedBytes !== totalBytes || expectedSha256 !== void 0 && expectedSha256 !== sha256) {
|
|
657
|
+
throw new Error(`offline file content response changed during transfer: ${filePath}`);
|
|
658
|
+
}
|
|
659
|
+
if (content.length === 0 && offset < totalBytes) {
|
|
660
|
+
throw new Error(`offline file content chunk was empty before EOF: ${filePath}`);
|
|
316
661
|
}
|
|
662
|
+
expectedBytes = totalBytes;
|
|
663
|
+
expectedSha256 = sha256;
|
|
664
|
+
mtimeMs = responseMtimeMs;
|
|
665
|
+
chunks.push(content);
|
|
666
|
+
hash.update(content);
|
|
667
|
+
offset += content.length;
|
|
668
|
+
} while (expectedBytes === void 0 || offset < expectedBytes);
|
|
669
|
+
if (expectedBytes === void 0 || expectedSha256 === void 0 || mtimeMs === void 0 || offset !== expectedBytes || hash.digest("hex") !== expectedSha256) {
|
|
670
|
+
throw new Error(`offline file content checksum mismatch: ${filePath}`);
|
|
317
671
|
}
|
|
318
|
-
return {
|
|
672
|
+
return {
|
|
673
|
+
content: Buffer.concat(chunks, expectedBytes),
|
|
674
|
+
sha256: expectedSha256,
|
|
675
|
+
bytes: expectedBytes,
|
|
676
|
+
mtimeMs
|
|
677
|
+
};
|
|
319
678
|
} catch {
|
|
320
679
|
}
|
|
321
680
|
}
|
|
322
|
-
return
|
|
681
|
+
return null;
|
|
682
|
+
}
|
|
683
|
+
async function postPeerFileContent(peerUrl, namespace, filePath, content, metadata, token, fetchImpl = globalThis.fetch) {
|
|
684
|
+
const base = peerUrl.replace(/\/+$/, "");
|
|
685
|
+
const routes = [
|
|
686
|
+
`/remnic/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`,
|
|
687
|
+
`/engram/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`
|
|
688
|
+
];
|
|
689
|
+
for (const route of routes) {
|
|
690
|
+
try {
|
|
691
|
+
let offset = 0;
|
|
692
|
+
do {
|
|
693
|
+
const chunk = content.subarray(
|
|
694
|
+
offset,
|
|
695
|
+
Math.min(content.length, offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2)
|
|
696
|
+
);
|
|
697
|
+
const headers = {
|
|
698
|
+
"content-type": "application/octet-stream",
|
|
699
|
+
"x-remnic-include-transcripts": "false",
|
|
700
|
+
"x-remnic-source-id": encodeURIComponent("remnic-converge"),
|
|
701
|
+
"x-remnic-file-path": encodeURIComponent(filePath),
|
|
702
|
+
"x-remnic-file-sha256": metadata.sha256,
|
|
703
|
+
"x-remnic-file-bytes": String(content.length),
|
|
704
|
+
"x-remnic-file-mtime-ms": String(metadata.mtimeMs),
|
|
705
|
+
"x-remnic-chunk-offset": String(offset),
|
|
706
|
+
...metadata.baseSha256 ? { "x-remnic-base-sha256": metadata.baseSha256 } : {},
|
|
707
|
+
...token ? { authorization: `Bearer ${token}` } : {}
|
|
708
|
+
};
|
|
709
|
+
const response = await fetchImpl(`${base}${route}`, {
|
|
710
|
+
method: "POST",
|
|
711
|
+
headers,
|
|
712
|
+
body: new Uint8Array(chunk)
|
|
713
|
+
});
|
|
714
|
+
if (!response.ok) throw new Error(`offline apply-file-content request failed: ${response.status}`);
|
|
715
|
+
const result = await response.json().catch(() => null);
|
|
716
|
+
if (!result || typeof result !== "object" || !("done" in result) || typeof result.done !== "boolean" || !("applied" in result) || typeof result.applied !== "boolean" || !("skipped" in result) || typeof result.skipped !== "boolean" || "conflict" in result && result.conflict) {
|
|
717
|
+
return false;
|
|
718
|
+
}
|
|
719
|
+
if (result.done) {
|
|
720
|
+
return result.skipped || result.applied && offset + chunk.length === content.length;
|
|
721
|
+
}
|
|
722
|
+
if (result.applied || result.skipped || chunk.length === 0) {
|
|
723
|
+
return false;
|
|
724
|
+
}
|
|
725
|
+
offset += chunk.length;
|
|
726
|
+
} while (offset < content.length);
|
|
727
|
+
return false;
|
|
728
|
+
} catch {
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
return false;
|
|
323
732
|
}
|
|
324
733
|
async function computeConvergePlan(options = {}) {
|
|
734
|
+
const baseMap = /* @__PURE__ */ new Map();
|
|
325
735
|
const namespacesToPlan = /* @__PURE__ */ new Set();
|
|
326
736
|
const localMap = /* @__PURE__ */ new Map();
|
|
327
737
|
const localTombstones = /* @__PURE__ */ new Map();
|
|
328
738
|
const peerMap = /* @__PURE__ */ new Map();
|
|
329
739
|
const peerTombstones = /* @__PURE__ */ new Map();
|
|
740
|
+
if (options.baseFilesByNamespace) {
|
|
741
|
+
for (const [ns, files] of options.baseFilesByNamespace) {
|
|
742
|
+
namespacesToPlan.add(ns);
|
|
743
|
+
baseMap.set(ns, files);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
330
746
|
if (options.localFilesByNamespace) {
|
|
331
747
|
for (const [ns, files] of options.localFilesByNamespace) {
|
|
332
748
|
namespacesToPlan.add(ns);
|
|
@@ -403,17 +819,315 @@ async function computeConvergePlan(options = {}) {
|
|
|
403
819
|
peerTombstones.set(ns, peerData.tombstones);
|
|
404
820
|
}
|
|
405
821
|
}
|
|
822
|
+
const memoryDir = options.cursorDir ?? config?.memoryDir;
|
|
823
|
+
if (!options.baseFilesByNamespace && memoryDir && options.peerUrl) {
|
|
824
|
+
for (const ns of namespacesToPlan) {
|
|
825
|
+
const cursorPath = defaultConvergeCursorPath(memoryDir, options.peerUrl, ns);
|
|
826
|
+
const cursor = await readConvergeCursor(cursorPath);
|
|
827
|
+
if (cursor?.baseFiles && cursor.baseFiles.length > 0) {
|
|
828
|
+
baseMap.set(ns, cursor.baseFiles);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
}
|
|
406
832
|
const inputs = [];
|
|
407
833
|
for (const ns of [...namespacesToPlan].sort()) {
|
|
408
834
|
inputs.push({
|
|
409
835
|
namespace: ns,
|
|
410
836
|
local: localMap.get(ns) ?? [],
|
|
411
837
|
peer: peerMap.get(ns) ?? [],
|
|
838
|
+
base: baseMap.get(ns),
|
|
412
839
|
tombstonedFileSha256: localTombstones.get(ns) ?? [],
|
|
413
840
|
peerTombstonedFileSha256: peerTombstones.get(ns) ?? []
|
|
414
841
|
});
|
|
415
842
|
}
|
|
416
|
-
return planReconciliation(inputs);
|
|
843
|
+
return planReconciliation(inputs, { conflictPolicy: options.conflictPolicy });
|
|
844
|
+
}
|
|
845
|
+
async function executeConvergeApply(options = {}) {
|
|
846
|
+
const conflictPolicy = options.conflictPolicy ?? "manual";
|
|
847
|
+
const plan = await computeConvergePlan({ ...options, conflictPolicy });
|
|
848
|
+
if (plan.converged && !options.dryRun) {
|
|
849
|
+
await updateCursorsForPlan(plan, options);
|
|
850
|
+
return {
|
|
851
|
+
converged: true,
|
|
852
|
+
status: "converged",
|
|
853
|
+
plan,
|
|
854
|
+
transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
|
|
855
|
+
cursorUpdated: true
|
|
856
|
+
};
|
|
857
|
+
}
|
|
858
|
+
const unresolvedCount = plan.byNamespace.reduce((acc, report) => acc + report.unresolved, 0);
|
|
859
|
+
if (unresolvedCount > 0 && conflictPolicy === "manual") {
|
|
860
|
+
return {
|
|
861
|
+
converged: false,
|
|
862
|
+
status: "stopped_unresolved_conflicts",
|
|
863
|
+
plan,
|
|
864
|
+
transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
|
|
865
|
+
cursorUpdated: false
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
const plannedTransfers = {
|
|
869
|
+
pulled: 0,
|
|
870
|
+
pushed: 0,
|
|
871
|
+
conflictsResolved: 0,
|
|
872
|
+
suppressed: 0,
|
|
873
|
+
failed: 0
|
|
874
|
+
};
|
|
875
|
+
for (const entry of plan.entries) {
|
|
876
|
+
if (entry.action === "pull") plannedTransfers.pulled += 1;
|
|
877
|
+
else if (entry.action === "push") plannedTransfers.pushed += 1;
|
|
878
|
+
else if (entry.action === "conflict") plannedTransfers.conflictsResolved += 1;
|
|
879
|
+
else if (entry.action === "suppress") plannedTransfers.suppressed += 1;
|
|
880
|
+
}
|
|
881
|
+
if (options.dryRun) {
|
|
882
|
+
return {
|
|
883
|
+
converged: false,
|
|
884
|
+
status: "dry_run",
|
|
885
|
+
plan,
|
|
886
|
+
transfers: plannedTransfers,
|
|
887
|
+
cursorUpdated: false
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const actualTransfers = {
|
|
891
|
+
pulled: 0,
|
|
892
|
+
pushed: 0,
|
|
893
|
+
conflictsResolved: 0,
|
|
894
|
+
suppressed: 0,
|
|
895
|
+
failed: 0
|
|
896
|
+
};
|
|
897
|
+
let resolvedToken;
|
|
898
|
+
if (options.peerToken) {
|
|
899
|
+
try {
|
|
900
|
+
resolvedToken = await resolveAgentAccessAuthToken(options.peerToken, {
|
|
901
|
+
resolveSecretRef: options.resolveSecretRef
|
|
902
|
+
});
|
|
903
|
+
} catch {
|
|
904
|
+
resolvedToken = options.peerToken;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
const fetchFn = options.fetchImpl ?? globalThis.fetch;
|
|
908
|
+
let config = options.config;
|
|
909
|
+
if (!config) {
|
|
910
|
+
try {
|
|
911
|
+
config = parseConfig2({});
|
|
912
|
+
} catch {
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
const rootMap = /* @__PURE__ */ new Map();
|
|
916
|
+
if (config) {
|
|
917
|
+
try {
|
|
918
|
+
const roots = await resolveCorpusNamespaceRoots({ config });
|
|
919
|
+
for (const r of roots) {
|
|
920
|
+
rootMap.set(r.namespace, r.rootDir);
|
|
921
|
+
}
|
|
922
|
+
} catch {
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
for (const entry of plan.entries) {
|
|
926
|
+
if (entry.action === "identical") continue;
|
|
927
|
+
let transferType = "none";
|
|
928
|
+
if (entry.action === "pull") {
|
|
929
|
+
transferType = "pull";
|
|
930
|
+
} else if (entry.action === "push") {
|
|
931
|
+
transferType = "push";
|
|
932
|
+
} else if (entry.action === "suppress") {
|
|
933
|
+
transferType = "suppress";
|
|
934
|
+
} else if (entry.action === "conflict") {
|
|
935
|
+
if (entry.resolution === "peer-wins") {
|
|
936
|
+
transferType = "pull";
|
|
937
|
+
} else if (entry.resolution === "local-wins") {
|
|
938
|
+
transferType = "push";
|
|
939
|
+
} else if (entry.resolution === "supersede-link") {
|
|
940
|
+
if (entry.newerSide === "peer") transferType = "pull";
|
|
941
|
+
else if (entry.newerSide === "local") transferType = "push";
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
if (transferType === "pull") {
|
|
945
|
+
let remoteFile = null;
|
|
946
|
+
const buffered = options.peerFileBuffers?.get(entry.namespace)?.get(entry.path);
|
|
947
|
+
if (buffered) {
|
|
948
|
+
const state = options.peerFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path);
|
|
949
|
+
remoteFile = {
|
|
950
|
+
content: buffered,
|
|
951
|
+
sha256: state?.sha256 ?? entry.peerSha256 ?? createHash2("sha256").update(buffered).digest("hex"),
|
|
952
|
+
bytes: buffered.length,
|
|
953
|
+
mtimeMs: state?.mtimeMs ?? 0
|
|
954
|
+
};
|
|
955
|
+
} else if (options.peerUrl) {
|
|
956
|
+
remoteFile = await fetchPeerFileContent(
|
|
957
|
+
options.peerUrl,
|
|
958
|
+
entry.namespace,
|
|
959
|
+
entry.path,
|
|
960
|
+
resolvedToken,
|
|
961
|
+
fetchFn
|
|
962
|
+
);
|
|
963
|
+
}
|
|
964
|
+
if (remoteFile !== null && (!entry.peerSha256 || remoteFile.sha256 === entry.peerSha256)) {
|
|
965
|
+
if (options.localFileBuffers) {
|
|
966
|
+
let nsMap = options.localFileBuffers.get(entry.namespace);
|
|
967
|
+
if (!nsMap) {
|
|
968
|
+
nsMap = /* @__PURE__ */ new Map();
|
|
969
|
+
options.localFileBuffers.set(entry.namespace, nsMap);
|
|
970
|
+
}
|
|
971
|
+
nsMap.set(entry.path, remoteFile.content);
|
|
972
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
973
|
+
else actualTransfers.pulled += 1;
|
|
974
|
+
} else {
|
|
975
|
+
const rootDir = rootMap.get(entry.namespace);
|
|
976
|
+
if (rootDir) {
|
|
977
|
+
const io = await createOfflineStorageIo(rootDir);
|
|
978
|
+
const expectedLocalSha256 = entry.action === "conflict" ? entry.localSha256 : entry.baseSha256;
|
|
979
|
+
let offset = 0;
|
|
980
|
+
let transferComplete = false;
|
|
981
|
+
do {
|
|
982
|
+
const chunk = remoteFile.content.subarray(
|
|
983
|
+
offset,
|
|
984
|
+
Math.min(
|
|
985
|
+
remoteFile.content.length,
|
|
986
|
+
offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2
|
|
987
|
+
)
|
|
988
|
+
);
|
|
989
|
+
const chunkResult = await applyOfflineSyncFileContentChunk({
|
|
990
|
+
root: rootDir,
|
|
991
|
+
sourceId: "remnic-converge",
|
|
992
|
+
path: entry.path,
|
|
993
|
+
sha256: remoteFile.sha256,
|
|
994
|
+
bytes: remoteFile.bytes,
|
|
995
|
+
mtimeMs: remoteFile.mtimeMs,
|
|
996
|
+
offset,
|
|
997
|
+
content: chunk,
|
|
998
|
+
...expectedLocalSha256 ? { baseSha256: expectedLocalSha256 } : {},
|
|
999
|
+
readFile: io.readFile,
|
|
1000
|
+
readFileDigest: io.readFileDigest,
|
|
1001
|
+
writeFile: io.writeFile,
|
|
1002
|
+
writeStagingFile: io.writeStagingFile,
|
|
1003
|
+
writeFileChunks: io.writeFileChunks
|
|
1004
|
+
});
|
|
1005
|
+
if (chunkResult.conflict) {
|
|
1006
|
+
break;
|
|
1007
|
+
}
|
|
1008
|
+
if (chunkResult.done) {
|
|
1009
|
+
transferComplete = chunkResult.applied || chunkResult.skipped;
|
|
1010
|
+
break;
|
|
1011
|
+
}
|
|
1012
|
+
if (chunkResult.applied || chunkResult.skipped || chunk.length === 0) {
|
|
1013
|
+
break;
|
|
1014
|
+
}
|
|
1015
|
+
offset += chunk.length;
|
|
1016
|
+
} while (offset < remoteFile.content.length);
|
|
1017
|
+
if (transferComplete) {
|
|
1018
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1019
|
+
else actualTransfers.pulled += 1;
|
|
1020
|
+
} else {
|
|
1021
|
+
actualTransfers.failed += 1;
|
|
1022
|
+
}
|
|
1023
|
+
} else {
|
|
1024
|
+
actualTransfers.failed += 1;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
} else {
|
|
1028
|
+
actualTransfers.failed += 1;
|
|
1029
|
+
}
|
|
1030
|
+
} else if (transferType === "push") {
|
|
1031
|
+
let content = null;
|
|
1032
|
+
let mtimeMs = options.localFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path)?.mtimeMs;
|
|
1033
|
+
if (options.localFileBuffers?.get(entry.namespace)?.has(entry.path)) {
|
|
1034
|
+
content = options.localFileBuffers.get(entry.namespace).get(entry.path);
|
|
1035
|
+
} else {
|
|
1036
|
+
const rootDir = rootMap.get(entry.namespace);
|
|
1037
|
+
if (rootDir) {
|
|
1038
|
+
const filePath = path2.join(rootDir, entry.path);
|
|
1039
|
+
try {
|
|
1040
|
+
const io = await createOfflineStorageIo(rootDir);
|
|
1041
|
+
content = await io.readFile({ root: rootDir, path: entry.path, filePath });
|
|
1042
|
+
mtimeMs ??= (await fs3.promises.stat(filePath)).mtimeMs;
|
|
1043
|
+
} catch {
|
|
1044
|
+
content = null;
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
if (content !== null) {
|
|
1049
|
+
if (options.peerFileBuffers) {
|
|
1050
|
+
let nsMap = options.peerFileBuffers.get(entry.namespace);
|
|
1051
|
+
if (!nsMap) {
|
|
1052
|
+
nsMap = /* @__PURE__ */ new Map();
|
|
1053
|
+
options.peerFileBuffers.set(entry.namespace, nsMap);
|
|
1054
|
+
}
|
|
1055
|
+
nsMap.set(entry.path, content);
|
|
1056
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1057
|
+
else actualTransfers.pushed += 1;
|
|
1058
|
+
} else if (options.peerUrl && entry.localSha256) {
|
|
1059
|
+
const expectedPeerSha256 = entry.action === "conflict" ? entry.peerSha256 : entry.baseSha256;
|
|
1060
|
+
const ok = await postPeerFileContent(
|
|
1061
|
+
options.peerUrl,
|
|
1062
|
+
entry.namespace,
|
|
1063
|
+
entry.path,
|
|
1064
|
+
content,
|
|
1065
|
+
{
|
|
1066
|
+
sha256: entry.localSha256,
|
|
1067
|
+
mtimeMs: mtimeMs ?? 0,
|
|
1068
|
+
...expectedPeerSha256 ? { baseSha256: expectedPeerSha256 } : {}
|
|
1069
|
+
},
|
|
1070
|
+
resolvedToken,
|
|
1071
|
+
fetchFn
|
|
1072
|
+
);
|
|
1073
|
+
if (ok) {
|
|
1074
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1075
|
+
else actualTransfers.pushed += 1;
|
|
1076
|
+
} else {
|
|
1077
|
+
actualTransfers.failed += 1;
|
|
1078
|
+
}
|
|
1079
|
+
} else {
|
|
1080
|
+
actualTransfers.failed += 1;
|
|
1081
|
+
}
|
|
1082
|
+
} else {
|
|
1083
|
+
actualTransfers.failed += 1;
|
|
1084
|
+
}
|
|
1085
|
+
} else if (transferType === "suppress") {
|
|
1086
|
+
actualTransfers.suppressed += 1;
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
let cursorUpdated = false;
|
|
1090
|
+
if (actualTransfers.failed === 0) {
|
|
1091
|
+
await updateCursorsForPlan(plan, options);
|
|
1092
|
+
cursorUpdated = true;
|
|
1093
|
+
}
|
|
1094
|
+
return {
|
|
1095
|
+
converged: actualTransfers.failed === 0,
|
|
1096
|
+
status: "applied",
|
|
1097
|
+
plan,
|
|
1098
|
+
transfers: actualTransfers,
|
|
1099
|
+
cursorUpdated
|
|
1100
|
+
};
|
|
1101
|
+
}
|
|
1102
|
+
async function updateCursorsForPlan(plan, options) {
|
|
1103
|
+
const peerUrl = options.peerUrl ?? "local";
|
|
1104
|
+
let memoryDir;
|
|
1105
|
+
if (options.cursorDir) {
|
|
1106
|
+
memoryDir = options.cursorDir;
|
|
1107
|
+
} else if (options.config) {
|
|
1108
|
+
memoryDir = options.config.memoryDir;
|
|
1109
|
+
}
|
|
1110
|
+
if (!memoryDir) return;
|
|
1111
|
+
const namespaces = new Set(plan.byNamespace.map((n) => n.namespace));
|
|
1112
|
+
for (const ns of namespaces) {
|
|
1113
|
+
const cursorPath = defaultConvergeCursorPath(memoryDir, peerUrl, ns);
|
|
1114
|
+
const nsEntries = plan.entries.filter((e) => e.namespace === ns);
|
|
1115
|
+
const baseFiles = nsEntries.map((e) => ({
|
|
1116
|
+
path: e.path,
|
|
1117
|
+
sha256: e.localSha256 ?? e.peerSha256 ?? "unknown"
|
|
1118
|
+
}));
|
|
1119
|
+
const cursorState = {
|
|
1120
|
+
version: 1,
|
|
1121
|
+
peerUrl,
|
|
1122
|
+
namespace: ns,
|
|
1123
|
+
lastConvergedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1124
|
+
baseFiles
|
|
1125
|
+
};
|
|
1126
|
+
try {
|
|
1127
|
+
await writeConvergeCursor(cursorPath, cursorState);
|
|
1128
|
+
} catch {
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
417
1131
|
}
|
|
418
1132
|
function formatConvergeReport(plan) {
|
|
419
1133
|
const lines = [];
|
|
@@ -435,25 +1149,49 @@ function formatConvergeReport(plan) {
|
|
|
435
1149
|
}
|
|
436
1150
|
return lines.join("\n");
|
|
437
1151
|
}
|
|
1152
|
+
function formatConvergeApplyReport(result) {
|
|
1153
|
+
const lines = [];
|
|
1154
|
+
lines.push(`Convergence Execution Status: ${result.status.toUpperCase()}`);
|
|
1155
|
+
lines.push(`Converged: ${result.converged ? "YES" : "NO"}`);
|
|
1156
|
+
lines.push("");
|
|
1157
|
+
lines.push("Transfers Executed:");
|
|
1158
|
+
lines.push(` pulled: ${result.transfers.pulled}`);
|
|
1159
|
+
lines.push(` pushed: ${result.transfers.pushed}`);
|
|
1160
|
+
lines.push(` conflictsResolved: ${result.transfers.conflictsResolved}`);
|
|
1161
|
+
lines.push(` suppressed: ${result.transfers.suppressed}`);
|
|
1162
|
+
lines.push(` failed: ${result.transfers.failed}`);
|
|
1163
|
+
lines.push("");
|
|
1164
|
+
lines.push(formatConvergeReport(result.plan));
|
|
1165
|
+
return lines.join("\n");
|
|
1166
|
+
}
|
|
438
1167
|
async function cmdConverge(action, rest, json) {
|
|
439
1168
|
if (action === "help" || action === "--help" || action === "-h" || rest.includes("--help") || rest.includes("-h")) {
|
|
440
|
-
console.log(`Usage: remnic converge plan
|
|
1169
|
+
console.log(`Usage: remnic converge <plan|apply> [options]
|
|
1170
|
+
|
|
1171
|
+
Subcommands:
|
|
1172
|
+
plan Compute and display reconciliation plan
|
|
1173
|
+
apply Execute bidirectional converge transport (alias: transport, sync)
|
|
441
1174
|
|
|
442
1175
|
Options:
|
|
443
1176
|
--peer <url> Peer server URL (or --remote-url / --remote)
|
|
444
1177
|
--token <token> Bearer token or SecretRef for peer authentication
|
|
1178
|
+
--conflict-policy <policy>
|
|
1179
|
+
Conflict resolution policy (manual|newest-wins|keep-both)
|
|
1180
|
+
--dry-run Simulate transfers without mutating disk or remote peer
|
|
445
1181
|
--json Output detailed JSON plan report
|
|
446
1182
|
`);
|
|
447
1183
|
return;
|
|
448
1184
|
}
|
|
449
|
-
if (action !== "plan") {
|
|
450
|
-
process.stderr.write(`converge: unknown action "${action}". Use: plan [options].
|
|
1185
|
+
if (action !== "plan" && action !== "apply" && action !== "transport" && action !== "sync") {
|
|
1186
|
+
process.stderr.write(`converge: unknown action "${action}". Use: plan or apply [options].
|
|
451
1187
|
`);
|
|
452
1188
|
process.exitCode = 2;
|
|
453
1189
|
return;
|
|
454
1190
|
}
|
|
455
1191
|
let peerUrl;
|
|
456
1192
|
let peerToken;
|
|
1193
|
+
let dryRun = false;
|
|
1194
|
+
let conflictPolicy;
|
|
457
1195
|
for (let i = 0; i < rest.length; i += 1) {
|
|
458
1196
|
const arg = rest[i];
|
|
459
1197
|
if ((arg === "--peer" || arg === "--remote-url" || arg === "--remote") && rest[i + 1]) {
|
|
@@ -462,13 +1200,35 @@ Options:
|
|
|
462
1200
|
} else if (arg === "--token" && rest[i + 1]) {
|
|
463
1201
|
peerToken = rest[i + 1];
|
|
464
1202
|
i += 1;
|
|
1203
|
+
} else if (arg === "--dry-run") {
|
|
1204
|
+
dryRun = true;
|
|
1205
|
+
} else if (arg === "--conflict-policy" && rest[i + 1]) {
|
|
1206
|
+
const pol = rest[i + 1];
|
|
1207
|
+
if (pol === "manual" || pol === "newest-wins" || pol === "keep-both") {
|
|
1208
|
+
conflictPolicy = pol;
|
|
1209
|
+
}
|
|
1210
|
+
i += 1;
|
|
465
1211
|
}
|
|
466
1212
|
}
|
|
467
|
-
|
|
1213
|
+
if (action === "plan") {
|
|
1214
|
+
const plan = await computeConvergePlan({ peerUrl, peerToken, conflictPolicy });
|
|
1215
|
+
if (json) {
|
|
1216
|
+
console.log(JSON.stringify(plan, null, 2));
|
|
1217
|
+
} else {
|
|
1218
|
+
console.log(formatConvergeReport(plan));
|
|
1219
|
+
}
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
const result = await executeConvergeApply({
|
|
1223
|
+
peerUrl,
|
|
1224
|
+
peerToken,
|
|
1225
|
+
dryRun,
|
|
1226
|
+
conflictPolicy
|
|
1227
|
+
});
|
|
468
1228
|
if (json) {
|
|
469
|
-
console.log(JSON.stringify(
|
|
1229
|
+
console.log(JSON.stringify(result, null, 2));
|
|
470
1230
|
} else {
|
|
471
|
-
console.log(
|
|
1231
|
+
console.log(formatConvergeApplyReport(result));
|
|
472
1232
|
}
|
|
473
1233
|
}
|
|
474
1234
|
|
|
@@ -605,7 +1365,7 @@ function renderReplayResult(result, targetNamespace, format) {
|
|
|
605
1365
|
}
|
|
606
1366
|
|
|
607
1367
|
// src/quarantine-replay.ts
|
|
608
|
-
import * as
|
|
1368
|
+
import * as fs4 from "fs";
|
|
609
1369
|
import { EngramAccessService, Orchestrator as Orchestrator2, initLogger, parseConfig as parseConfig3, resolveRemnicConfigRecord as resolveRemnicConfigRecord2 } from "@remnic/core";
|
|
610
1370
|
import { WriteQuarantineStore } from "@remnic/core/write-quarantine.js";
|
|
611
1371
|
function valueFlag(args, flag) {
|
|
@@ -654,7 +1414,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
|
|
|
654
1414
|
let orchestrator;
|
|
655
1415
|
try {
|
|
656
1416
|
const configPath = resolveConfigPath2();
|
|
657
|
-
const raw =
|
|
1417
|
+
const raw = fs4.existsSync(configPath) ? JSON.parse(fs4.readFileSync(configPath, "utf8")) : {};
|
|
658
1418
|
const config = parseConfig3(resolveRemnicConfigRecord2(raw));
|
|
659
1419
|
orchestrator = new Orchestrator2(config);
|
|
660
1420
|
await orchestrator.initialize();
|
|
@@ -686,7 +1446,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
|
|
|
686
1446
|
}
|
|
687
1447
|
|
|
688
1448
|
// src/offline-impression-rotation.ts
|
|
689
|
-
import
|
|
1449
|
+
import fs5 from "fs";
|
|
690
1450
|
import { parseConfig as parseConfig4, resolveRemnicConfigRecord as resolveRemnicConfigRecord3, drainPendingImpressionsForOfflineSync } from "@remnic/core";
|
|
691
1451
|
import { LastRecallStore } from "@remnic/core/recall-state";
|
|
692
1452
|
function parseConfigQuietly(raw) {
|
|
@@ -721,7 +1481,7 @@ function pickOfflineConfigRecord(raw) {
|
|
|
721
1481
|
function resolveOfflineImpressionRotation(configPath) {
|
|
722
1482
|
let raw;
|
|
723
1483
|
try {
|
|
724
|
-
raw =
|
|
1484
|
+
raw = fs5.existsSync(configPath) ? JSON.parse(fs5.readFileSync(configPath, "utf8")) : {};
|
|
725
1485
|
} catch {
|
|
726
1486
|
throw new Error(
|
|
727
1487
|
`cannot read recall-impression rotation from ${configPath}: config file could not be read as JSON`
|
|
@@ -749,265 +1509,6 @@ async function drainOfflineSyncImpressions(memoryDir, rotation) {
|
|
|
749
1509
|
);
|
|
750
1510
|
}
|
|
751
1511
|
|
|
752
|
-
// src/offline-storage-io.ts
|
|
753
|
-
import { mkdtemp, readdir, lstat, rm } from "fs/promises";
|
|
754
|
-
import fs5 from "fs";
|
|
755
|
-
import path2 from "path";
|
|
756
|
-
import { createHash, createDecipheriv } from "crypto";
|
|
757
|
-
import {
|
|
758
|
-
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
759
|
-
StorageManager
|
|
760
|
-
} from "@remnic/core";
|
|
761
|
-
import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
|
|
762
|
-
import {
|
|
763
|
-
AUTH_TAG_LENGTH,
|
|
764
|
-
ENVELOPE_HEADER_SIZE,
|
|
765
|
-
ENVELOPE_LAYOUT,
|
|
766
|
-
ENVELOPE_SALT_LENGTH,
|
|
767
|
-
ENVELOPE_VERSION,
|
|
768
|
-
FILE_FORMAT_FLAGS,
|
|
769
|
-
FILE_FORMAT_VERSION,
|
|
770
|
-
IV_LENGTH,
|
|
771
|
-
MAGIC_BYTES,
|
|
772
|
-
MAGIC_HEADER_SIZE,
|
|
773
|
-
SecureStoreLockedError,
|
|
774
|
-
filePathAad,
|
|
775
|
-
isEncryptedFile,
|
|
776
|
-
keyring,
|
|
777
|
-
readHeader,
|
|
778
|
-
secureStoreDir
|
|
779
|
-
} from "@remnic/core/secure-store";
|
|
780
|
-
async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
|
|
781
|
-
const storage = new StorageManager(memoryDir);
|
|
782
|
-
const header = await readHeader(memoryDir);
|
|
783
|
-
let secureStoreKey = null;
|
|
784
|
-
let secureStoreRequired = false;
|
|
785
|
-
if (header) {
|
|
786
|
-
secureStoreRequired = true;
|
|
787
|
-
storage.setSecureStoreRequired(true);
|
|
788
|
-
const key = keyring.getKey(secureStoreDir(memoryDir));
|
|
789
|
-
if (key) {
|
|
790
|
-
await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
|
|
791
|
-
secureStoreKey = key;
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
return { storage, secureStoreKey, secureStoreRequired };
|
|
795
|
-
}
|
|
796
|
-
async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
|
|
797
|
-
const memoryRoot = path2.resolve(memoryDir);
|
|
798
|
-
const stateDir = path2.dirname(filePath);
|
|
799
|
-
if (path2.basename(stateDir) !== "state" || path2.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
|
|
800
|
-
throw new Error(`invalid lifecycle ledger path: ${filePath}`);
|
|
801
|
-
}
|
|
802
|
-
const storageRoot = path2.resolve(path2.dirname(stateDir));
|
|
803
|
-
if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path2.sep}`)) {
|
|
804
|
-
throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
|
|
805
|
-
}
|
|
806
|
-
const storage = new StorageManager(storageRoot);
|
|
807
|
-
if (configured.secureStoreRequired) {
|
|
808
|
-
storage.setSecureStoreRequired(true);
|
|
809
|
-
}
|
|
810
|
-
if (configured.secureStoreKey) {
|
|
811
|
-
await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
|
|
812
|
-
}
|
|
813
|
-
return storage;
|
|
814
|
-
}
|
|
815
|
-
async function createOfflineStorageIo(memoryDir, configuredStorage) {
|
|
816
|
-
await cleanupOrphanedOfflineDecryptStaging(memoryDir);
|
|
817
|
-
const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
|
|
818
|
-
return {
|
|
819
|
-
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
820
|
-
readFileDigest: async ({ filePath }) => {
|
|
821
|
-
const hash = createHash("sha256");
|
|
822
|
-
let bytes = 0;
|
|
823
|
-
for await (const rawChunk of readOfflineSyncFileChunks({
|
|
824
|
-
filePath,
|
|
825
|
-
memoryDir,
|
|
826
|
-
secureStoreKey,
|
|
827
|
-
chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
|
|
828
|
-
})) {
|
|
829
|
-
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
|
|
830
|
-
hash.update(chunk);
|
|
831
|
-
bytes += chunk.length;
|
|
832
|
-
}
|
|
833
|
-
return {
|
|
834
|
-
sha256: hash.digest("hex"),
|
|
835
|
-
bytes
|
|
836
|
-
};
|
|
837
|
-
},
|
|
838
|
-
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
839
|
-
filePath,
|
|
840
|
-
memoryDir,
|
|
841
|
-
secureStoreKey,
|
|
842
|
-
chunkSize
|
|
843
|
-
}),
|
|
844
|
-
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
845
|
-
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
846
|
-
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
847
|
-
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
848
|
-
};
|
|
849
|
-
}
|
|
850
|
-
var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
|
|
851
|
-
async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
|
|
852
|
-
let entries;
|
|
853
|
-
try {
|
|
854
|
-
entries = await readdir(memoryDir);
|
|
855
|
-
} catch {
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
const now = Date.now();
|
|
859
|
-
for (const name of entries) {
|
|
860
|
-
if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
|
|
861
|
-
const dir = path2.join(memoryDir, name);
|
|
862
|
-
try {
|
|
863
|
-
const info = await lstat(dir);
|
|
864
|
-
if (!info.isDirectory() || info.isSymbolicLink()) continue;
|
|
865
|
-
if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
|
|
866
|
-
await rm(dir, { recursive: true, force: true });
|
|
867
|
-
} catch {
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
async function* readOfflineSyncFileChunks(options) {
|
|
872
|
-
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
|
|
873
|
-
if (!isEncryptedFile(header)) {
|
|
874
|
-
yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
|
|
875
|
-
return;
|
|
876
|
-
}
|
|
877
|
-
if (!options.secureStoreKey) {
|
|
878
|
-
throw new SecureStoreLockedError(
|
|
879
|
-
`secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
|
|
880
|
-
);
|
|
881
|
-
}
|
|
882
|
-
yield* readEncryptedOfflineFileChunks({
|
|
883
|
-
filePath: options.filePath,
|
|
884
|
-
memoryDir: options.memoryDir,
|
|
885
|
-
key: options.secureStoreKey,
|
|
886
|
-
chunkSize: options.chunkSize
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
async function readFilePrefix(filePath, length) {
|
|
890
|
-
const handle = await fs5.promises.open(filePath, "r");
|
|
891
|
-
try {
|
|
892
|
-
const out = Buffer.alloc(length);
|
|
893
|
-
const { bytesRead } = await handle.read(out, 0, length, 0);
|
|
894
|
-
return out.subarray(0, bytesRead);
|
|
895
|
-
} finally {
|
|
896
|
-
await handle.close();
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
async function* readPlainOfflineFileChunks(filePath, chunkSize) {
|
|
900
|
-
const stream = fs5.createReadStream(filePath, { highWaterMark: chunkSize });
|
|
901
|
-
for await (const chunk of stream) {
|
|
902
|
-
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
903
|
-
}
|
|
904
|
-
}
|
|
905
|
-
async function* readEncryptedOfflineFileChunks(options) {
|
|
906
|
-
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
907
|
-
if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
|
|
908
|
-
throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
|
|
909
|
-
}
|
|
910
|
-
const version = header.readUInt8(MAGIC_BYTES.length);
|
|
911
|
-
const flags = header.readUInt8(MAGIC_BYTES.length + 1);
|
|
912
|
-
if (version !== FILE_FORMAT_VERSION) {
|
|
913
|
-
throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
|
|
914
|
-
}
|
|
915
|
-
if (flags !== FILE_FORMAT_FLAGS) {
|
|
916
|
-
throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
|
|
917
|
-
}
|
|
918
|
-
const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
|
|
919
|
-
const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
|
|
920
|
-
if (envelopeVersion !== ENVELOPE_VERSION) {
|
|
921
|
-
throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
|
|
922
|
-
}
|
|
923
|
-
const salt = envelopeHeader.subarray(
|
|
924
|
-
ENVELOPE_LAYOUT.salt,
|
|
925
|
-
ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
|
|
926
|
-
);
|
|
927
|
-
const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
|
|
928
|
-
const authTag = envelopeHeader.subarray(
|
|
929
|
-
ENVELOPE_LAYOUT.authTag,
|
|
930
|
-
ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
|
|
931
|
-
);
|
|
932
|
-
const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
|
|
933
|
-
let lastError;
|
|
934
|
-
for (const aad of aadCandidates) {
|
|
935
|
-
const tempDir = await mkdtemp(path2.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
|
|
936
|
-
const tempPath = path2.join(tempDir, "content");
|
|
937
|
-
try {
|
|
938
|
-
const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
|
|
939
|
-
authTagLength: AUTH_TAG_LENGTH
|
|
940
|
-
});
|
|
941
|
-
decipher.setAuthTag(authTag);
|
|
942
|
-
decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
|
|
943
|
-
const output = fs5.createWriteStream(tempPath, { mode: 384 });
|
|
944
|
-
try {
|
|
945
|
-
const stream = fs5.createReadStream(options.filePath, {
|
|
946
|
-
start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
|
|
947
|
-
highWaterMark: options.chunkSize
|
|
948
|
-
});
|
|
949
|
-
for await (const encryptedChunk of stream) {
|
|
950
|
-
const plain = decipher.update(
|
|
951
|
-
Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
|
|
952
|
-
);
|
|
953
|
-
if (plain.length > 0 && !output.write(plain)) {
|
|
954
|
-
await new Promise((resolve, reject) => {
|
|
955
|
-
output.once("drain", resolve);
|
|
956
|
-
output.once("error", reject);
|
|
957
|
-
});
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
const finalPlain = decipher.final();
|
|
961
|
-
if (finalPlain.length > 0 && !output.write(finalPlain)) {
|
|
962
|
-
await new Promise((resolve, reject) => {
|
|
963
|
-
output.once("drain", resolve);
|
|
964
|
-
output.once("error", reject);
|
|
965
|
-
});
|
|
966
|
-
}
|
|
967
|
-
} finally {
|
|
968
|
-
await closeWriteStream(output);
|
|
969
|
-
}
|
|
970
|
-
yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
|
|
971
|
-
return;
|
|
972
|
-
} catch (error) {
|
|
973
|
-
lastError = error;
|
|
974
|
-
} finally {
|
|
975
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
|
|
979
|
-
}
|
|
980
|
-
function offlineFileAadCandidates(filePath, memoryDir) {
|
|
981
|
-
const candidates = [filePathAad(filePath, memoryDir)];
|
|
982
|
-
const relative = path2.relative(memoryDir, filePath);
|
|
983
|
-
if (!relative || relative.startsWith("..") || path2.isAbsolute(relative)) return candidates;
|
|
984
|
-
const parts = relative.split(path2.sep);
|
|
985
|
-
if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
|
|
986
|
-
candidates.push(filePathAad(filePath, path2.join(memoryDir, "namespaces", parts[1])));
|
|
987
|
-
}
|
|
988
|
-
const memoryParts = path2.resolve(memoryDir).split(path2.sep);
|
|
989
|
-
if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
|
|
990
|
-
const topLevelRoot = memoryParts.slice(0, -2).join(path2.sep) || path2.sep;
|
|
991
|
-
const topRelative = path2.relative(topLevelRoot, filePath);
|
|
992
|
-
if (topRelative && !topRelative.startsWith("..") && !path2.isAbsolute(topRelative) && topRelative.split(path2.sep)[0] === "namespaces" && topRelative.split(path2.sep)[1] === memoryParts.at(-1)) {
|
|
993
|
-
candidates.push(filePathAad(filePath, topLevelRoot));
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
return candidates;
|
|
997
|
-
}
|
|
998
|
-
async function closeWriteStream(stream) {
|
|
999
|
-
await new Promise((resolve, reject) => {
|
|
1000
|
-
stream.once("error", reject);
|
|
1001
|
-
stream.end(() => resolve());
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1004
|
-
function secureStoreEnvelopeHeaderAad(salt) {
|
|
1005
|
-
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
1006
|
-
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
1007
|
-
Buffer.from(salt).copy(out, 1);
|
|
1008
|
-
return out;
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
1512
|
// src/bench-build-freshness.ts
|
|
1012
1513
|
import {
|
|
1013
1514
|
existsSync as existsSync2,
|
|
@@ -5892,7 +6393,7 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
5892
6393
|
);
|
|
5893
6394
|
process.exit(1);
|
|
5894
6395
|
}
|
|
5895
|
-
const sourceResultSha256 =
|
|
6396
|
+
const sourceResultSha256 = createHash3("sha256").update(fs12.readFileSync(latest.path)).digest("hex");
|
|
5896
6397
|
const expandedManifestPath = expandTilde(manifestPath);
|
|
5897
6398
|
if (!bench.resolveLocalLabJudgeProviderConfig) {
|
|
5898
6399
|
console.error(
|
|
@@ -6756,7 +7257,7 @@ function attachPreparedJudgeCalibration(result, judgeCalibration) {
|
|
|
6756
7257
|
function hashCalibrationProviderConfig(config) {
|
|
6757
7258
|
const canonicalize = (value, key = "") => {
|
|
6758
7259
|
if (typeof value === "string" && /(?:api.?key|authorization|token|secret)/i.test(key)) {
|
|
6759
|
-
return { secretSha256:
|
|
7260
|
+
return { secretSha256: createHash3("sha256").update(value).digest("hex") };
|
|
6760
7261
|
}
|
|
6761
7262
|
if (Array.isArray(value)) return value.map((item) => canonicalize(item));
|
|
6762
7263
|
if (value && typeof value === "object") {
|
|
@@ -6767,7 +7268,7 @@ function hashCalibrationProviderConfig(config) {
|
|
|
6767
7268
|
}
|
|
6768
7269
|
return value;
|
|
6769
7270
|
};
|
|
6770
|
-
return
|
|
7271
|
+
return createHash3("sha256").update(JSON.stringify(canonicalize(config))).digest("hex");
|
|
6771
7272
|
}
|
|
6772
7273
|
function restoreOptionalEnv(key, previousValue) {
|
|
6773
7274
|
if (previousValue === void 0) {
|
|
@@ -9199,7 +9700,7 @@ async function cmdSync(action, rest, json) {
|
|
|
9199
9700
|
}
|
|
9200
9701
|
function localOfflineSourceId(memoryDir) {
|
|
9201
9702
|
const host = os.hostname() || "unknown-host";
|
|
9202
|
-
const dirHash =
|
|
9703
|
+
const dirHash = createHash3("sha256").update(path15.resolve(memoryDir)).digest("hex").slice(0, 16);
|
|
9203
9704
|
return `remnic-local:${host}:${dirHash}`;
|
|
9204
9705
|
}
|
|
9205
9706
|
function normalizeOfflineRemoteUrl(raw) {
|
|
@@ -9799,7 +10300,7 @@ function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
|
|
|
9799
10300
|
}
|
|
9800
10301
|
return target;
|
|
9801
10302
|
}
|
|
9802
|
-
var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES =
|
|
10303
|
+
var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3;
|
|
9803
10304
|
async function pushOfflineFileContent(args) {
|
|
9804
10305
|
if (args.readFileChunks) {
|
|
9805
10306
|
return pushOfflineFileContentFromChunkReader(args);
|
|
@@ -9807,7 +10308,7 @@ async function pushOfflineFileContent(args) {
|
|
|
9807
10308
|
let offset = 0;
|
|
9808
10309
|
let finalResult = null;
|
|
9809
10310
|
let remoteSatisfiedResult = null;
|
|
9810
|
-
const hash =
|
|
10311
|
+
const hash = createHash3("sha256");
|
|
9811
10312
|
let bytes = 0;
|
|
9812
10313
|
while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
|
|
9813
10314
|
const chunk = await readOfflineSyncFileContentChunk({
|
|
@@ -9868,7 +10369,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
9868
10369
|
if (stat.mtimeMs !== args.file.mtimeMs) {
|
|
9869
10370
|
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
9870
10371
|
}
|
|
9871
|
-
const hash =
|
|
10372
|
+
const hash = createHash3("sha256");
|
|
9872
10373
|
const chunks = args.readFileChunks({
|
|
9873
10374
|
root: path15.resolve(args.memoryDir),
|
|
9874
10375
|
path: args.file.path,
|
|
@@ -9882,7 +10383,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
9882
10383
|
for await (const rawChunk of chunks) {
|
|
9883
10384
|
const chunk = Buffer.from(rawChunk);
|
|
9884
10385
|
if (chunk.length === 0) continue;
|
|
9885
|
-
if (chunk.length >
|
|
10386
|
+
if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2) {
|
|
9886
10387
|
throw new Error(`local offline content chunk exceeds max size: ${args.file.path}`);
|
|
9887
10388
|
}
|
|
9888
10389
|
if (pending) {
|
|
@@ -9967,7 +10468,7 @@ async function hydrateOfflineFileContent(args) {
|
|
|
9967
10468
|
path: args.expected.path,
|
|
9968
10469
|
offset,
|
|
9969
10470
|
length: Math.min(
|
|
9970
|
-
|
|
10471
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
|
|
9971
10472
|
Math.max(1, args.expected.bytes - offset)
|
|
9972
10473
|
)
|
|
9973
10474
|
});
|
|
@@ -9978,7 +10479,7 @@ async function hydrateOfflineFileContent(args) {
|
|
|
9978
10479
|
throw new Error(`remote offline content chunk was empty before EOF: ${args.expected.path}`);
|
|
9979
10480
|
}
|
|
9980
10481
|
try {
|
|
9981
|
-
finalResult = await
|
|
10482
|
+
finalResult = await applyOfflineSyncFileContentChunk2({
|
|
9982
10483
|
root: args.memoryDir,
|
|
9983
10484
|
sourceId: args.sourceId,
|
|
9984
10485
|
path: args.expected.path,
|