@remnic/cli 9.42.0 → 9.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +859 -313
- package/package.json +29 -29
package/dist/index.js
CHANGED
|
@@ -21,7 +21,7 @@ async function persistEnrichmentCandidate(storage, entityName, candidate) {
|
|
|
21
21
|
import fs12 from "fs";
|
|
22
22
|
import os from "os";
|
|
23
23
|
import path15 from "path";
|
|
24
|
-
import { createHash as
|
|
24
|
+
import { createHash as createHash3 } from "crypto";
|
|
25
25
|
import * as childProcess2 from "child_process";
|
|
26
26
|
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
27
27
|
import { gzipSync } from "zlib";
|
|
@@ -110,10 +110,10 @@ import {
|
|
|
110
110
|
parseXrayCliOptions,
|
|
111
111
|
renderXray,
|
|
112
112
|
OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
|
|
113
|
-
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
114
|
-
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as
|
|
113
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
|
|
114
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3,
|
|
115
115
|
OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
|
|
116
|
-
applyOfflineSyncFileContentChunk,
|
|
116
|
+
applyOfflineSyncFileContentChunk as applyOfflineSyncFileContentChunk2,
|
|
117
117
|
applyOfflineSyncSnapshot,
|
|
118
118
|
buildOfflineSyncChangesetFromSnapshot,
|
|
119
119
|
drainPendingLifecycleForOfflineSync,
|
|
@@ -239,27 +239,297 @@ async function loadWecloneExportModule() {
|
|
|
239
239
|
}
|
|
240
240
|
|
|
241
241
|
// src/converge.ts
|
|
242
|
-
import * as
|
|
243
|
-
import
|
|
242
|
+
import * as fs3 from "fs";
|
|
243
|
+
import { createHash as createHash2 } from "crypto";
|
|
244
|
+
import * as path2 from "path";
|
|
244
245
|
import {
|
|
245
246
|
parseConfig as parseConfig2,
|
|
246
|
-
buildOfflineSyncSnapshotFromBase
|
|
247
|
+
buildOfflineSyncSnapshotFromBase,
|
|
248
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
|
|
249
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2,
|
|
250
|
+
applyOfflineSyncFileContentChunk
|
|
247
251
|
} from "@remnic/core";
|
|
248
252
|
import { resolveCorpusNamespaceRoots } from "@remnic/core/corpus-watermark.js";
|
|
249
253
|
import { listNamespaces } from "@remnic/core/namespaces/migrate.js";
|
|
250
254
|
import {
|
|
251
255
|
planReconciliation
|
|
252
256
|
} from "@remnic/core/reconcile/plan.js";
|
|
257
|
+
import {
|
|
258
|
+
defaultConvergeCursorPath,
|
|
259
|
+
readConvergeCursor,
|
|
260
|
+
writeConvergeCursor
|
|
261
|
+
} from "@remnic/core/reconcile/cursor.js";
|
|
262
|
+
|
|
263
|
+
// src/offline-storage-io.ts
|
|
264
|
+
import { mkdtemp, readdir, lstat, rm } from "fs/promises";
|
|
265
|
+
import fs2 from "fs";
|
|
266
|
+
import path from "path";
|
|
267
|
+
import { createHash, createDecipheriv } from "crypto";
|
|
268
|
+
import {
|
|
269
|
+
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
270
|
+
StorageManager
|
|
271
|
+
} from "@remnic/core";
|
|
272
|
+
import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
|
|
273
|
+
import {
|
|
274
|
+
AUTH_TAG_LENGTH,
|
|
275
|
+
ENVELOPE_HEADER_SIZE,
|
|
276
|
+
ENVELOPE_LAYOUT,
|
|
277
|
+
ENVELOPE_SALT_LENGTH,
|
|
278
|
+
ENVELOPE_VERSION,
|
|
279
|
+
FILE_FORMAT_FLAGS,
|
|
280
|
+
FILE_FORMAT_VERSION,
|
|
281
|
+
IV_LENGTH,
|
|
282
|
+
MAGIC_BYTES,
|
|
283
|
+
MAGIC_HEADER_SIZE,
|
|
284
|
+
SecureStoreLockedError,
|
|
285
|
+
filePathAad,
|
|
286
|
+
isEncryptedFile,
|
|
287
|
+
keyring,
|
|
288
|
+
readHeader,
|
|
289
|
+
secureStoreDir
|
|
290
|
+
} from "@remnic/core/secure-store";
|
|
291
|
+
async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
|
|
292
|
+
const storage = new StorageManager(memoryDir);
|
|
293
|
+
const header = await readHeader(memoryDir);
|
|
294
|
+
let secureStoreKey = null;
|
|
295
|
+
let secureStoreRequired = false;
|
|
296
|
+
if (header) {
|
|
297
|
+
secureStoreRequired = true;
|
|
298
|
+
storage.setSecureStoreRequired(true);
|
|
299
|
+
const key = keyring.getKey(secureStoreDir(memoryDir));
|
|
300
|
+
if (key) {
|
|
301
|
+
await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
|
|
302
|
+
secureStoreKey = key;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return { storage, secureStoreKey, secureStoreRequired };
|
|
306
|
+
}
|
|
307
|
+
async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
|
|
308
|
+
const memoryRoot = path.resolve(memoryDir);
|
|
309
|
+
const stateDir = path.dirname(filePath);
|
|
310
|
+
if (path.basename(stateDir) !== "state" || path.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
|
|
311
|
+
throw new Error(`invalid lifecycle ledger path: ${filePath}`);
|
|
312
|
+
}
|
|
313
|
+
const storageRoot = path.resolve(path.dirname(stateDir));
|
|
314
|
+
if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path.sep}`)) {
|
|
315
|
+
throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
|
|
316
|
+
}
|
|
317
|
+
const storage = new StorageManager(storageRoot);
|
|
318
|
+
if (configured.secureStoreRequired) {
|
|
319
|
+
storage.setSecureStoreRequired(true);
|
|
320
|
+
}
|
|
321
|
+
if (configured.secureStoreKey) {
|
|
322
|
+
await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
|
|
323
|
+
}
|
|
324
|
+
return storage;
|
|
325
|
+
}
|
|
326
|
+
async function createOfflineStorageIo(memoryDir, configuredStorage) {
|
|
327
|
+
await cleanupOrphanedOfflineDecryptStaging(memoryDir);
|
|
328
|
+
const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
|
|
329
|
+
return {
|
|
330
|
+
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
331
|
+
readFileDigest: async ({ filePath }) => {
|
|
332
|
+
const hash = createHash("sha256");
|
|
333
|
+
let bytes = 0;
|
|
334
|
+
for await (const rawChunk of readOfflineSyncFileChunks({
|
|
335
|
+
filePath,
|
|
336
|
+
memoryDir,
|
|
337
|
+
secureStoreKey,
|
|
338
|
+
chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
|
|
339
|
+
})) {
|
|
340
|
+
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
|
|
341
|
+
hash.update(chunk);
|
|
342
|
+
bytes += chunk.length;
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
sha256: hash.digest("hex"),
|
|
346
|
+
bytes
|
|
347
|
+
};
|
|
348
|
+
},
|
|
349
|
+
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
350
|
+
filePath,
|
|
351
|
+
memoryDir,
|
|
352
|
+
secureStoreKey,
|
|
353
|
+
chunkSize
|
|
354
|
+
}),
|
|
355
|
+
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
356
|
+
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
357
|
+
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
358
|
+
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
|
|
362
|
+
async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
|
|
363
|
+
let entries;
|
|
364
|
+
try {
|
|
365
|
+
entries = await readdir(memoryDir);
|
|
366
|
+
} catch {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
const now = Date.now();
|
|
370
|
+
for (const name of entries) {
|
|
371
|
+
if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
|
|
372
|
+
const dir = path.join(memoryDir, name);
|
|
373
|
+
try {
|
|
374
|
+
const info = await lstat(dir);
|
|
375
|
+
if (!info.isDirectory() || info.isSymbolicLink()) continue;
|
|
376
|
+
if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
|
|
377
|
+
await rm(dir, { recursive: true, force: true });
|
|
378
|
+
} catch {
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
async function* readOfflineSyncFileChunks(options) {
|
|
383
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
|
|
384
|
+
if (!isEncryptedFile(header)) {
|
|
385
|
+
yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (!options.secureStoreKey) {
|
|
389
|
+
throw new SecureStoreLockedError(
|
|
390
|
+
`secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
yield* readEncryptedOfflineFileChunks({
|
|
394
|
+
filePath: options.filePath,
|
|
395
|
+
memoryDir: options.memoryDir,
|
|
396
|
+
key: options.secureStoreKey,
|
|
397
|
+
chunkSize: options.chunkSize
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
async function readFilePrefix(filePath, length) {
|
|
401
|
+
const handle = await fs2.promises.open(filePath, "r");
|
|
402
|
+
try {
|
|
403
|
+
const out = Buffer.alloc(length);
|
|
404
|
+
const { bytesRead } = await handle.read(out, 0, length, 0);
|
|
405
|
+
return out.subarray(0, bytesRead);
|
|
406
|
+
} finally {
|
|
407
|
+
await handle.close();
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
async function* readPlainOfflineFileChunks(filePath, chunkSize) {
|
|
411
|
+
const stream = fs2.createReadStream(filePath, { highWaterMark: chunkSize });
|
|
412
|
+
for await (const chunk of stream) {
|
|
413
|
+
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
async function* readEncryptedOfflineFileChunks(options) {
|
|
417
|
+
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
418
|
+
if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
|
|
419
|
+
throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
|
|
420
|
+
}
|
|
421
|
+
const version = header.readUInt8(MAGIC_BYTES.length);
|
|
422
|
+
const flags = header.readUInt8(MAGIC_BYTES.length + 1);
|
|
423
|
+
if (version !== FILE_FORMAT_VERSION) {
|
|
424
|
+
throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
|
|
425
|
+
}
|
|
426
|
+
if (flags !== FILE_FORMAT_FLAGS) {
|
|
427
|
+
throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
|
|
428
|
+
}
|
|
429
|
+
const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
|
|
430
|
+
const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
|
|
431
|
+
if (envelopeVersion !== ENVELOPE_VERSION) {
|
|
432
|
+
throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
|
|
433
|
+
}
|
|
434
|
+
const salt = envelopeHeader.subarray(
|
|
435
|
+
ENVELOPE_LAYOUT.salt,
|
|
436
|
+
ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
|
|
437
|
+
);
|
|
438
|
+
const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
|
|
439
|
+
const authTag = envelopeHeader.subarray(
|
|
440
|
+
ENVELOPE_LAYOUT.authTag,
|
|
441
|
+
ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
|
|
442
|
+
);
|
|
443
|
+
const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
|
|
444
|
+
let lastError;
|
|
445
|
+
for (const aad of aadCandidates) {
|
|
446
|
+
const tempDir = await mkdtemp(path.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
|
|
447
|
+
const tempPath = path.join(tempDir, "content");
|
|
448
|
+
try {
|
|
449
|
+
const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
|
|
450
|
+
authTagLength: AUTH_TAG_LENGTH
|
|
451
|
+
});
|
|
452
|
+
decipher.setAuthTag(authTag);
|
|
453
|
+
decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
|
|
454
|
+
const output = fs2.createWriteStream(tempPath, { mode: 384 });
|
|
455
|
+
try {
|
|
456
|
+
const stream = fs2.createReadStream(options.filePath, {
|
|
457
|
+
start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
|
|
458
|
+
highWaterMark: options.chunkSize
|
|
459
|
+
});
|
|
460
|
+
for await (const encryptedChunk of stream) {
|
|
461
|
+
const plain = decipher.update(
|
|
462
|
+
Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
|
|
463
|
+
);
|
|
464
|
+
if (plain.length > 0 && !output.write(plain)) {
|
|
465
|
+
await new Promise((resolve, reject) => {
|
|
466
|
+
output.once("drain", resolve);
|
|
467
|
+
output.once("error", reject);
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
const finalPlain = decipher.final();
|
|
472
|
+
if (finalPlain.length > 0 && !output.write(finalPlain)) {
|
|
473
|
+
await new Promise((resolve, reject) => {
|
|
474
|
+
output.once("drain", resolve);
|
|
475
|
+
output.once("error", reject);
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
} finally {
|
|
479
|
+
await closeWriteStream(output);
|
|
480
|
+
}
|
|
481
|
+
yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
|
|
482
|
+
return;
|
|
483
|
+
} catch (error) {
|
|
484
|
+
lastError = error;
|
|
485
|
+
} finally {
|
|
486
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
|
|
490
|
+
}
|
|
491
|
+
function offlineFileAadCandidates(filePath, memoryDir) {
|
|
492
|
+
const candidates = [filePathAad(filePath, memoryDir)];
|
|
493
|
+
const relative = path.relative(memoryDir, filePath);
|
|
494
|
+
if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return candidates;
|
|
495
|
+
const parts = relative.split(path.sep);
|
|
496
|
+
if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
|
|
497
|
+
candidates.push(filePathAad(filePath, path.join(memoryDir, "namespaces", parts[1])));
|
|
498
|
+
}
|
|
499
|
+
const memoryParts = path.resolve(memoryDir).split(path.sep);
|
|
500
|
+
if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
|
|
501
|
+
const topLevelRoot = memoryParts.slice(0, -2).join(path.sep) || path.sep;
|
|
502
|
+
const topRelative = path.relative(topLevelRoot, filePath);
|
|
503
|
+
if (topRelative && !topRelative.startsWith("..") && !path.isAbsolute(topRelative) && topRelative.split(path.sep)[0] === "namespaces" && topRelative.split(path.sep)[1] === memoryParts.at(-1)) {
|
|
504
|
+
candidates.push(filePathAad(filePath, topLevelRoot));
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return candidates;
|
|
508
|
+
}
|
|
509
|
+
async function closeWriteStream(stream) {
|
|
510
|
+
await new Promise((resolve, reject) => {
|
|
511
|
+
stream.once("error", reject);
|
|
512
|
+
stream.end(() => resolve());
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
function secureStoreEnvelopeHeaderAad(salt) {
|
|
516
|
+
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
517
|
+
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
518
|
+
Buffer.from(salt).copy(out, 1);
|
|
519
|
+
return out;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// src/converge.ts
|
|
253
523
|
import { resolveAgentAccessAuthToken } from "@remnic/core/resolve-auth-token.js";
|
|
254
524
|
async function readLocalTombstones(rootDir) {
|
|
255
525
|
const shaSet = /* @__PURE__ */ new Set();
|
|
256
526
|
const candidates = [
|
|
257
|
-
|
|
258
|
-
|
|
527
|
+
path2.join(rootDir, "state", "tombstones.jsonl"),
|
|
528
|
+
path2.join(rootDir, "tombstones.jsonl")
|
|
259
529
|
];
|
|
260
530
|
for (const tombPath of candidates) {
|
|
261
531
|
try {
|
|
262
|
-
const content = await
|
|
532
|
+
const content = await fs3.promises.readFile(tombPath, "utf-8");
|
|
263
533
|
for (const line of content.split("\n")) {
|
|
264
534
|
const trimmed = line.trim();
|
|
265
535
|
if (!trimmed) continue;
|
|
@@ -289,44 +559,222 @@ async function fetchPeerSnapshot(peerUrl, namespace, token, fetchImpl = globalTh
|
|
|
289
559
|
`/engram/v1/offline-sync/snapshot?namespace=${encodeURIComponent(namespace)}&content=false`
|
|
290
560
|
];
|
|
291
561
|
const headers = token ? { authorization: `Bearer ${token}` } : {};
|
|
562
|
+
let lastFailure = "no snapshot route responded";
|
|
292
563
|
for (const route of routes) {
|
|
564
|
+
let response;
|
|
293
565
|
try {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
566
|
+
response = await fetchImpl(`${base}${route}`, { headers });
|
|
567
|
+
} catch (error) {
|
|
568
|
+
lastFailure = error instanceof Error ? error.message : String(error);
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
if (!response.ok) {
|
|
572
|
+
lastFailure = `HTTP ${response.status}`;
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
let data;
|
|
576
|
+
try {
|
|
577
|
+
data = await response.json();
|
|
578
|
+
} catch {
|
|
579
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: response was not JSON`);
|
|
580
|
+
}
|
|
581
|
+
if (!data || typeof data !== "object" || !("files" in data) || !Array.isArray(data.files)) {
|
|
582
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: files must be an array`);
|
|
583
|
+
}
|
|
584
|
+
const files = data.files.map((item, index) => {
|
|
585
|
+
if (!item || typeof item !== "object" || !("path" in item) || typeof item.path !== "string" || !("sha256" in item) || typeof item.sha256 !== "string") {
|
|
586
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed file at index ${index}`);
|
|
309
587
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
588
|
+
return {
|
|
589
|
+
path: item.path,
|
|
590
|
+
sha256: item.sha256,
|
|
591
|
+
mtimeMs: "mtimeMs" in item && typeof item.mtimeMs === "number" ? item.mtimeMs : void 0,
|
|
592
|
+
bytes: "bytes" in item && typeof item.bytes === "number" ? item.bytes : void 0
|
|
593
|
+
};
|
|
594
|
+
});
|
|
595
|
+
const rawTombstones = "tombstones" in data ? data.tombstones : void 0;
|
|
596
|
+
if (rawTombstones !== void 0 && !Array.isArray(rawTombstones)) {
|
|
597
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: tombstones must be an array`);
|
|
598
|
+
}
|
|
599
|
+
const tombstones = /* @__PURE__ */ new Set();
|
|
600
|
+
for (const tombstone of rawTombstones ?? []) {
|
|
601
|
+
if (typeof tombstone !== "string" || !/^[0-9a-f]{64}$/i.test(tombstone)) {
|
|
602
|
+
throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed tombstone`);
|
|
603
|
+
}
|
|
604
|
+
tombstones.add(tombstone.toLowerCase());
|
|
605
|
+
}
|
|
606
|
+
return { files, tombstones };
|
|
607
|
+
}
|
|
608
|
+
throw new Error(`failed to fetch peer snapshot for namespace ${namespace}: ${lastFailure}`);
|
|
609
|
+
}
|
|
610
|
+
function requiredResponseNumber(response, name) {
|
|
611
|
+
const raw = response.headers.get(name);
|
|
612
|
+
const value = raw === null ? Number.NaN : Number(raw);
|
|
613
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
614
|
+
throw new Error(`offline file content response had invalid ${name}`);
|
|
615
|
+
}
|
|
616
|
+
return value;
|
|
617
|
+
}
|
|
618
|
+
async function fetchPeerFileContent(peerUrl, namespace, filePath, token, fetchImpl = globalThis.fetch) {
|
|
619
|
+
const base = peerUrl.replace(/\/+$/, "");
|
|
620
|
+
const routes = [
|
|
621
|
+
"/remnic/v1/offline-sync/file-content",
|
|
622
|
+
"/engram/v1/offline-sync/file-content"
|
|
623
|
+
];
|
|
624
|
+
const headers = {
|
|
625
|
+
"content-type": "application/json",
|
|
626
|
+
...token ? { authorization: `Bearer ${token}` } : {}
|
|
627
|
+
};
|
|
628
|
+
for (const route of routes) {
|
|
629
|
+
try {
|
|
630
|
+
const chunks = [];
|
|
631
|
+
const hash = createHash2("sha256");
|
|
632
|
+
let offset = 0;
|
|
633
|
+
let expectedBytes;
|
|
634
|
+
let expectedSha256;
|
|
635
|
+
let mtimeMs;
|
|
636
|
+
do {
|
|
637
|
+
const response = await fetchImpl(`${base}${route}`, {
|
|
638
|
+
method: "POST",
|
|
639
|
+
headers,
|
|
640
|
+
body: JSON.stringify({
|
|
641
|
+
namespace,
|
|
642
|
+
includeTranscripts: false,
|
|
643
|
+
path: filePath,
|
|
644
|
+
offset,
|
|
645
|
+
length: OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES
|
|
646
|
+
})
|
|
647
|
+
});
|
|
648
|
+
if (!response.ok) throw new Error(`offline file content request failed: ${response.status}`);
|
|
649
|
+
const content = Buffer.from(await response.arrayBuffer());
|
|
650
|
+
const chunkOffset = requiredResponseNumber(response, "x-remnic-chunk-offset");
|
|
651
|
+
const chunkBytes = requiredResponseNumber(response, "x-remnic-chunk-bytes");
|
|
652
|
+
const totalBytes = requiredResponseNumber(response, "x-remnic-file-bytes");
|
|
653
|
+
const responseMtimeMs = requiredResponseNumber(response, "x-remnic-file-mtime-ms");
|
|
654
|
+
const sha256 = response.headers.get("x-remnic-file-sha256");
|
|
655
|
+
const encodedPath = response.headers.get("x-remnic-file-path");
|
|
656
|
+
if (!sha256 || chunkOffset !== offset || chunkBytes !== content.length || encodedPath !== null && decodeURIComponent(encodedPath) !== filePath || expectedBytes !== void 0 && expectedBytes !== totalBytes || expectedSha256 !== void 0 && expectedSha256 !== sha256) {
|
|
657
|
+
throw new Error(`offline file content response changed during transfer: ${filePath}`);
|
|
658
|
+
}
|
|
659
|
+
if (content.length === 0 && offset < totalBytes) {
|
|
660
|
+
throw new Error(`offline file content chunk was empty before EOF: ${filePath}`);
|
|
316
661
|
}
|
|
662
|
+
expectedBytes = totalBytes;
|
|
663
|
+
expectedSha256 = sha256;
|
|
664
|
+
mtimeMs = responseMtimeMs;
|
|
665
|
+
chunks.push(content);
|
|
666
|
+
hash.update(content);
|
|
667
|
+
offset += content.length;
|
|
668
|
+
} while (expectedBytes === void 0 || offset < expectedBytes);
|
|
669
|
+
if (expectedBytes === void 0 || expectedSha256 === void 0 || mtimeMs === void 0 || offset !== expectedBytes || hash.digest("hex") !== expectedSha256) {
|
|
670
|
+
throw new Error(`offline file content checksum mismatch: ${filePath}`);
|
|
317
671
|
}
|
|
318
|
-
return {
|
|
672
|
+
return {
|
|
673
|
+
content: Buffer.concat(chunks, expectedBytes),
|
|
674
|
+
sha256: expectedSha256,
|
|
675
|
+
bytes: expectedBytes,
|
|
676
|
+
mtimeMs
|
|
677
|
+
};
|
|
678
|
+
} catch {
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
return null;
|
|
682
|
+
}
|
|
683
|
+
function withoutTrailingSlashes(value) {
|
|
684
|
+
let end = value.length;
|
|
685
|
+
while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
|
|
686
|
+
return value.slice(0, end);
|
|
687
|
+
}
|
|
688
|
+
async function postPeerFileContent(peerUrl, namespace, filePath, content, metadata, token, fetchImpl = globalThis.fetch) {
|
|
689
|
+
const base = withoutTrailingSlashes(peerUrl);
|
|
690
|
+
const routes = [
|
|
691
|
+
`/remnic/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`,
|
|
692
|
+
`/engram/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`
|
|
693
|
+
];
|
|
694
|
+
let previousAttemptFailed = false;
|
|
695
|
+
for (const route of routes) {
|
|
696
|
+
try {
|
|
697
|
+
let offset = 0;
|
|
698
|
+
do {
|
|
699
|
+
const chunk = content.subarray(
|
|
700
|
+
offset,
|
|
701
|
+
Math.min(content.length, offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2)
|
|
702
|
+
);
|
|
703
|
+
const headers = {
|
|
704
|
+
"content-type": "application/octet-stream",
|
|
705
|
+
"x-remnic-include-transcripts": "false",
|
|
706
|
+
"x-remnic-source-id": encodeURIComponent("remnic-converge"),
|
|
707
|
+
"x-remnic-file-path": encodeURIComponent(filePath),
|
|
708
|
+
"x-remnic-file-sha256": metadata.sha256,
|
|
709
|
+
"x-remnic-file-bytes": String(content.length),
|
|
710
|
+
"x-remnic-file-mtime-ms": String(metadata.mtimeMs),
|
|
711
|
+
"x-remnic-chunk-offset": String(offset),
|
|
712
|
+
...metadata.baseSha256 ? { "x-remnic-base-sha256": metadata.baseSha256 } : {},
|
|
713
|
+
...token ? { authorization: `Bearer ${token}` } : {}
|
|
714
|
+
};
|
|
715
|
+
const response = await fetchImpl(`${base}${route}`, {
|
|
716
|
+
method: "POST",
|
|
717
|
+
headers,
|
|
718
|
+
body: new Uint8Array(chunk)
|
|
719
|
+
});
|
|
720
|
+
if (!response.ok) throw new Error(`offline apply-file-content request failed: ${response.status}`);
|
|
721
|
+
const result = await response.json().catch(() => null);
|
|
722
|
+
if (!result || typeof result !== "object" || !("done" in result) || typeof result.done !== "boolean" || !("applied" in result) || typeof result.applied !== "boolean" || !("skipped" in result) || typeof result.skipped !== "boolean" || "conflict" in result && result.conflict) {
|
|
723
|
+
return false;
|
|
724
|
+
}
|
|
725
|
+
if (result.done) {
|
|
726
|
+
if (result.skipped) return previousAttemptFailed ? "applied" : "skipped";
|
|
727
|
+
if (result.applied && offset + chunk.length === content.length) return "applied";
|
|
728
|
+
return false;
|
|
729
|
+
}
|
|
730
|
+
if (result.applied || result.skipped || chunk.length === 0) {
|
|
731
|
+
return false;
|
|
732
|
+
}
|
|
733
|
+
offset += chunk.length;
|
|
734
|
+
} while (offset < content.length);
|
|
735
|
+
return false;
|
|
319
736
|
} catch {
|
|
737
|
+
previousAttemptFailed = true;
|
|
320
738
|
}
|
|
321
739
|
}
|
|
322
|
-
return
|
|
740
|
+
return false;
|
|
741
|
+
}
|
|
742
|
+
async function postPeerConvergenceComplete(peerUrl, namespaces, token, fetchImpl = globalThis.fetch) {
|
|
743
|
+
const base = withoutTrailingSlashes(peerUrl);
|
|
744
|
+
const query = namespaces.map((namespace) => `namespace=${encodeURIComponent(namespace)}`).join("&");
|
|
745
|
+
const routes = [
|
|
746
|
+
"/remnic/v1/offline-sync/convergence-complete",
|
|
747
|
+
"/engram/v1/offline-sync/convergence-complete"
|
|
748
|
+
];
|
|
749
|
+
for (const route of routes) {
|
|
750
|
+
const response = await fetchImpl(`${base}${route}?${query}`, {
|
|
751
|
+
method: "POST",
|
|
752
|
+
headers: {
|
|
753
|
+
"x-remnic-source-id": encodeURIComponent("remnic-converge"),
|
|
754
|
+
...token ? { authorization: `Bearer ${token}` } : {}
|
|
755
|
+
}
|
|
756
|
+
}).catch(() => null);
|
|
757
|
+
if (!response?.ok) continue;
|
|
758
|
+
const result = await response.json().catch(() => null);
|
|
759
|
+
if (result && typeof result === "object" && "namespaces" in result && Array.isArray(result.namespaces) && result.namespaces.length === namespaces.length && result.namespaces.every((namespace, index) => namespace === namespaces[index]) && "refreshed" in result && result.refreshed === true) {
|
|
760
|
+
return true;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
return false;
|
|
323
764
|
}
|
|
324
765
|
async function computeConvergePlan(options = {}) {
|
|
766
|
+
const baseMap = /* @__PURE__ */ new Map();
|
|
325
767
|
const namespacesToPlan = /* @__PURE__ */ new Set();
|
|
326
768
|
const localMap = /* @__PURE__ */ new Map();
|
|
327
769
|
const localTombstones = /* @__PURE__ */ new Map();
|
|
328
770
|
const peerMap = /* @__PURE__ */ new Map();
|
|
329
771
|
const peerTombstones = /* @__PURE__ */ new Map();
|
|
772
|
+
if (options.baseFilesByNamespace) {
|
|
773
|
+
for (const [ns, files] of options.baseFilesByNamespace) {
|
|
774
|
+
namespacesToPlan.add(ns);
|
|
775
|
+
baseMap.set(ns, files);
|
|
776
|
+
}
|
|
777
|
+
}
|
|
330
778
|
if (options.localFilesByNamespace) {
|
|
331
779
|
for (const [ns, files] of options.localFilesByNamespace) {
|
|
332
780
|
namespacesToPlan.add(ns);
|
|
@@ -403,17 +851,328 @@ async function computeConvergePlan(options = {}) {
|
|
|
403
851
|
peerTombstones.set(ns, peerData.tombstones);
|
|
404
852
|
}
|
|
405
853
|
}
|
|
854
|
+
const memoryDir = options.cursorDir ?? config?.memoryDir;
|
|
855
|
+
if (!options.baseFilesByNamespace && memoryDir && options.peerUrl) {
|
|
856
|
+
for (const ns of namespacesToPlan) {
|
|
857
|
+
const cursorPath = defaultConvergeCursorPath(memoryDir, options.peerUrl, ns);
|
|
858
|
+
const cursor = await readConvergeCursor(cursorPath);
|
|
859
|
+
if (cursor?.baseFiles && cursor.baseFiles.length > 0) {
|
|
860
|
+
baseMap.set(ns, cursor.baseFiles);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
406
864
|
const inputs = [];
|
|
407
865
|
for (const ns of [...namespacesToPlan].sort()) {
|
|
408
866
|
inputs.push({
|
|
409
867
|
namespace: ns,
|
|
410
868
|
local: localMap.get(ns) ?? [],
|
|
411
869
|
peer: peerMap.get(ns) ?? [],
|
|
870
|
+
base: baseMap.get(ns),
|
|
412
871
|
tombstonedFileSha256: localTombstones.get(ns) ?? [],
|
|
413
872
|
peerTombstonedFileSha256: peerTombstones.get(ns) ?? []
|
|
414
873
|
});
|
|
415
874
|
}
|
|
416
|
-
return planReconciliation(inputs);
|
|
875
|
+
return planReconciliation(inputs, { conflictPolicy: options.conflictPolicy });
|
|
876
|
+
}
|
|
877
|
+
async function executeConvergeApply(options = {}) {
|
|
878
|
+
const conflictPolicy = options.conflictPolicy ?? "manual";
|
|
879
|
+
const plan = await computeConvergePlan({ ...options, conflictPolicy });
|
|
880
|
+
if (plan.converged && !options.dryRun) {
|
|
881
|
+
await updateCursorsForPlan(plan, options);
|
|
882
|
+
return {
|
|
883
|
+
converged: true,
|
|
884
|
+
status: "converged",
|
|
885
|
+
plan,
|
|
886
|
+
transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
|
|
887
|
+
cursorUpdated: true
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
const unresolvedCount = plan.byNamespace.reduce((acc, report) => acc + report.unresolved, 0);
|
|
891
|
+
if (unresolvedCount > 0 && conflictPolicy === "manual") {
|
|
892
|
+
return {
|
|
893
|
+
converged: false,
|
|
894
|
+
status: "stopped_unresolved_conflicts",
|
|
895
|
+
plan,
|
|
896
|
+
transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
|
|
897
|
+
cursorUpdated: false
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
const plannedTransfers = {
|
|
901
|
+
pulled: 0,
|
|
902
|
+
pushed: 0,
|
|
903
|
+
conflictsResolved: 0,
|
|
904
|
+
suppressed: 0,
|
|
905
|
+
failed: 0
|
|
906
|
+
};
|
|
907
|
+
for (const entry of plan.entries) {
|
|
908
|
+
if (entry.action === "pull") plannedTransfers.pulled += 1;
|
|
909
|
+
else if (entry.action === "push") plannedTransfers.pushed += 1;
|
|
910
|
+
else if (entry.action === "conflict") plannedTransfers.conflictsResolved += 1;
|
|
911
|
+
else if (entry.action === "suppress") plannedTransfers.suppressed += 1;
|
|
912
|
+
}
|
|
913
|
+
if (options.dryRun) {
|
|
914
|
+
return {
|
|
915
|
+
converged: false,
|
|
916
|
+
status: "dry_run",
|
|
917
|
+
plan,
|
|
918
|
+
transfers: plannedTransfers,
|
|
919
|
+
cursorUpdated: false
|
|
920
|
+
};
|
|
921
|
+
}
|
|
922
|
+
const actualTransfers = {
|
|
923
|
+
pulled: 0,
|
|
924
|
+
pushed: 0,
|
|
925
|
+
conflictsResolved: 0,
|
|
926
|
+
suppressed: 0,
|
|
927
|
+
failed: 0
|
|
928
|
+
};
|
|
929
|
+
const peerMutatedNamespaces = /* @__PURE__ */ new Set();
|
|
930
|
+
let resolvedToken;
|
|
931
|
+
if (options.peerToken) {
|
|
932
|
+
try {
|
|
933
|
+
resolvedToken = await resolveAgentAccessAuthToken(options.peerToken, {
|
|
934
|
+
resolveSecretRef: options.resolveSecretRef
|
|
935
|
+
});
|
|
936
|
+
} catch {
|
|
937
|
+
resolvedToken = options.peerToken;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
const fetchFn = options.fetchImpl ?? globalThis.fetch;
|
|
941
|
+
let config = options.config;
|
|
942
|
+
if (!config) {
|
|
943
|
+
try {
|
|
944
|
+
config = parseConfig2({});
|
|
945
|
+
} catch {
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
const rootMap = /* @__PURE__ */ new Map();
|
|
949
|
+
if (config) {
|
|
950
|
+
try {
|
|
951
|
+
const roots = await resolveCorpusNamespaceRoots({ config });
|
|
952
|
+
for (const r of roots) {
|
|
953
|
+
rootMap.set(r.namespace, r.rootDir);
|
|
954
|
+
}
|
|
955
|
+
} catch {
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
for (const entry of plan.entries) {
|
|
959
|
+
if (entry.action === "identical") continue;
|
|
960
|
+
let transferType = "none";
|
|
961
|
+
if (entry.action === "pull") {
|
|
962
|
+
transferType = "pull";
|
|
963
|
+
} else if (entry.action === "push") {
|
|
964
|
+
transferType = "push";
|
|
965
|
+
} else if (entry.action === "suppress") {
|
|
966
|
+
transferType = "suppress";
|
|
967
|
+
} else if (entry.action === "conflict") {
|
|
968
|
+
if (entry.resolution === "peer-wins") {
|
|
969
|
+
transferType = "pull";
|
|
970
|
+
} else if (entry.resolution === "local-wins") {
|
|
971
|
+
transferType = "push";
|
|
972
|
+
} else if (entry.resolution === "supersede-link") {
|
|
973
|
+
if (entry.newerSide === "peer") transferType = "pull";
|
|
974
|
+
else if (entry.newerSide === "local") transferType = "push";
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
if (transferType === "pull") {
|
|
978
|
+
let remoteFile = null;
|
|
979
|
+
const buffered = options.peerFileBuffers?.get(entry.namespace)?.get(entry.path);
|
|
980
|
+
if (buffered) {
|
|
981
|
+
const state = options.peerFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path);
|
|
982
|
+
remoteFile = {
|
|
983
|
+
content: buffered,
|
|
984
|
+
sha256: state?.sha256 ?? entry.peerSha256 ?? createHash2("sha256").update(buffered).digest("hex"),
|
|
985
|
+
bytes: buffered.length,
|
|
986
|
+
mtimeMs: state?.mtimeMs ?? 0
|
|
987
|
+
};
|
|
988
|
+
} else if (options.peerUrl) {
|
|
989
|
+
remoteFile = await fetchPeerFileContent(
|
|
990
|
+
options.peerUrl,
|
|
991
|
+
entry.namespace,
|
|
992
|
+
entry.path,
|
|
993
|
+
resolvedToken,
|
|
994
|
+
fetchFn
|
|
995
|
+
);
|
|
996
|
+
}
|
|
997
|
+
if (remoteFile !== null && (!entry.peerSha256 || remoteFile.sha256 === entry.peerSha256)) {
|
|
998
|
+
if (options.localFileBuffers) {
|
|
999
|
+
let nsMap = options.localFileBuffers.get(entry.namespace);
|
|
1000
|
+
if (!nsMap) {
|
|
1001
|
+
nsMap = /* @__PURE__ */ new Map();
|
|
1002
|
+
options.localFileBuffers.set(entry.namespace, nsMap);
|
|
1003
|
+
}
|
|
1004
|
+
nsMap.set(entry.path, remoteFile.content);
|
|
1005
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1006
|
+
else actualTransfers.pulled += 1;
|
|
1007
|
+
} else {
|
|
1008
|
+
const rootDir = rootMap.get(entry.namespace);
|
|
1009
|
+
if (rootDir) {
|
|
1010
|
+
const io = await createOfflineStorageIo(rootDir);
|
|
1011
|
+
const expectedLocalSha256 = entry.action === "conflict" ? entry.localSha256 : entry.baseSha256;
|
|
1012
|
+
let offset = 0;
|
|
1013
|
+
let transferComplete = false;
|
|
1014
|
+
do {
|
|
1015
|
+
const chunk = remoteFile.content.subarray(
|
|
1016
|
+
offset,
|
|
1017
|
+
Math.min(
|
|
1018
|
+
remoteFile.content.length,
|
|
1019
|
+
offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2
|
|
1020
|
+
)
|
|
1021
|
+
);
|
|
1022
|
+
const chunkResult = await applyOfflineSyncFileContentChunk({
|
|
1023
|
+
root: rootDir,
|
|
1024
|
+
sourceId: "remnic-converge",
|
|
1025
|
+
path: entry.path,
|
|
1026
|
+
sha256: remoteFile.sha256,
|
|
1027
|
+
bytes: remoteFile.bytes,
|
|
1028
|
+
mtimeMs: remoteFile.mtimeMs,
|
|
1029
|
+
offset,
|
|
1030
|
+
content: chunk,
|
|
1031
|
+
...expectedLocalSha256 ? { baseSha256: expectedLocalSha256 } : {},
|
|
1032
|
+
readFile: io.readFile,
|
|
1033
|
+
readFileDigest: io.readFileDigest,
|
|
1034
|
+
writeFile: io.writeFile,
|
|
1035
|
+
writeStagingFile: io.writeStagingFile,
|
|
1036
|
+
writeFileChunks: io.writeFileChunks
|
|
1037
|
+
});
|
|
1038
|
+
if (chunkResult.conflict) {
|
|
1039
|
+
break;
|
|
1040
|
+
}
|
|
1041
|
+
if (chunkResult.done) {
|
|
1042
|
+
transferComplete = chunkResult.applied || chunkResult.skipped;
|
|
1043
|
+
break;
|
|
1044
|
+
}
|
|
1045
|
+
if (chunkResult.applied || chunkResult.skipped || chunk.length === 0) {
|
|
1046
|
+
break;
|
|
1047
|
+
}
|
|
1048
|
+
offset += chunk.length;
|
|
1049
|
+
} while (offset < remoteFile.content.length);
|
|
1050
|
+
if (transferComplete) {
|
|
1051
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1052
|
+
else actualTransfers.pulled += 1;
|
|
1053
|
+
} else {
|
|
1054
|
+
actualTransfers.failed += 1;
|
|
1055
|
+
}
|
|
1056
|
+
} else {
|
|
1057
|
+
actualTransfers.failed += 1;
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
} else {
|
|
1061
|
+
actualTransfers.failed += 1;
|
|
1062
|
+
}
|
|
1063
|
+
} else if (transferType === "push") {
|
|
1064
|
+
let content = null;
|
|
1065
|
+
let mtimeMs = options.localFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path)?.mtimeMs;
|
|
1066
|
+
if (options.localFileBuffers?.get(entry.namespace)?.has(entry.path)) {
|
|
1067
|
+
content = options.localFileBuffers.get(entry.namespace).get(entry.path);
|
|
1068
|
+
} else {
|
|
1069
|
+
const rootDir = rootMap.get(entry.namespace);
|
|
1070
|
+
if (rootDir) {
|
|
1071
|
+
const filePath = path2.join(rootDir, entry.path);
|
|
1072
|
+
try {
|
|
1073
|
+
const io = await createOfflineStorageIo(rootDir);
|
|
1074
|
+
content = await io.readFile({ root: rootDir, path: entry.path, filePath });
|
|
1075
|
+
mtimeMs ??= (await fs3.promises.stat(filePath)).mtimeMs;
|
|
1076
|
+
} catch {
|
|
1077
|
+
content = null;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
if (content !== null) {
|
|
1082
|
+
if (options.peerFileBuffers) {
|
|
1083
|
+
let nsMap = options.peerFileBuffers.get(entry.namespace);
|
|
1084
|
+
if (!nsMap) {
|
|
1085
|
+
nsMap = /* @__PURE__ */ new Map();
|
|
1086
|
+
options.peerFileBuffers.set(entry.namespace, nsMap);
|
|
1087
|
+
}
|
|
1088
|
+
nsMap.set(entry.path, content);
|
|
1089
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1090
|
+
else actualTransfers.pushed += 1;
|
|
1091
|
+
} else if (options.peerUrl && entry.localSha256) {
|
|
1092
|
+
const expectedPeerSha256 = entry.action === "conflict" ? entry.peerSha256 : entry.baseSha256;
|
|
1093
|
+
const applied = await postPeerFileContent(
|
|
1094
|
+
options.peerUrl,
|
|
1095
|
+
entry.namespace,
|
|
1096
|
+
entry.path,
|
|
1097
|
+
content,
|
|
1098
|
+
{
|
|
1099
|
+
sha256: entry.localSha256,
|
|
1100
|
+
mtimeMs: mtimeMs ?? 0,
|
|
1101
|
+
...expectedPeerSha256 ? { baseSha256: expectedPeerSha256 } : {}
|
|
1102
|
+
},
|
|
1103
|
+
resolvedToken,
|
|
1104
|
+
fetchFn
|
|
1105
|
+
);
|
|
1106
|
+
if (applied) {
|
|
1107
|
+
if (applied === "applied") peerMutatedNamespaces.add(entry.namespace);
|
|
1108
|
+
if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
|
|
1109
|
+
else actualTransfers.pushed += 1;
|
|
1110
|
+
} else {
|
|
1111
|
+
actualTransfers.failed += 1;
|
|
1112
|
+
}
|
|
1113
|
+
} else {
|
|
1114
|
+
actualTransfers.failed += 1;
|
|
1115
|
+
}
|
|
1116
|
+
} else {
|
|
1117
|
+
actualTransfers.failed += 1;
|
|
1118
|
+
}
|
|
1119
|
+
} else if (transferType === "suppress") {
|
|
1120
|
+
actualTransfers.suppressed += 1;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
if (options.peerUrl && peerMutatedNamespaces.size > 0) {
|
|
1124
|
+
const namespaces = [...peerMutatedNamespaces].sort();
|
|
1125
|
+
if (!await postPeerConvergenceComplete(
|
|
1126
|
+
options.peerUrl,
|
|
1127
|
+
namespaces,
|
|
1128
|
+
resolvedToken,
|
|
1129
|
+
fetchFn
|
|
1130
|
+
)) {
|
|
1131
|
+
actualTransfers.failed += 1;
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
let cursorUpdated = false;
|
|
1135
|
+
if (actualTransfers.failed === 0) {
|
|
1136
|
+
await updateCursorsForPlan(plan, options);
|
|
1137
|
+
cursorUpdated = true;
|
|
1138
|
+
}
|
|
1139
|
+
return {
|
|
1140
|
+
converged: actualTransfers.failed === 0,
|
|
1141
|
+
status: "applied",
|
|
1142
|
+
plan,
|
|
1143
|
+
transfers: actualTransfers,
|
|
1144
|
+
cursorUpdated
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
async function updateCursorsForPlan(plan, options) {
|
|
1148
|
+
const peerUrl = options.peerUrl ?? "local";
|
|
1149
|
+
let memoryDir;
|
|
1150
|
+
if (options.cursorDir) {
|
|
1151
|
+
memoryDir = options.cursorDir;
|
|
1152
|
+
} else if (options.config) {
|
|
1153
|
+
memoryDir = options.config.memoryDir;
|
|
1154
|
+
}
|
|
1155
|
+
if (!memoryDir) return;
|
|
1156
|
+
const namespaces = new Set(plan.byNamespace.map((n) => n.namespace));
|
|
1157
|
+
for (const ns of namespaces) {
|
|
1158
|
+
const cursorPath = defaultConvergeCursorPath(memoryDir, peerUrl, ns);
|
|
1159
|
+
const nsEntries = plan.entries.filter((e) => e.namespace === ns);
|
|
1160
|
+
const baseFiles = nsEntries.map((e) => ({
|
|
1161
|
+
path: e.path,
|
|
1162
|
+
sha256: e.localSha256 ?? e.peerSha256 ?? "unknown"
|
|
1163
|
+
}));
|
|
1164
|
+
const cursorState = {
|
|
1165
|
+
version: 1,
|
|
1166
|
+
peerUrl,
|
|
1167
|
+
namespace: ns,
|
|
1168
|
+
lastConvergedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1169
|
+
baseFiles
|
|
1170
|
+
};
|
|
1171
|
+
try {
|
|
1172
|
+
await writeConvergeCursor(cursorPath, cursorState);
|
|
1173
|
+
} catch {
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
417
1176
|
}
|
|
418
1177
|
function formatConvergeReport(plan) {
|
|
419
1178
|
const lines = [];
|
|
@@ -435,25 +1194,49 @@ function formatConvergeReport(plan) {
|
|
|
435
1194
|
}
|
|
436
1195
|
return lines.join("\n");
|
|
437
1196
|
}
|
|
1197
|
+
function formatConvergeApplyReport(result) {
|
|
1198
|
+
const lines = [];
|
|
1199
|
+
lines.push(`Convergence Execution Status: ${result.status.toUpperCase()}`);
|
|
1200
|
+
lines.push(`Converged: ${result.converged ? "YES" : "NO"}`);
|
|
1201
|
+
lines.push("");
|
|
1202
|
+
lines.push("Transfers Executed:");
|
|
1203
|
+
lines.push(` pulled: ${result.transfers.pulled}`);
|
|
1204
|
+
lines.push(` pushed: ${result.transfers.pushed}`);
|
|
1205
|
+
lines.push(` conflictsResolved: ${result.transfers.conflictsResolved}`);
|
|
1206
|
+
lines.push(` suppressed: ${result.transfers.suppressed}`);
|
|
1207
|
+
lines.push(` failed: ${result.transfers.failed}`);
|
|
1208
|
+
lines.push("");
|
|
1209
|
+
lines.push(formatConvergeReport(result.plan));
|
|
1210
|
+
return lines.join("\n");
|
|
1211
|
+
}
|
|
438
1212
|
async function cmdConverge(action, rest, json) {
|
|
439
1213
|
if (action === "help" || action === "--help" || action === "-h" || rest.includes("--help") || rest.includes("-h")) {
|
|
440
|
-
console.log(`Usage: remnic converge plan
|
|
1214
|
+
console.log(`Usage: remnic converge <plan|apply> [options]
|
|
1215
|
+
|
|
1216
|
+
Subcommands:
|
|
1217
|
+
plan Compute and display reconciliation plan
|
|
1218
|
+
apply Execute bidirectional converge transport (alias: transport, sync)
|
|
441
1219
|
|
|
442
1220
|
Options:
|
|
443
1221
|
--peer <url> Peer server URL (or --remote-url / --remote)
|
|
444
1222
|
--token <token> Bearer token or SecretRef for peer authentication
|
|
1223
|
+
--conflict-policy <policy>
|
|
1224
|
+
Conflict resolution policy (manual|newest-wins|keep-both)
|
|
1225
|
+
--dry-run Simulate transfers without mutating disk or remote peer
|
|
445
1226
|
--json Output detailed JSON plan report
|
|
446
1227
|
`);
|
|
447
1228
|
return;
|
|
448
1229
|
}
|
|
449
|
-
if (action !== "plan") {
|
|
450
|
-
process.stderr.write(`converge: unknown action "${action}". Use: plan [options].
|
|
1230
|
+
if (action !== "plan" && action !== "apply" && action !== "transport" && action !== "sync") {
|
|
1231
|
+
process.stderr.write(`converge: unknown action "${action}". Use: plan or apply [options].
|
|
451
1232
|
`);
|
|
452
1233
|
process.exitCode = 2;
|
|
453
1234
|
return;
|
|
454
1235
|
}
|
|
455
1236
|
let peerUrl;
|
|
456
1237
|
let peerToken;
|
|
1238
|
+
let dryRun = false;
|
|
1239
|
+
let conflictPolicy;
|
|
457
1240
|
for (let i = 0; i < rest.length; i += 1) {
|
|
458
1241
|
const arg = rest[i];
|
|
459
1242
|
if ((arg === "--peer" || arg === "--remote-url" || arg === "--remote") && rest[i + 1]) {
|
|
@@ -462,13 +1245,35 @@ Options:
|
|
|
462
1245
|
} else if (arg === "--token" && rest[i + 1]) {
|
|
463
1246
|
peerToken = rest[i + 1];
|
|
464
1247
|
i += 1;
|
|
1248
|
+
} else if (arg === "--dry-run") {
|
|
1249
|
+
dryRun = true;
|
|
1250
|
+
} else if (arg === "--conflict-policy" && rest[i + 1]) {
|
|
1251
|
+
const pol = rest[i + 1];
|
|
1252
|
+
if (pol === "manual" || pol === "newest-wins" || pol === "keep-both") {
|
|
1253
|
+
conflictPolicy = pol;
|
|
1254
|
+
}
|
|
1255
|
+
i += 1;
|
|
465
1256
|
}
|
|
466
1257
|
}
|
|
467
|
-
|
|
1258
|
+
if (action === "plan") {
|
|
1259
|
+
const plan = await computeConvergePlan({ peerUrl, peerToken, conflictPolicy });
|
|
1260
|
+
if (json) {
|
|
1261
|
+
console.log(JSON.stringify(plan, null, 2));
|
|
1262
|
+
} else {
|
|
1263
|
+
console.log(formatConvergeReport(plan));
|
|
1264
|
+
}
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
const result = await executeConvergeApply({
|
|
1268
|
+
peerUrl,
|
|
1269
|
+
peerToken,
|
|
1270
|
+
dryRun,
|
|
1271
|
+
conflictPolicy
|
|
1272
|
+
});
|
|
468
1273
|
if (json) {
|
|
469
|
-
console.log(JSON.stringify(
|
|
1274
|
+
console.log(JSON.stringify(result, null, 2));
|
|
470
1275
|
} else {
|
|
471
|
-
console.log(
|
|
1276
|
+
console.log(formatConvergeApplyReport(result));
|
|
472
1277
|
}
|
|
473
1278
|
}
|
|
474
1279
|
|
|
@@ -605,7 +1410,7 @@ function renderReplayResult(result, targetNamespace, format) {
|
|
|
605
1410
|
}
|
|
606
1411
|
|
|
607
1412
|
// src/quarantine-replay.ts
|
|
608
|
-
import * as
|
|
1413
|
+
import * as fs4 from "fs";
|
|
609
1414
|
import { EngramAccessService, Orchestrator as Orchestrator2, initLogger, parseConfig as parseConfig3, resolveRemnicConfigRecord as resolveRemnicConfigRecord2 } from "@remnic/core";
|
|
610
1415
|
import { WriteQuarantineStore } from "@remnic/core/write-quarantine.js";
|
|
611
1416
|
function valueFlag(args, flag) {
|
|
@@ -654,7 +1459,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
|
|
|
654
1459
|
let orchestrator;
|
|
655
1460
|
try {
|
|
656
1461
|
const configPath = resolveConfigPath2();
|
|
657
|
-
const raw =
|
|
1462
|
+
const raw = fs4.existsSync(configPath) ? JSON.parse(fs4.readFileSync(configPath, "utf8")) : {};
|
|
658
1463
|
const config = parseConfig3(resolveRemnicConfigRecord2(raw));
|
|
659
1464
|
orchestrator = new Orchestrator2(config);
|
|
660
1465
|
await orchestrator.initialize();
|
|
@@ -686,7 +1491,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
|
|
|
686
1491
|
}
|
|
687
1492
|
|
|
688
1493
|
// src/offline-impression-rotation.ts
|
|
689
|
-
import
|
|
1494
|
+
import fs5 from "fs";
|
|
690
1495
|
import { parseConfig as parseConfig4, resolveRemnicConfigRecord as resolveRemnicConfigRecord3, drainPendingImpressionsForOfflineSync } from "@remnic/core";
|
|
691
1496
|
import { LastRecallStore } from "@remnic/core/recall-state";
|
|
692
1497
|
function parseConfigQuietly(raw) {
|
|
@@ -721,7 +1526,7 @@ function pickOfflineConfigRecord(raw) {
|
|
|
721
1526
|
function resolveOfflineImpressionRotation(configPath) {
|
|
722
1527
|
let raw;
|
|
723
1528
|
try {
|
|
724
|
-
raw =
|
|
1529
|
+
raw = fs5.existsSync(configPath) ? JSON.parse(fs5.readFileSync(configPath, "utf8")) : {};
|
|
725
1530
|
} catch {
|
|
726
1531
|
throw new Error(
|
|
727
1532
|
`cannot read recall-impression rotation from ${configPath}: config file could not be read as JSON`
|
|
@@ -749,265 +1554,6 @@ async function drainOfflineSyncImpressions(memoryDir, rotation) {
|
|
|
749
1554
|
);
|
|
750
1555
|
}
|
|
751
1556
|
|
|
752
|
-
// src/offline-storage-io.ts
|
|
753
|
-
import { mkdtemp, readdir, lstat, rm } from "fs/promises";
|
|
754
|
-
import fs5 from "fs";
|
|
755
|
-
import path2 from "path";
|
|
756
|
-
import { createHash, createDecipheriv } from "crypto";
|
|
757
|
-
import {
|
|
758
|
-
OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
|
|
759
|
-
StorageManager
|
|
760
|
-
} from "@remnic/core";
|
|
761
|
-
import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
|
|
762
|
-
import {
|
|
763
|
-
AUTH_TAG_LENGTH,
|
|
764
|
-
ENVELOPE_HEADER_SIZE,
|
|
765
|
-
ENVELOPE_LAYOUT,
|
|
766
|
-
ENVELOPE_SALT_LENGTH,
|
|
767
|
-
ENVELOPE_VERSION,
|
|
768
|
-
FILE_FORMAT_FLAGS,
|
|
769
|
-
FILE_FORMAT_VERSION,
|
|
770
|
-
IV_LENGTH,
|
|
771
|
-
MAGIC_BYTES,
|
|
772
|
-
MAGIC_HEADER_SIZE,
|
|
773
|
-
SecureStoreLockedError,
|
|
774
|
-
filePathAad,
|
|
775
|
-
isEncryptedFile,
|
|
776
|
-
keyring,
|
|
777
|
-
readHeader,
|
|
778
|
-
secureStoreDir
|
|
779
|
-
} from "@remnic/core/secure-store";
|
|
780
|
-
async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
|
|
781
|
-
const storage = new StorageManager(memoryDir);
|
|
782
|
-
const header = await readHeader(memoryDir);
|
|
783
|
-
let secureStoreKey = null;
|
|
784
|
-
let secureStoreRequired = false;
|
|
785
|
-
if (header) {
|
|
786
|
-
secureStoreRequired = true;
|
|
787
|
-
storage.setSecureStoreRequired(true);
|
|
788
|
-
const key = keyring.getKey(secureStoreDir(memoryDir));
|
|
789
|
-
if (key) {
|
|
790
|
-
await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
|
|
791
|
-
secureStoreKey = key;
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
return { storage, secureStoreKey, secureStoreRequired };
|
|
795
|
-
}
|
|
796
|
-
async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
|
|
797
|
-
const memoryRoot = path2.resolve(memoryDir);
|
|
798
|
-
const stateDir = path2.dirname(filePath);
|
|
799
|
-
if (path2.basename(stateDir) !== "state" || path2.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
|
|
800
|
-
throw new Error(`invalid lifecycle ledger path: ${filePath}`);
|
|
801
|
-
}
|
|
802
|
-
const storageRoot = path2.resolve(path2.dirname(stateDir));
|
|
803
|
-
if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path2.sep}`)) {
|
|
804
|
-
throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
|
|
805
|
-
}
|
|
806
|
-
const storage = new StorageManager(storageRoot);
|
|
807
|
-
if (configured.secureStoreRequired) {
|
|
808
|
-
storage.setSecureStoreRequired(true);
|
|
809
|
-
}
|
|
810
|
-
if (configured.secureStoreKey) {
|
|
811
|
-
await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
|
|
812
|
-
}
|
|
813
|
-
return storage;
|
|
814
|
-
}
|
|
815
|
-
async function createOfflineStorageIo(memoryDir, configuredStorage) {
|
|
816
|
-
await cleanupOrphanedOfflineDecryptStaging(memoryDir);
|
|
817
|
-
const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
|
|
818
|
-
return {
|
|
819
|
-
readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
|
|
820
|
-
readFileDigest: async ({ filePath }) => {
|
|
821
|
-
const hash = createHash("sha256");
|
|
822
|
-
let bytes = 0;
|
|
823
|
-
for await (const rawChunk of readOfflineSyncFileChunks({
|
|
824
|
-
filePath,
|
|
825
|
-
memoryDir,
|
|
826
|
-
secureStoreKey,
|
|
827
|
-
chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
|
|
828
|
-
})) {
|
|
829
|
-
const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
|
|
830
|
-
hash.update(chunk);
|
|
831
|
-
bytes += chunk.length;
|
|
832
|
-
}
|
|
833
|
-
return {
|
|
834
|
-
sha256: hash.digest("hex"),
|
|
835
|
-
bytes
|
|
836
|
-
};
|
|
837
|
-
},
|
|
838
|
-
readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
|
|
839
|
-
filePath,
|
|
840
|
-
memoryDir,
|
|
841
|
-
secureStoreKey,
|
|
842
|
-
chunkSize
|
|
843
|
-
}),
|
|
844
|
-
writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
|
|
845
|
-
writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
|
|
846
|
-
writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
|
|
847
|
-
deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
|
|
848
|
-
};
|
|
849
|
-
}
|
|
850
|
-
var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
|
|
851
|
-
async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
|
|
852
|
-
let entries;
|
|
853
|
-
try {
|
|
854
|
-
entries = await readdir(memoryDir);
|
|
855
|
-
} catch {
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
const now = Date.now();
|
|
859
|
-
for (const name of entries) {
|
|
860
|
-
if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
|
|
861
|
-
const dir = path2.join(memoryDir, name);
|
|
862
|
-
try {
|
|
863
|
-
const info = await lstat(dir);
|
|
864
|
-
if (!info.isDirectory() || info.isSymbolicLink()) continue;
|
|
865
|
-
if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
|
|
866
|
-
await rm(dir, { recursive: true, force: true });
|
|
867
|
-
} catch {
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
async function* readOfflineSyncFileChunks(options) {
|
|
872
|
-
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
|
|
873
|
-
if (!isEncryptedFile(header)) {
|
|
874
|
-
yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
|
|
875
|
-
return;
|
|
876
|
-
}
|
|
877
|
-
if (!options.secureStoreKey) {
|
|
878
|
-
throw new SecureStoreLockedError(
|
|
879
|
-
`secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
|
|
880
|
-
);
|
|
881
|
-
}
|
|
882
|
-
yield* readEncryptedOfflineFileChunks({
|
|
883
|
-
filePath: options.filePath,
|
|
884
|
-
memoryDir: options.memoryDir,
|
|
885
|
-
key: options.secureStoreKey,
|
|
886
|
-
chunkSize: options.chunkSize
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
async function readFilePrefix(filePath, length) {
|
|
890
|
-
const handle = await fs5.promises.open(filePath, "r");
|
|
891
|
-
try {
|
|
892
|
-
const out = Buffer.alloc(length);
|
|
893
|
-
const { bytesRead } = await handle.read(out, 0, length, 0);
|
|
894
|
-
return out.subarray(0, bytesRead);
|
|
895
|
-
} finally {
|
|
896
|
-
await handle.close();
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
async function* readPlainOfflineFileChunks(filePath, chunkSize) {
|
|
900
|
-
const stream = fs5.createReadStream(filePath, { highWaterMark: chunkSize });
|
|
901
|
-
for await (const chunk of stream) {
|
|
902
|
-
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
903
|
-
}
|
|
904
|
-
}
|
|
905
|
-
async function* readEncryptedOfflineFileChunks(options) {
|
|
906
|
-
const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
907
|
-
if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
|
|
908
|
-
throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
|
|
909
|
-
}
|
|
910
|
-
const version = header.readUInt8(MAGIC_BYTES.length);
|
|
911
|
-
const flags = header.readUInt8(MAGIC_BYTES.length + 1);
|
|
912
|
-
if (version !== FILE_FORMAT_VERSION) {
|
|
913
|
-
throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
|
|
914
|
-
}
|
|
915
|
-
if (flags !== FILE_FORMAT_FLAGS) {
|
|
916
|
-
throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
|
|
917
|
-
}
|
|
918
|
-
const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
|
|
919
|
-
const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
|
|
920
|
-
if (envelopeVersion !== ENVELOPE_VERSION) {
|
|
921
|
-
throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
|
|
922
|
-
}
|
|
923
|
-
const salt = envelopeHeader.subarray(
|
|
924
|
-
ENVELOPE_LAYOUT.salt,
|
|
925
|
-
ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
|
|
926
|
-
);
|
|
927
|
-
const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
|
|
928
|
-
const authTag = envelopeHeader.subarray(
|
|
929
|
-
ENVELOPE_LAYOUT.authTag,
|
|
930
|
-
ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
|
|
931
|
-
);
|
|
932
|
-
const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
|
|
933
|
-
let lastError;
|
|
934
|
-
for (const aad of aadCandidates) {
|
|
935
|
-
const tempDir = await mkdtemp(path2.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
|
|
936
|
-
const tempPath = path2.join(tempDir, "content");
|
|
937
|
-
try {
|
|
938
|
-
const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
|
|
939
|
-
authTagLength: AUTH_TAG_LENGTH
|
|
940
|
-
});
|
|
941
|
-
decipher.setAuthTag(authTag);
|
|
942
|
-
decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
|
|
943
|
-
const output = fs5.createWriteStream(tempPath, { mode: 384 });
|
|
944
|
-
try {
|
|
945
|
-
const stream = fs5.createReadStream(options.filePath, {
|
|
946
|
-
start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
|
|
947
|
-
highWaterMark: options.chunkSize
|
|
948
|
-
});
|
|
949
|
-
for await (const encryptedChunk of stream) {
|
|
950
|
-
const plain = decipher.update(
|
|
951
|
-
Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
|
|
952
|
-
);
|
|
953
|
-
if (plain.length > 0 && !output.write(plain)) {
|
|
954
|
-
await new Promise((resolve, reject) => {
|
|
955
|
-
output.once("drain", resolve);
|
|
956
|
-
output.once("error", reject);
|
|
957
|
-
});
|
|
958
|
-
}
|
|
959
|
-
}
|
|
960
|
-
const finalPlain = decipher.final();
|
|
961
|
-
if (finalPlain.length > 0 && !output.write(finalPlain)) {
|
|
962
|
-
await new Promise((resolve, reject) => {
|
|
963
|
-
output.once("drain", resolve);
|
|
964
|
-
output.once("error", reject);
|
|
965
|
-
});
|
|
966
|
-
}
|
|
967
|
-
} finally {
|
|
968
|
-
await closeWriteStream(output);
|
|
969
|
-
}
|
|
970
|
-
yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
|
|
971
|
-
return;
|
|
972
|
-
} catch (error) {
|
|
973
|
-
lastError = error;
|
|
974
|
-
} finally {
|
|
975
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
|
|
979
|
-
}
|
|
980
|
-
function offlineFileAadCandidates(filePath, memoryDir) {
|
|
981
|
-
const candidates = [filePathAad(filePath, memoryDir)];
|
|
982
|
-
const relative = path2.relative(memoryDir, filePath);
|
|
983
|
-
if (!relative || relative.startsWith("..") || path2.isAbsolute(relative)) return candidates;
|
|
984
|
-
const parts = relative.split(path2.sep);
|
|
985
|
-
if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
|
|
986
|
-
candidates.push(filePathAad(filePath, path2.join(memoryDir, "namespaces", parts[1])));
|
|
987
|
-
}
|
|
988
|
-
const memoryParts = path2.resolve(memoryDir).split(path2.sep);
|
|
989
|
-
if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
|
|
990
|
-
const topLevelRoot = memoryParts.slice(0, -2).join(path2.sep) || path2.sep;
|
|
991
|
-
const topRelative = path2.relative(topLevelRoot, filePath);
|
|
992
|
-
if (topRelative && !topRelative.startsWith("..") && !path2.isAbsolute(topRelative) && topRelative.split(path2.sep)[0] === "namespaces" && topRelative.split(path2.sep)[1] === memoryParts.at(-1)) {
|
|
993
|
-
candidates.push(filePathAad(filePath, topLevelRoot));
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
return candidates;
|
|
997
|
-
}
|
|
998
|
-
async function closeWriteStream(stream) {
|
|
999
|
-
await new Promise((resolve, reject) => {
|
|
1000
|
-
stream.once("error", reject);
|
|
1001
|
-
stream.end(() => resolve());
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1004
|
-
function secureStoreEnvelopeHeaderAad(salt) {
|
|
1005
|
-
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
1006
|
-
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
1007
|
-
Buffer.from(salt).copy(out, 1);
|
|
1008
|
-
return out;
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
1557
|
// src/bench-build-freshness.ts
|
|
1012
1558
|
import {
|
|
1013
1559
|
existsSync as existsSync2,
|
|
@@ -5892,7 +6438,7 @@ async function calibrateBenchJudges(parsed, rawArgs) {
|
|
|
5892
6438
|
);
|
|
5893
6439
|
process.exit(1);
|
|
5894
6440
|
}
|
|
5895
|
-
const sourceResultSha256 =
|
|
6441
|
+
const sourceResultSha256 = createHash3("sha256").update(fs12.readFileSync(latest.path)).digest("hex");
|
|
5896
6442
|
const expandedManifestPath = expandTilde(manifestPath);
|
|
5897
6443
|
if (!bench.resolveLocalLabJudgeProviderConfig) {
|
|
5898
6444
|
console.error(
|
|
@@ -6756,7 +7302,7 @@ function attachPreparedJudgeCalibration(result, judgeCalibration) {
|
|
|
6756
7302
|
function hashCalibrationProviderConfig(config) {
|
|
6757
7303
|
const canonicalize = (value, key = "") => {
|
|
6758
7304
|
if (typeof value === "string" && /(?:api.?key|authorization|token|secret)/i.test(key)) {
|
|
6759
|
-
return { secretSha256:
|
|
7305
|
+
return { secretSha256: createHash3("sha256").update(value).digest("hex") };
|
|
6760
7306
|
}
|
|
6761
7307
|
if (Array.isArray(value)) return value.map((item) => canonicalize(item));
|
|
6762
7308
|
if (value && typeof value === "object") {
|
|
@@ -6767,7 +7313,7 @@ function hashCalibrationProviderConfig(config) {
|
|
|
6767
7313
|
}
|
|
6768
7314
|
return value;
|
|
6769
7315
|
};
|
|
6770
|
-
return
|
|
7316
|
+
return createHash3("sha256").update(JSON.stringify(canonicalize(config))).digest("hex");
|
|
6771
7317
|
}
|
|
6772
7318
|
function restoreOptionalEnv(key, previousValue) {
|
|
6773
7319
|
if (previousValue === void 0) {
|
|
@@ -9199,7 +9745,7 @@ async function cmdSync(action, rest, json) {
|
|
|
9199
9745
|
}
|
|
9200
9746
|
function localOfflineSourceId(memoryDir) {
|
|
9201
9747
|
const host = os.hostname() || "unknown-host";
|
|
9202
|
-
const dirHash =
|
|
9748
|
+
const dirHash = createHash3("sha256").update(path15.resolve(memoryDir)).digest("hex").slice(0, 16);
|
|
9203
9749
|
return `remnic-local:${host}:${dirHash}`;
|
|
9204
9750
|
}
|
|
9205
9751
|
function normalizeOfflineRemoteUrl(raw) {
|
|
@@ -9799,7 +10345,7 @@ function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
|
|
|
9799
10345
|
}
|
|
9800
10346
|
return target;
|
|
9801
10347
|
}
|
|
9802
|
-
var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES =
|
|
10348
|
+
var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3;
|
|
9803
10349
|
async function pushOfflineFileContent(args) {
|
|
9804
10350
|
if (args.readFileChunks) {
|
|
9805
10351
|
return pushOfflineFileContentFromChunkReader(args);
|
|
@@ -9807,7 +10353,7 @@ async function pushOfflineFileContent(args) {
|
|
|
9807
10353
|
let offset = 0;
|
|
9808
10354
|
let finalResult = null;
|
|
9809
10355
|
let remoteSatisfiedResult = null;
|
|
9810
|
-
const hash =
|
|
10356
|
+
const hash = createHash3("sha256");
|
|
9811
10357
|
let bytes = 0;
|
|
9812
10358
|
while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
|
|
9813
10359
|
const chunk = await readOfflineSyncFileContentChunk({
|
|
@@ -9868,7 +10414,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
9868
10414
|
if (stat.mtimeMs !== args.file.mtimeMs) {
|
|
9869
10415
|
throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
|
|
9870
10416
|
}
|
|
9871
|
-
const hash =
|
|
10417
|
+
const hash = createHash3("sha256");
|
|
9872
10418
|
const chunks = args.readFileChunks({
|
|
9873
10419
|
root: path15.resolve(args.memoryDir),
|
|
9874
10420
|
path: args.file.path,
|
|
@@ -9882,7 +10428,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
|
|
|
9882
10428
|
for await (const rawChunk of chunks) {
|
|
9883
10429
|
const chunk = Buffer.from(rawChunk);
|
|
9884
10430
|
if (chunk.length === 0) continue;
|
|
9885
|
-
if (chunk.length >
|
|
10431
|
+
if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2) {
|
|
9886
10432
|
throw new Error(`local offline content chunk exceeds max size: ${args.file.path}`);
|
|
9887
10433
|
}
|
|
9888
10434
|
if (pending) {
|
|
@@ -9967,7 +10513,7 @@ async function hydrateOfflineFileContent(args) {
|
|
|
9967
10513
|
path: args.expected.path,
|
|
9968
10514
|
offset,
|
|
9969
10515
|
length: Math.min(
|
|
9970
|
-
|
|
10516
|
+
OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
|
|
9971
10517
|
Math.max(1, args.expected.bytes - offset)
|
|
9972
10518
|
)
|
|
9973
10519
|
});
|
|
@@ -9978,7 +10524,7 @@ async function hydrateOfflineFileContent(args) {
|
|
|
9978
10524
|
throw new Error(`remote offline content chunk was empty before EOF: ${args.expected.path}`);
|
|
9979
10525
|
}
|
|
9980
10526
|
try {
|
|
9981
|
-
finalResult = await
|
|
10527
|
+
finalResult = await applyOfflineSyncFileContentChunk2({
|
|
9982
10528
|
root: args.memoryDir,
|
|
9983
10529
|
sourceId: args.sourceId,
|
|
9984
10530
|
path: args.expected.path,
|