@remnic/cli 9.42.0 → 9.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +859 -313
  2. package/package.json +29 -29
package/dist/index.js CHANGED
@@ -21,7 +21,7 @@ async function persistEnrichmentCandidate(storage, entityName, candidate) {
21
21
  import fs12 from "fs";
22
22
  import os from "os";
23
23
  import path15 from "path";
24
- import { createHash as createHash2 } from "crypto";
24
+ import { createHash as createHash3 } from "crypto";
25
25
  import * as childProcess2 from "child_process";
26
26
  import { fileURLToPath as fileURLToPath4 } from "url";
27
27
  import { gzipSync } from "zlib";
@@ -110,10 +110,10 @@ import {
110
110
  parseXrayCliOptions,
111
111
  renderXray,
112
112
  OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
113
- OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
114
- OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2,
113
+ OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
114
+ OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3,
115
115
  OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES,
116
- applyOfflineSyncFileContentChunk,
116
+ applyOfflineSyncFileContentChunk as applyOfflineSyncFileContentChunk2,
117
117
  applyOfflineSyncSnapshot,
118
118
  buildOfflineSyncChangesetFromSnapshot,
119
119
  drainPendingLifecycleForOfflineSync,
@@ -239,27 +239,297 @@ async function loadWecloneExportModule() {
239
239
  }
240
240
 
241
241
  // src/converge.ts
242
- import * as fs2 from "fs";
243
- import * as path from "path";
242
+ import * as fs3 from "fs";
243
+ import { createHash as createHash2 } from "crypto";
244
+ import * as path2 from "path";
244
245
  import {
245
246
  parseConfig as parseConfig2,
246
- buildOfflineSyncSnapshotFromBase
247
+ buildOfflineSyncSnapshotFromBase,
248
+ OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
249
+ OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES as OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2,
250
+ applyOfflineSyncFileContentChunk
247
251
  } from "@remnic/core";
248
252
  import { resolveCorpusNamespaceRoots } from "@remnic/core/corpus-watermark.js";
249
253
  import { listNamespaces } from "@remnic/core/namespaces/migrate.js";
250
254
  import {
251
255
  planReconciliation
252
256
  } from "@remnic/core/reconcile/plan.js";
257
+ import {
258
+ defaultConvergeCursorPath,
259
+ readConvergeCursor,
260
+ writeConvergeCursor
261
+ } from "@remnic/core/reconcile/cursor.js";
262
+
263
+ // src/offline-storage-io.ts
264
+ import { mkdtemp, readdir, lstat, rm } from "fs/promises";
265
+ import fs2 from "fs";
266
+ import path from "path";
267
+ import { createHash, createDecipheriv } from "crypto";
268
+ import {
269
+ OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
270
+ StorageManager
271
+ } from "@remnic/core";
272
+ import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
273
+ import {
274
+ AUTH_TAG_LENGTH,
275
+ ENVELOPE_HEADER_SIZE,
276
+ ENVELOPE_LAYOUT,
277
+ ENVELOPE_SALT_LENGTH,
278
+ ENVELOPE_VERSION,
279
+ FILE_FORMAT_FLAGS,
280
+ FILE_FORMAT_VERSION,
281
+ IV_LENGTH,
282
+ MAGIC_BYTES,
283
+ MAGIC_HEADER_SIZE,
284
+ SecureStoreLockedError,
285
+ filePathAad,
286
+ isEncryptedFile,
287
+ keyring,
288
+ readHeader,
289
+ secureStoreDir
290
+ } from "@remnic/core/secure-store";
291
+ async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
292
+ const storage = new StorageManager(memoryDir);
293
+ const header = await readHeader(memoryDir);
294
+ let secureStoreKey = null;
295
+ let secureStoreRequired = false;
296
+ if (header) {
297
+ secureStoreRequired = true;
298
+ storage.setSecureStoreRequired(true);
299
+ const key = keyring.getKey(secureStoreDir(memoryDir));
300
+ if (key) {
301
+ await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
302
+ secureStoreKey = key;
303
+ }
304
+ }
305
+ return { storage, secureStoreKey, secureStoreRequired };
306
+ }
307
+ async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
308
+ const memoryRoot = path.resolve(memoryDir);
309
+ const stateDir = path.dirname(filePath);
310
+ if (path.basename(stateDir) !== "state" || path.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
311
+ throw new Error(`invalid lifecycle ledger path: ${filePath}`);
312
+ }
313
+ const storageRoot = path.resolve(path.dirname(stateDir));
314
+ if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path.sep}`)) {
315
+ throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
316
+ }
317
+ const storage = new StorageManager(storageRoot);
318
+ if (configured.secureStoreRequired) {
319
+ storage.setSecureStoreRequired(true);
320
+ }
321
+ if (configured.secureStoreKey) {
322
+ await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
323
+ }
324
+ return storage;
325
+ }
326
+ async function createOfflineStorageIo(memoryDir, configuredStorage) {
327
+ await cleanupOrphanedOfflineDecryptStaging(memoryDir);
328
+ const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
329
+ return {
330
+ readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
331
+ readFileDigest: async ({ filePath }) => {
332
+ const hash = createHash("sha256");
333
+ let bytes = 0;
334
+ for await (const rawChunk of readOfflineSyncFileChunks({
335
+ filePath,
336
+ memoryDir,
337
+ secureStoreKey,
338
+ chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
339
+ })) {
340
+ const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
341
+ hash.update(chunk);
342
+ bytes += chunk.length;
343
+ }
344
+ return {
345
+ sha256: hash.digest("hex"),
346
+ bytes
347
+ };
348
+ },
349
+ readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
350
+ filePath,
351
+ memoryDir,
352
+ secureStoreKey,
353
+ chunkSize
354
+ }),
355
+ writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
356
+ writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
357
+ writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
358
+ deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
359
+ };
360
+ }
361
+ var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
362
+ async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
363
+ let entries;
364
+ try {
365
+ entries = await readdir(memoryDir);
366
+ } catch {
367
+ return;
368
+ }
369
+ const now = Date.now();
370
+ for (const name of entries) {
371
+ if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
372
+ const dir = path.join(memoryDir, name);
373
+ try {
374
+ const info = await lstat(dir);
375
+ if (!info.isDirectory() || info.isSymbolicLink()) continue;
376
+ if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
377
+ await rm(dir, { recursive: true, force: true });
378
+ } catch {
379
+ }
380
+ }
381
+ }
382
+ async function* readOfflineSyncFileChunks(options) {
383
+ const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
384
+ if (!isEncryptedFile(header)) {
385
+ yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
386
+ return;
387
+ }
388
+ if (!options.secureStoreKey) {
389
+ throw new SecureStoreLockedError(
390
+ `secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
391
+ );
392
+ }
393
+ yield* readEncryptedOfflineFileChunks({
394
+ filePath: options.filePath,
395
+ memoryDir: options.memoryDir,
396
+ key: options.secureStoreKey,
397
+ chunkSize: options.chunkSize
398
+ });
399
+ }
400
+ async function readFilePrefix(filePath, length) {
401
+ const handle = await fs2.promises.open(filePath, "r");
402
+ try {
403
+ const out = Buffer.alloc(length);
404
+ const { bytesRead } = await handle.read(out, 0, length, 0);
405
+ return out.subarray(0, bytesRead);
406
+ } finally {
407
+ await handle.close();
408
+ }
409
+ }
410
+ async function* readPlainOfflineFileChunks(filePath, chunkSize) {
411
+ const stream = fs2.createReadStream(filePath, { highWaterMark: chunkSize });
412
+ for await (const chunk of stream) {
413
+ yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
414
+ }
415
+ }
416
+ async function* readEncryptedOfflineFileChunks(options) {
417
+ const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
418
+ if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
419
+ throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
420
+ }
421
+ const version = header.readUInt8(MAGIC_BYTES.length);
422
+ const flags = header.readUInt8(MAGIC_BYTES.length + 1);
423
+ if (version !== FILE_FORMAT_VERSION) {
424
+ throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
425
+ }
426
+ if (flags !== FILE_FORMAT_FLAGS) {
427
+ throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
428
+ }
429
+ const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
430
+ const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
431
+ if (envelopeVersion !== ENVELOPE_VERSION) {
432
+ throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
433
+ }
434
+ const salt = envelopeHeader.subarray(
435
+ ENVELOPE_LAYOUT.salt,
436
+ ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
437
+ );
438
+ const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
439
+ const authTag = envelopeHeader.subarray(
440
+ ENVELOPE_LAYOUT.authTag,
441
+ ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
442
+ );
443
+ const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
444
+ let lastError;
445
+ for (const aad of aadCandidates) {
446
+ const tempDir = await mkdtemp(path.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
447
+ const tempPath = path.join(tempDir, "content");
448
+ try {
449
+ const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
450
+ authTagLength: AUTH_TAG_LENGTH
451
+ });
452
+ decipher.setAuthTag(authTag);
453
+ decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
454
+ const output = fs2.createWriteStream(tempPath, { mode: 384 });
455
+ try {
456
+ const stream = fs2.createReadStream(options.filePath, {
457
+ start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
458
+ highWaterMark: options.chunkSize
459
+ });
460
+ for await (const encryptedChunk of stream) {
461
+ const plain = decipher.update(
462
+ Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
463
+ );
464
+ if (plain.length > 0 && !output.write(plain)) {
465
+ await new Promise((resolve, reject) => {
466
+ output.once("drain", resolve);
467
+ output.once("error", reject);
468
+ });
469
+ }
470
+ }
471
+ const finalPlain = decipher.final();
472
+ if (finalPlain.length > 0 && !output.write(finalPlain)) {
473
+ await new Promise((resolve, reject) => {
474
+ output.once("drain", resolve);
475
+ output.once("error", reject);
476
+ });
477
+ }
478
+ } finally {
479
+ await closeWriteStream(output);
480
+ }
481
+ yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
482
+ return;
483
+ } catch (error) {
484
+ lastError = error;
485
+ } finally {
486
+ await rm(tempDir, { recursive: true, force: true });
487
+ }
488
+ }
489
+ throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
490
+ }
491
+ function offlineFileAadCandidates(filePath, memoryDir) {
492
+ const candidates = [filePathAad(filePath, memoryDir)];
493
+ const relative = path.relative(memoryDir, filePath);
494
+ if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) return candidates;
495
+ const parts = relative.split(path.sep);
496
+ if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
497
+ candidates.push(filePathAad(filePath, path.join(memoryDir, "namespaces", parts[1])));
498
+ }
499
+ const memoryParts = path.resolve(memoryDir).split(path.sep);
500
+ if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
501
+ const topLevelRoot = memoryParts.slice(0, -2).join(path.sep) || path.sep;
502
+ const topRelative = path.relative(topLevelRoot, filePath);
503
+ if (topRelative && !topRelative.startsWith("..") && !path.isAbsolute(topRelative) && topRelative.split(path.sep)[0] === "namespaces" && topRelative.split(path.sep)[1] === memoryParts.at(-1)) {
504
+ candidates.push(filePathAad(filePath, topLevelRoot));
505
+ }
506
+ }
507
+ return candidates;
508
+ }
509
+ async function closeWriteStream(stream) {
510
+ await new Promise((resolve, reject) => {
511
+ stream.once("error", reject);
512
+ stream.end(() => resolve());
513
+ });
514
+ }
515
+ function secureStoreEnvelopeHeaderAad(salt) {
516
+ const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
517
+ out.writeUInt8(ENVELOPE_VERSION, 0);
518
+ Buffer.from(salt).copy(out, 1);
519
+ return out;
520
+ }
521
+
522
+ // src/converge.ts
253
523
  import { resolveAgentAccessAuthToken } from "@remnic/core/resolve-auth-token.js";
254
524
  async function readLocalTombstones(rootDir) {
255
525
  const shaSet = /* @__PURE__ */ new Set();
256
526
  const candidates = [
257
- path.join(rootDir, "state", "tombstones.jsonl"),
258
- path.join(rootDir, "tombstones.jsonl")
527
+ path2.join(rootDir, "state", "tombstones.jsonl"),
528
+ path2.join(rootDir, "tombstones.jsonl")
259
529
  ];
260
530
  for (const tombPath of candidates) {
261
531
  try {
262
- const content = await fs2.promises.readFile(tombPath, "utf-8");
532
+ const content = await fs3.promises.readFile(tombPath, "utf-8");
263
533
  for (const line of content.split("\n")) {
264
534
  const trimmed = line.trim();
265
535
  if (!trimmed) continue;
@@ -289,44 +559,222 @@ async function fetchPeerSnapshot(peerUrl, namespace, token, fetchImpl = globalTh
289
559
  `/engram/v1/offline-sync/snapshot?namespace=${encodeURIComponent(namespace)}&content=false`
290
560
  ];
291
561
  const headers = token ? { authorization: `Bearer ${token}` } : {};
562
+ let lastFailure = "no snapshot route responded";
292
563
  for (const route of routes) {
564
+ let response;
293
565
  try {
294
- const res = await fetchImpl(`${base}${route}`, { headers });
295
- if (!res.ok) continue;
296
- const data = await res.json();
297
- const files = [];
298
- if (Array.isArray(data.files)) {
299
- for (const item of data.files) {
300
- if (item && typeof item.path === "string" && typeof item.sha256 === "string") {
301
- files.push({
302
- path: item.path,
303
- sha256: item.sha256,
304
- mtimeMs: typeof item.mtimeMs === "number" ? item.mtimeMs : void 0,
305
- bytes: typeof item.bytes === "number" ? item.bytes : void 0
306
- });
307
- }
308
- }
566
+ response = await fetchImpl(`${base}${route}`, { headers });
567
+ } catch (error) {
568
+ lastFailure = error instanceof Error ? error.message : String(error);
569
+ continue;
570
+ }
571
+ if (!response.ok) {
572
+ lastFailure = `HTTP ${response.status}`;
573
+ continue;
574
+ }
575
+ let data;
576
+ try {
577
+ data = await response.json();
578
+ } catch {
579
+ throw new Error(`invalid peer snapshot for namespace ${namespace}: response was not JSON`);
580
+ }
581
+ if (!data || typeof data !== "object" || !("files" in data) || !Array.isArray(data.files)) {
582
+ throw new Error(`invalid peer snapshot for namespace ${namespace}: files must be an array`);
583
+ }
584
+ const files = data.files.map((item, index) => {
585
+ if (!item || typeof item !== "object" || !("path" in item) || typeof item.path !== "string" || !("sha256" in item) || typeof item.sha256 !== "string") {
586
+ throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed file at index ${index}`);
309
587
  }
310
- const tombstones = /* @__PURE__ */ new Set();
311
- if (Array.isArray(data.tombstones)) {
312
- for (const tomb of data.tombstones) {
313
- if (typeof tomb === "string" && /^[0-9a-f]{64}$/i.test(tomb)) {
314
- tombstones.add(tomb.toLowerCase());
315
- }
588
+ return {
589
+ path: item.path,
590
+ sha256: item.sha256,
591
+ mtimeMs: "mtimeMs" in item && typeof item.mtimeMs === "number" ? item.mtimeMs : void 0,
592
+ bytes: "bytes" in item && typeof item.bytes === "number" ? item.bytes : void 0
593
+ };
594
+ });
595
+ const rawTombstones = "tombstones" in data ? data.tombstones : void 0;
596
+ if (rawTombstones !== void 0 && !Array.isArray(rawTombstones)) {
597
+ throw new Error(`invalid peer snapshot for namespace ${namespace}: tombstones must be an array`);
598
+ }
599
+ const tombstones = /* @__PURE__ */ new Set();
600
+ for (const tombstone of rawTombstones ?? []) {
601
+ if (typeof tombstone !== "string" || !/^[0-9a-f]{64}$/i.test(tombstone)) {
602
+ throw new Error(`invalid peer snapshot for namespace ${namespace}: malformed tombstone`);
603
+ }
604
+ tombstones.add(tombstone.toLowerCase());
605
+ }
606
+ return { files, tombstones };
607
+ }
608
+ throw new Error(`failed to fetch peer snapshot for namespace ${namespace}: ${lastFailure}`);
609
+ }
610
+ function requiredResponseNumber(response, name) {
611
+ const raw = response.headers.get(name);
612
+ const value = raw === null ? Number.NaN : Number(raw);
613
+ if (!Number.isFinite(value) || value < 0) {
614
+ throw new Error(`offline file content response had invalid ${name}`);
615
+ }
616
+ return value;
617
+ }
618
+ async function fetchPeerFileContent(peerUrl, namespace, filePath, token, fetchImpl = globalThis.fetch) {
619
+ const base = peerUrl.replace(/\/+$/, "");
620
+ const routes = [
621
+ "/remnic/v1/offline-sync/file-content",
622
+ "/engram/v1/offline-sync/file-content"
623
+ ];
624
+ const headers = {
625
+ "content-type": "application/json",
626
+ ...token ? { authorization: `Bearer ${token}` } : {}
627
+ };
628
+ for (const route of routes) {
629
+ try {
630
+ const chunks = [];
631
+ const hash = createHash2("sha256");
632
+ let offset = 0;
633
+ let expectedBytes;
634
+ let expectedSha256;
635
+ let mtimeMs;
636
+ do {
637
+ const response = await fetchImpl(`${base}${route}`, {
638
+ method: "POST",
639
+ headers,
640
+ body: JSON.stringify({
641
+ namespace,
642
+ includeTranscripts: false,
643
+ path: filePath,
644
+ offset,
645
+ length: OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES
646
+ })
647
+ });
648
+ if (!response.ok) throw new Error(`offline file content request failed: ${response.status}`);
649
+ const content = Buffer.from(await response.arrayBuffer());
650
+ const chunkOffset = requiredResponseNumber(response, "x-remnic-chunk-offset");
651
+ const chunkBytes = requiredResponseNumber(response, "x-remnic-chunk-bytes");
652
+ const totalBytes = requiredResponseNumber(response, "x-remnic-file-bytes");
653
+ const responseMtimeMs = requiredResponseNumber(response, "x-remnic-file-mtime-ms");
654
+ const sha256 = response.headers.get("x-remnic-file-sha256");
655
+ const encodedPath = response.headers.get("x-remnic-file-path");
656
+ if (!sha256 || chunkOffset !== offset || chunkBytes !== content.length || encodedPath !== null && decodeURIComponent(encodedPath) !== filePath || expectedBytes !== void 0 && expectedBytes !== totalBytes || expectedSha256 !== void 0 && expectedSha256 !== sha256) {
657
+ throw new Error(`offline file content response changed during transfer: ${filePath}`);
658
+ }
659
+ if (content.length === 0 && offset < totalBytes) {
660
+ throw new Error(`offline file content chunk was empty before EOF: ${filePath}`);
316
661
  }
662
+ expectedBytes = totalBytes;
663
+ expectedSha256 = sha256;
664
+ mtimeMs = responseMtimeMs;
665
+ chunks.push(content);
666
+ hash.update(content);
667
+ offset += content.length;
668
+ } while (expectedBytes === void 0 || offset < expectedBytes);
669
+ if (expectedBytes === void 0 || expectedSha256 === void 0 || mtimeMs === void 0 || offset !== expectedBytes || hash.digest("hex") !== expectedSha256) {
670
+ throw new Error(`offline file content checksum mismatch: ${filePath}`);
317
671
  }
318
- return { files, tombstones };
672
+ return {
673
+ content: Buffer.concat(chunks, expectedBytes),
674
+ sha256: expectedSha256,
675
+ bytes: expectedBytes,
676
+ mtimeMs
677
+ };
678
+ } catch {
679
+ }
680
+ }
681
+ return null;
682
+ }
683
+ function withoutTrailingSlashes(value) {
684
+ let end = value.length;
685
+ while (end > 0 && value.charCodeAt(end - 1) === 47) end -= 1;
686
+ return value.slice(0, end);
687
+ }
688
+ async function postPeerFileContent(peerUrl, namespace, filePath, content, metadata, token, fetchImpl = globalThis.fetch) {
689
+ const base = withoutTrailingSlashes(peerUrl);
690
+ const routes = [
691
+ `/remnic/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`,
692
+ `/engram/v1/offline-sync/apply-file-content?namespace=${encodeURIComponent(namespace)}`
693
+ ];
694
+ let previousAttemptFailed = false;
695
+ for (const route of routes) {
696
+ try {
697
+ let offset = 0;
698
+ do {
699
+ const chunk = content.subarray(
700
+ offset,
701
+ Math.min(content.length, offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2)
702
+ );
703
+ const headers = {
704
+ "content-type": "application/octet-stream",
705
+ "x-remnic-include-transcripts": "false",
706
+ "x-remnic-source-id": encodeURIComponent("remnic-converge"),
707
+ "x-remnic-file-path": encodeURIComponent(filePath),
708
+ "x-remnic-file-sha256": metadata.sha256,
709
+ "x-remnic-file-bytes": String(content.length),
710
+ "x-remnic-file-mtime-ms": String(metadata.mtimeMs),
711
+ "x-remnic-chunk-offset": String(offset),
712
+ ...metadata.baseSha256 ? { "x-remnic-base-sha256": metadata.baseSha256 } : {},
713
+ ...token ? { authorization: `Bearer ${token}` } : {}
714
+ };
715
+ const response = await fetchImpl(`${base}${route}`, {
716
+ method: "POST",
717
+ headers,
718
+ body: new Uint8Array(chunk)
719
+ });
720
+ if (!response.ok) throw new Error(`offline apply-file-content request failed: ${response.status}`);
721
+ const result = await response.json().catch(() => null);
722
+ if (!result || typeof result !== "object" || !("done" in result) || typeof result.done !== "boolean" || !("applied" in result) || typeof result.applied !== "boolean" || !("skipped" in result) || typeof result.skipped !== "boolean" || "conflict" in result && result.conflict) {
723
+ return false;
724
+ }
725
+ if (result.done) {
726
+ if (result.skipped) return previousAttemptFailed ? "applied" : "skipped";
727
+ if (result.applied && offset + chunk.length === content.length) return "applied";
728
+ return false;
729
+ }
730
+ if (result.applied || result.skipped || chunk.length === 0) {
731
+ return false;
732
+ }
733
+ offset += chunk.length;
734
+ } while (offset < content.length);
735
+ return false;
319
736
  } catch {
737
+ previousAttemptFailed = true;
320
738
  }
321
739
  }
322
- return { files: [], tombstones: /* @__PURE__ */ new Set() };
740
+ return false;
741
+ }
742
+ async function postPeerConvergenceComplete(peerUrl, namespaces, token, fetchImpl = globalThis.fetch) {
743
+ const base = withoutTrailingSlashes(peerUrl);
744
+ const query = namespaces.map((namespace) => `namespace=${encodeURIComponent(namespace)}`).join("&");
745
+ const routes = [
746
+ "/remnic/v1/offline-sync/convergence-complete",
747
+ "/engram/v1/offline-sync/convergence-complete"
748
+ ];
749
+ for (const route of routes) {
750
+ const response = await fetchImpl(`${base}${route}?${query}`, {
751
+ method: "POST",
752
+ headers: {
753
+ "x-remnic-source-id": encodeURIComponent("remnic-converge"),
754
+ ...token ? { authorization: `Bearer ${token}` } : {}
755
+ }
756
+ }).catch(() => null);
757
+ if (!response?.ok) continue;
758
+ const result = await response.json().catch(() => null);
759
+ if (result && typeof result === "object" && "namespaces" in result && Array.isArray(result.namespaces) && result.namespaces.length === namespaces.length && result.namespaces.every((namespace, index) => namespace === namespaces[index]) && "refreshed" in result && result.refreshed === true) {
760
+ return true;
761
+ }
762
+ }
763
+ return false;
323
764
  }
324
765
  async function computeConvergePlan(options = {}) {
766
+ const baseMap = /* @__PURE__ */ new Map();
325
767
  const namespacesToPlan = /* @__PURE__ */ new Set();
326
768
  const localMap = /* @__PURE__ */ new Map();
327
769
  const localTombstones = /* @__PURE__ */ new Map();
328
770
  const peerMap = /* @__PURE__ */ new Map();
329
771
  const peerTombstones = /* @__PURE__ */ new Map();
772
+ if (options.baseFilesByNamespace) {
773
+ for (const [ns, files] of options.baseFilesByNamespace) {
774
+ namespacesToPlan.add(ns);
775
+ baseMap.set(ns, files);
776
+ }
777
+ }
330
778
  if (options.localFilesByNamespace) {
331
779
  for (const [ns, files] of options.localFilesByNamespace) {
332
780
  namespacesToPlan.add(ns);
@@ -403,17 +851,328 @@ async function computeConvergePlan(options = {}) {
403
851
  peerTombstones.set(ns, peerData.tombstones);
404
852
  }
405
853
  }
854
+ const memoryDir = options.cursorDir ?? config?.memoryDir;
855
+ if (!options.baseFilesByNamespace && memoryDir && options.peerUrl) {
856
+ for (const ns of namespacesToPlan) {
857
+ const cursorPath = defaultConvergeCursorPath(memoryDir, options.peerUrl, ns);
858
+ const cursor = await readConvergeCursor(cursorPath);
859
+ if (cursor?.baseFiles && cursor.baseFiles.length > 0) {
860
+ baseMap.set(ns, cursor.baseFiles);
861
+ }
862
+ }
863
+ }
406
864
  const inputs = [];
407
865
  for (const ns of [...namespacesToPlan].sort()) {
408
866
  inputs.push({
409
867
  namespace: ns,
410
868
  local: localMap.get(ns) ?? [],
411
869
  peer: peerMap.get(ns) ?? [],
870
+ base: baseMap.get(ns),
412
871
  tombstonedFileSha256: localTombstones.get(ns) ?? [],
413
872
  peerTombstonedFileSha256: peerTombstones.get(ns) ?? []
414
873
  });
415
874
  }
416
- return planReconciliation(inputs);
875
+ return planReconciliation(inputs, { conflictPolicy: options.conflictPolicy });
876
+ }
877
+ async function executeConvergeApply(options = {}) {
878
+ const conflictPolicy = options.conflictPolicy ?? "manual";
879
+ const plan = await computeConvergePlan({ ...options, conflictPolicy });
880
+ if (plan.converged && !options.dryRun) {
881
+ await updateCursorsForPlan(plan, options);
882
+ return {
883
+ converged: true,
884
+ status: "converged",
885
+ plan,
886
+ transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
887
+ cursorUpdated: true
888
+ };
889
+ }
890
+ const unresolvedCount = plan.byNamespace.reduce((acc, report) => acc + report.unresolved, 0);
891
+ if (unresolvedCount > 0 && conflictPolicy === "manual") {
892
+ return {
893
+ converged: false,
894
+ status: "stopped_unresolved_conflicts",
895
+ plan,
896
+ transfers: { pulled: 0, pushed: 0, conflictsResolved: 0, suppressed: 0, failed: 0 },
897
+ cursorUpdated: false
898
+ };
899
+ }
900
+ const plannedTransfers = {
901
+ pulled: 0,
902
+ pushed: 0,
903
+ conflictsResolved: 0,
904
+ suppressed: 0,
905
+ failed: 0
906
+ };
907
+ for (const entry of plan.entries) {
908
+ if (entry.action === "pull") plannedTransfers.pulled += 1;
909
+ else if (entry.action === "push") plannedTransfers.pushed += 1;
910
+ else if (entry.action === "conflict") plannedTransfers.conflictsResolved += 1;
911
+ else if (entry.action === "suppress") plannedTransfers.suppressed += 1;
912
+ }
913
+ if (options.dryRun) {
914
+ return {
915
+ converged: false,
916
+ status: "dry_run",
917
+ plan,
918
+ transfers: plannedTransfers,
919
+ cursorUpdated: false
920
+ };
921
+ }
922
+ const actualTransfers = {
923
+ pulled: 0,
924
+ pushed: 0,
925
+ conflictsResolved: 0,
926
+ suppressed: 0,
927
+ failed: 0
928
+ };
929
+ const peerMutatedNamespaces = /* @__PURE__ */ new Set();
930
+ let resolvedToken;
931
+ if (options.peerToken) {
932
+ try {
933
+ resolvedToken = await resolveAgentAccessAuthToken(options.peerToken, {
934
+ resolveSecretRef: options.resolveSecretRef
935
+ });
936
+ } catch {
937
+ resolvedToken = options.peerToken;
938
+ }
939
+ }
940
+ const fetchFn = options.fetchImpl ?? globalThis.fetch;
941
+ let config = options.config;
942
+ if (!config) {
943
+ try {
944
+ config = parseConfig2({});
945
+ } catch {
946
+ }
947
+ }
948
+ const rootMap = /* @__PURE__ */ new Map();
949
+ if (config) {
950
+ try {
951
+ const roots = await resolveCorpusNamespaceRoots({ config });
952
+ for (const r of roots) {
953
+ rootMap.set(r.namespace, r.rootDir);
954
+ }
955
+ } catch {
956
+ }
957
+ }
958
+ for (const entry of plan.entries) {
959
+ if (entry.action === "identical") continue;
960
+ let transferType = "none";
961
+ if (entry.action === "pull") {
962
+ transferType = "pull";
963
+ } else if (entry.action === "push") {
964
+ transferType = "push";
965
+ } else if (entry.action === "suppress") {
966
+ transferType = "suppress";
967
+ } else if (entry.action === "conflict") {
968
+ if (entry.resolution === "peer-wins") {
969
+ transferType = "pull";
970
+ } else if (entry.resolution === "local-wins") {
971
+ transferType = "push";
972
+ } else if (entry.resolution === "supersede-link") {
973
+ if (entry.newerSide === "peer") transferType = "pull";
974
+ else if (entry.newerSide === "local") transferType = "push";
975
+ }
976
+ }
977
+ if (transferType === "pull") {
978
+ let remoteFile = null;
979
+ const buffered = options.peerFileBuffers?.get(entry.namespace)?.get(entry.path);
980
+ if (buffered) {
981
+ const state = options.peerFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path);
982
+ remoteFile = {
983
+ content: buffered,
984
+ sha256: state?.sha256 ?? entry.peerSha256 ?? createHash2("sha256").update(buffered).digest("hex"),
985
+ bytes: buffered.length,
986
+ mtimeMs: state?.mtimeMs ?? 0
987
+ };
988
+ } else if (options.peerUrl) {
989
+ remoteFile = await fetchPeerFileContent(
990
+ options.peerUrl,
991
+ entry.namespace,
992
+ entry.path,
993
+ resolvedToken,
994
+ fetchFn
995
+ );
996
+ }
997
+ if (remoteFile !== null && (!entry.peerSha256 || remoteFile.sha256 === entry.peerSha256)) {
998
+ if (options.localFileBuffers) {
999
+ let nsMap = options.localFileBuffers.get(entry.namespace);
1000
+ if (!nsMap) {
1001
+ nsMap = /* @__PURE__ */ new Map();
1002
+ options.localFileBuffers.set(entry.namespace, nsMap);
1003
+ }
1004
+ nsMap.set(entry.path, remoteFile.content);
1005
+ if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
1006
+ else actualTransfers.pulled += 1;
1007
+ } else {
1008
+ const rootDir = rootMap.get(entry.namespace);
1009
+ if (rootDir) {
1010
+ const io = await createOfflineStorageIo(rootDir);
1011
+ const expectedLocalSha256 = entry.action === "conflict" ? entry.localSha256 : entry.baseSha256;
1012
+ let offset = 0;
1013
+ let transferComplete = false;
1014
+ do {
1015
+ const chunk = remoteFile.content.subarray(
1016
+ offset,
1017
+ Math.min(
1018
+ remoteFile.content.length,
1019
+ offset + OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2
1020
+ )
1021
+ );
1022
+ const chunkResult = await applyOfflineSyncFileContentChunk({
1023
+ root: rootDir,
1024
+ sourceId: "remnic-converge",
1025
+ path: entry.path,
1026
+ sha256: remoteFile.sha256,
1027
+ bytes: remoteFile.bytes,
1028
+ mtimeMs: remoteFile.mtimeMs,
1029
+ offset,
1030
+ content: chunk,
1031
+ ...expectedLocalSha256 ? { baseSha256: expectedLocalSha256 } : {},
1032
+ readFile: io.readFile,
1033
+ readFileDigest: io.readFileDigest,
1034
+ writeFile: io.writeFile,
1035
+ writeStagingFile: io.writeStagingFile,
1036
+ writeFileChunks: io.writeFileChunks
1037
+ });
1038
+ if (chunkResult.conflict) {
1039
+ break;
1040
+ }
1041
+ if (chunkResult.done) {
1042
+ transferComplete = chunkResult.applied || chunkResult.skipped;
1043
+ break;
1044
+ }
1045
+ if (chunkResult.applied || chunkResult.skipped || chunk.length === 0) {
1046
+ break;
1047
+ }
1048
+ offset += chunk.length;
1049
+ } while (offset < remoteFile.content.length);
1050
+ if (transferComplete) {
1051
+ if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
1052
+ else actualTransfers.pulled += 1;
1053
+ } else {
1054
+ actualTransfers.failed += 1;
1055
+ }
1056
+ } else {
1057
+ actualTransfers.failed += 1;
1058
+ }
1059
+ }
1060
+ } else {
1061
+ actualTransfers.failed += 1;
1062
+ }
1063
+ } else if (transferType === "push") {
1064
+ let content = null;
1065
+ let mtimeMs = options.localFilesByNamespace?.get(entry.namespace)?.find((file) => file.path === entry.path)?.mtimeMs;
1066
+ if (options.localFileBuffers?.get(entry.namespace)?.has(entry.path)) {
1067
+ content = options.localFileBuffers.get(entry.namespace).get(entry.path);
1068
+ } else {
1069
+ const rootDir = rootMap.get(entry.namespace);
1070
+ if (rootDir) {
1071
+ const filePath = path2.join(rootDir, entry.path);
1072
+ try {
1073
+ const io = await createOfflineStorageIo(rootDir);
1074
+ content = await io.readFile({ root: rootDir, path: entry.path, filePath });
1075
+ mtimeMs ??= (await fs3.promises.stat(filePath)).mtimeMs;
1076
+ } catch {
1077
+ content = null;
1078
+ }
1079
+ }
1080
+ }
1081
+ if (content !== null) {
1082
+ if (options.peerFileBuffers) {
1083
+ let nsMap = options.peerFileBuffers.get(entry.namespace);
1084
+ if (!nsMap) {
1085
+ nsMap = /* @__PURE__ */ new Map();
1086
+ options.peerFileBuffers.set(entry.namespace, nsMap);
1087
+ }
1088
+ nsMap.set(entry.path, content);
1089
+ if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
1090
+ else actualTransfers.pushed += 1;
1091
+ } else if (options.peerUrl && entry.localSha256) {
1092
+ const expectedPeerSha256 = entry.action === "conflict" ? entry.peerSha256 : entry.baseSha256;
1093
+ const applied = await postPeerFileContent(
1094
+ options.peerUrl,
1095
+ entry.namespace,
1096
+ entry.path,
1097
+ content,
1098
+ {
1099
+ sha256: entry.localSha256,
1100
+ mtimeMs: mtimeMs ?? 0,
1101
+ ...expectedPeerSha256 ? { baseSha256: expectedPeerSha256 } : {}
1102
+ },
1103
+ resolvedToken,
1104
+ fetchFn
1105
+ );
1106
+ if (applied) {
1107
+ if (applied === "applied") peerMutatedNamespaces.add(entry.namespace);
1108
+ if (entry.action === "conflict") actualTransfers.conflictsResolved += 1;
1109
+ else actualTransfers.pushed += 1;
1110
+ } else {
1111
+ actualTransfers.failed += 1;
1112
+ }
1113
+ } else {
1114
+ actualTransfers.failed += 1;
1115
+ }
1116
+ } else {
1117
+ actualTransfers.failed += 1;
1118
+ }
1119
+ } else if (transferType === "suppress") {
1120
+ actualTransfers.suppressed += 1;
1121
+ }
1122
+ }
1123
+ if (options.peerUrl && peerMutatedNamespaces.size > 0) {
1124
+ const namespaces = [...peerMutatedNamespaces].sort();
1125
+ if (!await postPeerConvergenceComplete(
1126
+ options.peerUrl,
1127
+ namespaces,
1128
+ resolvedToken,
1129
+ fetchFn
1130
+ )) {
1131
+ actualTransfers.failed += 1;
1132
+ }
1133
+ }
1134
+ let cursorUpdated = false;
1135
+ if (actualTransfers.failed === 0) {
1136
+ await updateCursorsForPlan(plan, options);
1137
+ cursorUpdated = true;
1138
+ }
1139
+ return {
1140
+ converged: actualTransfers.failed === 0,
1141
+ status: "applied",
1142
+ plan,
1143
+ transfers: actualTransfers,
1144
+ cursorUpdated
1145
+ };
1146
+ }
1147
+ async function updateCursorsForPlan(plan, options) {
1148
+ const peerUrl = options.peerUrl ?? "local";
1149
+ let memoryDir;
1150
+ if (options.cursorDir) {
1151
+ memoryDir = options.cursorDir;
1152
+ } else if (options.config) {
1153
+ memoryDir = options.config.memoryDir;
1154
+ }
1155
+ if (!memoryDir) return;
1156
+ const namespaces = new Set(plan.byNamespace.map((n) => n.namespace));
1157
+ for (const ns of namespaces) {
1158
+ const cursorPath = defaultConvergeCursorPath(memoryDir, peerUrl, ns);
1159
+ const nsEntries = plan.entries.filter((e) => e.namespace === ns);
1160
+ const baseFiles = nsEntries.map((e) => ({
1161
+ path: e.path,
1162
+ sha256: e.localSha256 ?? e.peerSha256 ?? "unknown"
1163
+ }));
1164
+ const cursorState = {
1165
+ version: 1,
1166
+ peerUrl,
1167
+ namespace: ns,
1168
+ lastConvergedAt: (/* @__PURE__ */ new Date()).toISOString(),
1169
+ baseFiles
1170
+ };
1171
+ try {
1172
+ await writeConvergeCursor(cursorPath, cursorState);
1173
+ } catch {
1174
+ }
1175
+ }
417
1176
  }
418
1177
  function formatConvergeReport(plan) {
419
1178
  const lines = [];
@@ -435,25 +1194,49 @@ function formatConvergeReport(plan) {
435
1194
  }
436
1195
  return lines.join("\n");
437
1196
  }
1197
+ function formatConvergeApplyReport(result) {
1198
+ const lines = [];
1199
+ lines.push(`Convergence Execution Status: ${result.status.toUpperCase()}`);
1200
+ lines.push(`Converged: ${result.converged ? "YES" : "NO"}`);
1201
+ lines.push("");
1202
+ lines.push("Transfers Executed:");
1203
+ lines.push(` pulled: ${result.transfers.pulled}`);
1204
+ lines.push(` pushed: ${result.transfers.pushed}`);
1205
+ lines.push(` conflictsResolved: ${result.transfers.conflictsResolved}`);
1206
+ lines.push(` suppressed: ${result.transfers.suppressed}`);
1207
+ lines.push(` failed: ${result.transfers.failed}`);
1208
+ lines.push("");
1209
+ lines.push(formatConvergeReport(result.plan));
1210
+ return lines.join("\n");
1211
+ }
438
1212
  async function cmdConverge(action, rest, json) {
439
1213
  if (action === "help" || action === "--help" || action === "-h" || rest.includes("--help") || rest.includes("-h")) {
440
- console.log(`Usage: remnic converge plan [--peer <url>] [--token <token>] [--json]
1214
+ console.log(`Usage: remnic converge <plan|apply> [options]
1215
+
1216
+ Subcommands:
1217
+ plan Compute and display reconciliation plan
1218
+ apply Execute bidirectional converge transport (alias: transport, sync)
441
1219
 
442
1220
  Options:
443
1221
  --peer <url> Peer server URL (or --remote-url / --remote)
444
1222
  --token <token> Bearer token or SecretRef for peer authentication
1223
+ --conflict-policy <policy>
1224
+ Conflict resolution policy (manual|newest-wins|keep-both)
1225
+ --dry-run Simulate transfers without mutating disk or remote peer
445
1226
  --json Output detailed JSON plan report
446
1227
  `);
447
1228
  return;
448
1229
  }
449
- if (action !== "plan") {
450
- process.stderr.write(`converge: unknown action "${action}". Use: plan [options].
1230
+ if (action !== "plan" && action !== "apply" && action !== "transport" && action !== "sync") {
1231
+ process.stderr.write(`converge: unknown action "${action}". Use: plan or apply [options].
451
1232
  `);
452
1233
  process.exitCode = 2;
453
1234
  return;
454
1235
  }
455
1236
  let peerUrl;
456
1237
  let peerToken;
1238
+ let dryRun = false;
1239
+ let conflictPolicy;
457
1240
  for (let i = 0; i < rest.length; i += 1) {
458
1241
  const arg = rest[i];
459
1242
  if ((arg === "--peer" || arg === "--remote-url" || arg === "--remote") && rest[i + 1]) {
@@ -462,13 +1245,35 @@ Options:
462
1245
  } else if (arg === "--token" && rest[i + 1]) {
463
1246
  peerToken = rest[i + 1];
464
1247
  i += 1;
1248
+ } else if (arg === "--dry-run") {
1249
+ dryRun = true;
1250
+ } else if (arg === "--conflict-policy" && rest[i + 1]) {
1251
+ const pol = rest[i + 1];
1252
+ if (pol === "manual" || pol === "newest-wins" || pol === "keep-both") {
1253
+ conflictPolicy = pol;
1254
+ }
1255
+ i += 1;
465
1256
  }
466
1257
  }
467
- const plan = await computeConvergePlan({ peerUrl, peerToken });
1258
+ if (action === "plan") {
1259
+ const plan = await computeConvergePlan({ peerUrl, peerToken, conflictPolicy });
1260
+ if (json) {
1261
+ console.log(JSON.stringify(plan, null, 2));
1262
+ } else {
1263
+ console.log(formatConvergeReport(plan));
1264
+ }
1265
+ return;
1266
+ }
1267
+ const result = await executeConvergeApply({
1268
+ peerUrl,
1269
+ peerToken,
1270
+ dryRun,
1271
+ conflictPolicy
1272
+ });
468
1273
  if (json) {
469
- console.log(JSON.stringify(plan, null, 2));
1274
+ console.log(JSON.stringify(result, null, 2));
470
1275
  } else {
471
- console.log(formatConvergeReport(plan));
1276
+ console.log(formatConvergeApplyReport(result));
472
1277
  }
473
1278
  }
474
1279
 
@@ -605,7 +1410,7 @@ function renderReplayResult(result, targetNamespace, format) {
605
1410
  }
606
1411
 
607
1412
  // src/quarantine-replay.ts
608
- import * as fs3 from "fs";
1413
+ import * as fs4 from "fs";
609
1414
  import { EngramAccessService, Orchestrator as Orchestrator2, initLogger, parseConfig as parseConfig3, resolveRemnicConfigRecord as resolveRemnicConfigRecord2 } from "@remnic/core";
610
1415
  import { WriteQuarantineStore } from "@remnic/core/write-quarantine.js";
611
1416
  function valueFlag(args, flag) {
@@ -654,7 +1459,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
654
1459
  let orchestrator;
655
1460
  try {
656
1461
  const configPath = resolveConfigPath2();
657
- const raw = fs3.existsSync(configPath) ? JSON.parse(fs3.readFileSync(configPath, "utf8")) : {};
1462
+ const raw = fs4.existsSync(configPath) ? JSON.parse(fs4.readFileSync(configPath, "utf8")) : {};
658
1463
  const config = parseConfig3(resolveRemnicConfigRecord2(raw));
659
1464
  orchestrator = new Orchestrator2(config);
660
1465
  await orchestrator.initialize();
@@ -686,7 +1491,7 @@ async function runQuarantineReplay(rest, format, resolveConfigPath2) {
686
1491
  }
687
1492
 
688
1493
  // src/offline-impression-rotation.ts
689
- import fs4 from "fs";
1494
+ import fs5 from "fs";
690
1495
  import { parseConfig as parseConfig4, resolveRemnicConfigRecord as resolveRemnicConfigRecord3, drainPendingImpressionsForOfflineSync } from "@remnic/core";
691
1496
  import { LastRecallStore } from "@remnic/core/recall-state";
692
1497
  function parseConfigQuietly(raw) {
@@ -721,7 +1526,7 @@ function pickOfflineConfigRecord(raw) {
721
1526
  function resolveOfflineImpressionRotation(configPath) {
722
1527
  let raw;
723
1528
  try {
724
- raw = fs4.existsSync(configPath) ? JSON.parse(fs4.readFileSync(configPath, "utf8")) : {};
1529
+ raw = fs5.existsSync(configPath) ? JSON.parse(fs5.readFileSync(configPath, "utf8")) : {};
725
1530
  } catch {
726
1531
  throw new Error(
727
1532
  `cannot read recall-impression rotation from ${configPath}: config file could not be read as JSON`
@@ -749,265 +1554,6 @@ async function drainOfflineSyncImpressions(memoryDir, rotation) {
749
1554
  );
750
1555
  }
751
1556
 
752
- // src/offline-storage-io.ts
753
- import { mkdtemp, readdir, lstat, rm } from "fs/promises";
754
- import fs5 from "fs";
755
- import path2 from "path";
756
- import { createHash, createDecipheriv } from "crypto";
757
- import {
758
- OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
759
- StorageManager
760
- } from "@remnic/core";
761
- import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
762
- import {
763
- AUTH_TAG_LENGTH,
764
- ENVELOPE_HEADER_SIZE,
765
- ENVELOPE_LAYOUT,
766
- ENVELOPE_SALT_LENGTH,
767
- ENVELOPE_VERSION,
768
- FILE_FORMAT_FLAGS,
769
- FILE_FORMAT_VERSION,
770
- IV_LENGTH,
771
- MAGIC_BYTES,
772
- MAGIC_HEADER_SIZE,
773
- SecureStoreLockedError,
774
- filePathAad,
775
- isEncryptedFile,
776
- keyring,
777
- readHeader,
778
- secureStoreDir
779
- } from "@remnic/core/secure-store";
780
- async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
781
- const storage = new StorageManager(memoryDir);
782
- const header = await readHeader(memoryDir);
783
- let secureStoreKey = null;
784
- let secureStoreRequired = false;
785
- if (header) {
786
- secureStoreRequired = true;
787
- storage.setSecureStoreRequired(true);
788
- const key = keyring.getKey(secureStoreDir(memoryDir));
789
- if (key) {
790
- await storage.setSecureStoreKeyAndWait(key, secureStoreEncryptOnWrite);
791
- secureStoreKey = key;
792
- }
793
- }
794
- return { storage, secureStoreKey, secureStoreRequired };
795
- }
796
- async function createOfflineStorageForPath(memoryDir, filePath, configured, secureStoreEncryptOnWrite) {
797
- const memoryRoot = path2.resolve(memoryDir);
798
- const stateDir = path2.dirname(filePath);
799
- if (path2.basename(stateDir) !== "state" || path2.basename(filePath) !== "memory-lifecycle-ledger.jsonl") {
800
- throw new Error(`invalid lifecycle ledger path: ${filePath}`);
801
- }
802
- const storageRoot = path2.resolve(path2.dirname(stateDir));
803
- if (storageRoot !== memoryRoot && !storageRoot.startsWith(`${memoryRoot}${path2.sep}`)) {
804
- throw new Error(`lifecycle ledger path is outside the offline memory directory: ${filePath}`);
805
- }
806
- const storage = new StorageManager(storageRoot);
807
- if (configured.secureStoreRequired) {
808
- storage.setSecureStoreRequired(true);
809
- }
810
- if (configured.secureStoreKey) {
811
- await storage.setSecureStoreKeyAndWait(configured.secureStoreKey, secureStoreEncryptOnWrite);
812
- }
813
- return storage;
814
- }
815
- async function createOfflineStorageIo(memoryDir, configuredStorage) {
816
- await cleanupOrphanedOfflineDecryptStaging(memoryDir);
817
- const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
818
- return {
819
- readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
820
- readFileDigest: async ({ filePath }) => {
821
- const hash = createHash("sha256");
822
- let bytes = 0;
823
- for await (const rawChunk of readOfflineSyncFileChunks({
824
- filePath,
825
- memoryDir,
826
- secureStoreKey,
827
- chunkSize: OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES
828
- })) {
829
- const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
830
- hash.update(chunk);
831
- bytes += chunk.length;
832
- }
833
- return {
834
- sha256: hash.digest("hex"),
835
- bytes
836
- };
837
- },
838
- readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
839
- filePath,
840
- memoryDir,
841
- secureStoreKey,
842
- chunkSize
843
- }),
844
- writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
845
- writeStagingFile: async ({ filePath, content }) => storage.writeOfflineSyncStagingFile(filePath, content),
846
- writeFileChunks: async ({ filePath, chunks }) => storage.writeOfflineSyncFileChunks(filePath, chunks),
847
- deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
848
- };
849
- }
850
- var OFFLINE_DECRYPT_STAGING_ORPHAN_MS = 60 * 60 * 1e3;
851
- async function cleanupOrphanedOfflineDecryptStaging(memoryDir) {
852
- let entries;
853
- try {
854
- entries = await readdir(memoryDir);
855
- } catch {
856
- return;
857
- }
858
- const now = Date.now();
859
- for (const name of entries) {
860
- if (!name.startsWith(OFFLINE_DECRYPT_STAGING_DIR_PREFIX)) continue;
861
- const dir = path2.join(memoryDir, name);
862
- try {
863
- const info = await lstat(dir);
864
- if (!info.isDirectory() || info.isSymbolicLink()) continue;
865
- if (now - info.mtimeMs < OFFLINE_DECRYPT_STAGING_ORPHAN_MS) continue;
866
- await rm(dir, { recursive: true, force: true });
867
- } catch {
868
- }
869
- }
870
- }
871
- async function* readOfflineSyncFileChunks(options) {
872
- const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
873
- if (!isEncryptedFile(header)) {
874
- yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
875
- return;
876
- }
877
- if (!options.secureStoreKey) {
878
- throw new SecureStoreLockedError(
879
- `secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
880
- );
881
- }
882
- yield* readEncryptedOfflineFileChunks({
883
- filePath: options.filePath,
884
- memoryDir: options.memoryDir,
885
- key: options.secureStoreKey,
886
- chunkSize: options.chunkSize
887
- });
888
- }
889
- async function readFilePrefix(filePath, length) {
890
- const handle = await fs5.promises.open(filePath, "r");
891
- try {
892
- const out = Buffer.alloc(length);
893
- const { bytesRead } = await handle.read(out, 0, length, 0);
894
- return out.subarray(0, bytesRead);
895
- } finally {
896
- await handle.close();
897
- }
898
- }
899
- async function* readPlainOfflineFileChunks(filePath, chunkSize) {
900
- const stream = fs5.createReadStream(filePath, { highWaterMark: chunkSize });
901
- for await (const chunk of stream) {
902
- yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
903
- }
904
- }
905
- async function* readEncryptedOfflineFileChunks(options) {
906
- const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
907
- if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
908
- throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
909
- }
910
- const version = header.readUInt8(MAGIC_BYTES.length);
911
- const flags = header.readUInt8(MAGIC_BYTES.length + 1);
912
- if (version !== FILE_FORMAT_VERSION) {
913
- throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
914
- }
915
- if (flags !== FILE_FORMAT_FLAGS) {
916
- throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
917
- }
918
- const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
919
- const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
920
- if (envelopeVersion !== ENVELOPE_VERSION) {
921
- throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
922
- }
923
- const salt = envelopeHeader.subarray(
924
- ENVELOPE_LAYOUT.salt,
925
- ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
926
- );
927
- const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
928
- const authTag = envelopeHeader.subarray(
929
- ENVELOPE_LAYOUT.authTag,
930
- ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
931
- );
932
- const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
933
- let lastError;
934
- for (const aad of aadCandidates) {
935
- const tempDir = await mkdtemp(path2.join(options.memoryDir, OFFLINE_DECRYPT_STAGING_DIR_PREFIX));
936
- const tempPath = path2.join(tempDir, "content");
937
- try {
938
- const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
939
- authTagLength: AUTH_TAG_LENGTH
940
- });
941
- decipher.setAuthTag(authTag);
942
- decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), aad]));
943
- const output = fs5.createWriteStream(tempPath, { mode: 384 });
944
- try {
945
- const stream = fs5.createReadStream(options.filePath, {
946
- start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
947
- highWaterMark: options.chunkSize
948
- });
949
- for await (const encryptedChunk of stream) {
950
- const plain = decipher.update(
951
- Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
952
- );
953
- if (plain.length > 0 && !output.write(plain)) {
954
- await new Promise((resolve, reject) => {
955
- output.once("drain", resolve);
956
- output.once("error", reject);
957
- });
958
- }
959
- }
960
- const finalPlain = decipher.final();
961
- if (finalPlain.length > 0 && !output.write(finalPlain)) {
962
- await new Promise((resolve, reject) => {
963
- output.once("drain", resolve);
964
- output.once("error", reject);
965
- });
966
- }
967
- } finally {
968
- await closeWriteStream(output);
969
- }
970
- yield* readPlainOfflineFileChunks(tempPath, options.chunkSize);
971
- return;
972
- } catch (error) {
973
- lastError = error;
974
- } finally {
975
- await rm(tempDir, { recursive: true, force: true });
976
- }
977
- }
978
- throw lastError instanceof Error ? lastError : new Error(`secure-store could not decrypt file: ${options.filePath}`);
979
- }
980
- function offlineFileAadCandidates(filePath, memoryDir) {
981
- const candidates = [filePathAad(filePath, memoryDir)];
982
- const relative = path2.relative(memoryDir, filePath);
983
- if (!relative || relative.startsWith("..") || path2.isAbsolute(relative)) return candidates;
984
- const parts = relative.split(path2.sep);
985
- if (parts[0] === "namespaces" && parts.length >= 3 && parts[1]) {
986
- candidates.push(filePathAad(filePath, path2.join(memoryDir, "namespaces", parts[1])));
987
- }
988
- const memoryParts = path2.resolve(memoryDir).split(path2.sep);
989
- if (memoryParts.length >= 3 && memoryParts.at(-2) === "namespaces" && memoryParts.at(-1)) {
990
- const topLevelRoot = memoryParts.slice(0, -2).join(path2.sep) || path2.sep;
991
- const topRelative = path2.relative(topLevelRoot, filePath);
992
- if (topRelative && !topRelative.startsWith("..") && !path2.isAbsolute(topRelative) && topRelative.split(path2.sep)[0] === "namespaces" && topRelative.split(path2.sep)[1] === memoryParts.at(-1)) {
993
- candidates.push(filePathAad(filePath, topLevelRoot));
994
- }
995
- }
996
- return candidates;
997
- }
998
- async function closeWriteStream(stream) {
999
- await new Promise((resolve, reject) => {
1000
- stream.once("error", reject);
1001
- stream.end(() => resolve());
1002
- });
1003
- }
1004
- function secureStoreEnvelopeHeaderAad(salt) {
1005
- const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
1006
- out.writeUInt8(ENVELOPE_VERSION, 0);
1007
- Buffer.from(salt).copy(out, 1);
1008
- return out;
1009
- }
1010
-
1011
1557
  // src/bench-build-freshness.ts
1012
1558
  import {
1013
1559
  existsSync as existsSync2,
@@ -5892,7 +6438,7 @@ async function calibrateBenchJudges(parsed, rawArgs) {
5892
6438
  );
5893
6439
  process.exit(1);
5894
6440
  }
5895
- const sourceResultSha256 = createHash2("sha256").update(fs12.readFileSync(latest.path)).digest("hex");
6441
+ const sourceResultSha256 = createHash3("sha256").update(fs12.readFileSync(latest.path)).digest("hex");
5896
6442
  const expandedManifestPath = expandTilde(manifestPath);
5897
6443
  if (!bench.resolveLocalLabJudgeProviderConfig) {
5898
6444
  console.error(
@@ -6756,7 +7302,7 @@ function attachPreparedJudgeCalibration(result, judgeCalibration) {
6756
7302
  function hashCalibrationProviderConfig(config) {
6757
7303
  const canonicalize = (value, key = "") => {
6758
7304
  if (typeof value === "string" && /(?:api.?key|authorization|token|secret)/i.test(key)) {
6759
- return { secretSha256: createHash2("sha256").update(value).digest("hex") };
7305
+ return { secretSha256: createHash3("sha256").update(value).digest("hex") };
6760
7306
  }
6761
7307
  if (Array.isArray(value)) return value.map((item) => canonicalize(item));
6762
7308
  if (value && typeof value === "object") {
@@ -6767,7 +7313,7 @@ function hashCalibrationProviderConfig(config) {
6767
7313
  }
6768
7314
  return value;
6769
7315
  };
6770
- return createHash2("sha256").update(JSON.stringify(canonicalize(config))).digest("hex");
7316
+ return createHash3("sha256").update(JSON.stringify(canonicalize(config))).digest("hex");
6771
7317
  }
6772
7318
  function restoreOptionalEnv(key, previousValue) {
6773
7319
  if (previousValue === void 0) {
@@ -9199,7 +9745,7 @@ async function cmdSync(action, rest, json) {
9199
9745
  }
9200
9746
  function localOfflineSourceId(memoryDir) {
9201
9747
  const host = os.hostname() || "unknown-host";
9202
- const dirHash = createHash2("sha256").update(path15.resolve(memoryDir)).digest("hex").slice(0, 16);
9748
+ const dirHash = createHash3("sha256").update(path15.resolve(memoryDir)).digest("hex").slice(0, 16);
9203
9749
  return `remnic-local:${host}:${dirHash}`;
9204
9750
  }
9205
9751
  function normalizeOfflineRemoteUrl(raw) {
@@ -9799,7 +10345,7 @@ function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
9799
10345
  }
9800
10346
  return target;
9801
10347
  }
9802
- var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES2;
10348
+ var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES3;
9803
10349
  async function pushOfflineFileContent(args) {
9804
10350
  if (args.readFileChunks) {
9805
10351
  return pushOfflineFileContentFromChunkReader(args);
@@ -9807,7 +10353,7 @@ async function pushOfflineFileContent(args) {
9807
10353
  let offset = 0;
9808
10354
  let finalResult = null;
9809
10355
  let remoteSatisfiedResult = null;
9810
- const hash = createHash2("sha256");
10356
+ const hash = createHash3("sha256");
9811
10357
  let bytes = 0;
9812
10358
  while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
9813
10359
  const chunk = await readOfflineSyncFileContentChunk({
@@ -9868,7 +10414,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
9868
10414
  if (stat.mtimeMs !== args.file.mtimeMs) {
9869
10415
  throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
9870
10416
  }
9871
- const hash = createHash2("sha256");
10417
+ const hash = createHash3("sha256");
9872
10418
  const chunks = args.readFileChunks({
9873
10419
  root: path15.resolve(args.memoryDir),
9874
10420
  path: args.file.path,
@@ -9882,7 +10428,7 @@ async function pushOfflineFileContentFromChunkReader(args) {
9882
10428
  for await (const rawChunk of chunks) {
9883
10429
  const chunk = Buffer.from(rawChunk);
9884
10430
  if (chunk.length === 0) continue;
9885
- if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
10431
+ if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2) {
9886
10432
  throw new Error(`local offline content chunk exceeds max size: ${args.file.path}`);
9887
10433
  }
9888
10434
  if (pending) {
@@ -9967,7 +10513,7 @@ async function hydrateOfflineFileContent(args) {
9967
10513
  path: args.expected.path,
9968
10514
  offset,
9969
10515
  length: Math.min(
9970
- OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
10516
+ OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES2,
9971
10517
  Math.max(1, args.expected.bytes - offset)
9972
10518
  )
9973
10519
  });
@@ -9978,7 +10524,7 @@ async function hydrateOfflineFileContent(args) {
9978
10524
  throw new Error(`remote offline content chunk was empty before EOF: ${args.expected.path}`);
9979
10525
  }
9980
10526
  try {
9981
- finalResult = await applyOfflineSyncFileContentChunk({
10527
+ finalResult = await applyOfflineSyncFileContentChunk2({
9982
10528
  root: args.memoryDir,
9983
10529
  sourceId: args.sourceId,
9984
10530
  path: args.expected.path,