@remnic/core 1.1.21 → 1.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/access-cli.js +15 -15
  2. package/dist/access-http.d.ts +9 -1
  3. package/dist/access-http.js +9 -9
  4. package/dist/access-mcp.d.ts +1 -1
  5. package/dist/access-mcp.js +8 -8
  6. package/dist/access-schema.js +3 -3
  7. package/dist/{access-service-DT9L2DW4.d.ts → access-service-CEyV8XJ5.d.ts} +19 -2
  8. package/dist/access-service.d.ts +1 -1
  9. package/dist/access-service.js +6 -6
  10. package/dist/briefing.js +3 -3
  11. package/dist/causal-consolidation.js +4 -4
  12. package/dist/{chunk-YO3AZEE5.js → chunk-25YQM6XW.js} +3 -3
  13. package/dist/{chunk-TLM762GT.js → chunk-2WIPXV3Y.js} +2 -2
  14. package/dist/{chunk-QOHBYVZG.js → chunk-3F24QTRI.js} +2 -2
  15. package/dist/{chunk-5IQC4OG6.js → chunk-4H6DURG6.js} +2 -2
  16. package/dist/{chunk-NOQ74SJN.js → chunk-7D6O46PF.js} +2 -2
  17. package/dist/{chunk-SLKSC522.js → chunk-7E7SZRPP.js} +2 -2
  18. package/dist/{chunk-7Q2P774N.js → chunk-F33CJ5CH.js} +13 -3
  19. package/dist/chunk-F33CJ5CH.js.map +1 -0
  20. package/dist/{chunk-APW7AQOJ.js → chunk-FHXVW3L4.js} +4 -4
  21. package/dist/{chunk-PFFKUJM2.js → chunk-HWF42K6J.js} +103 -4
  22. package/dist/chunk-HWF42K6J.js.map +1 -0
  23. package/dist/{chunk-FSODDMR2.js → chunk-IANK6Y5W.js} +2 -2
  24. package/dist/{chunk-BYYIIXIJ.js → chunk-JKXFF3NT.js} +361 -29
  25. package/dist/chunk-JKXFF3NT.js.map +1 -0
  26. package/dist/{chunk-P7F6DJPA.js → chunk-MM5EBZVW.js} +42 -5
  27. package/dist/chunk-MM5EBZVW.js.map +1 -0
  28. package/dist/{chunk-GGCJ253V.js → chunk-MVAOT247.js} +8 -8
  29. package/dist/{chunk-SH5S7XYD.js → chunk-MXFBBHJU.js} +72 -2
  30. package/dist/chunk-MXFBBHJU.js.map +1 -0
  31. package/dist/{chunk-SZKCBLS5.js → chunk-PUXCIHRL.js} +2 -2
  32. package/dist/{chunk-2IRT26RZ.js → chunk-QYHQ2JHL.js} +2 -2
  33. package/dist/{chunk-73DAPA62.js → chunk-RA73CTVY.js} +12 -12
  34. package/dist/{chunk-CN4P6SVA.js → chunk-RCZRL5BE.js} +2 -2
  35. package/dist/{chunk-SGIXDVSF.js → chunk-S27EXIHY.js} +2 -2
  36. package/dist/{chunk-5ML4TH3E.js → chunk-TFORLO3O.js} +4 -4
  37. package/dist/{chunk-TOFUTKQN.js → chunk-TR4DK5OH.js} +2 -2
  38. package/dist/{chunk-6ORWKANA.js → chunk-VYU7PXUS.js} +2 -2
  39. package/dist/{chunk-FFU4GMST.js → chunk-WNARATI3.js} +2 -2
  40. package/dist/{chunk-KSFBM6TV.js → chunk-YITUHONZ.js} +2 -2
  41. package/dist/{cli-BN0CkYzI.d.ts → cli-BguVmIwO.d.ts} +1 -1
  42. package/dist/cli.d.ts +2 -2
  43. package/dist/cli.js +18 -18
  44. package/dist/compounding/engine.js +3 -3
  45. package/dist/connectors/codex-materialize-runner.js +3 -3
  46. package/dist/connectors/index.js +3 -3
  47. package/dist/entity-retrieval.js +3 -3
  48. package/dist/index.d.ts +4 -4
  49. package/dist/index.js +26 -24
  50. package/dist/index.js.map +1 -1
  51. package/dist/maintenance/memory-governance.js +3 -3
  52. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
  53. package/dist/maintenance/rebuild-memory-projection.js +4 -4
  54. package/dist/mcp-memory-inspector-app.d.ts +1 -1
  55. package/dist/namespaces/migrate.js +4 -4
  56. package/dist/namespaces/storage.js +3 -3
  57. package/dist/offline-sync.d.ts +36 -1
  58. package/dist/offline-sync.js +4 -2
  59. package/dist/operator-toolkit.js +6 -6
  60. package/dist/orchestrator.js +11 -11
  61. package/dist/schemas.d.ts +22 -22
  62. package/dist/secure-store/index.js +2 -2
  63. package/dist/semantic-consolidation.js +4 -4
  64. package/dist/semantic-rule-promotion.js +3 -3
  65. package/dist/semantic-rule-verifier.js +3 -3
  66. package/dist/storage.d.ts +2 -0
  67. package/dist/storage.js +2 -2
  68. package/dist/transfer/types.d.ts +12 -12
  69. package/dist/verified-recall.js +3 -3
  70. package/package.json +1 -1
  71. package/src/access-http.test.ts +176 -0
  72. package/src/access-http.ts +116 -0
  73. package/src/access-service-offline-file-content.test.ts +37 -0
  74. package/src/access-service.ts +70 -0
  75. package/src/index.ts +2 -0
  76. package/src/offline-sync.test.ts +448 -64
  77. package/src/offline-sync.ts +477 -29
  78. package/src/secure-store/secure-fs.ts +84 -3
  79. package/src/storage.ts +12 -0
  80. package/dist/chunk-7Q2P774N.js.map +0 -1
  81. package/dist/chunk-BYYIIXIJ.js.map +0 -1
  82. package/dist/chunk-P7F6DJPA.js.map +0 -1
  83. package/dist/chunk-PFFKUJM2.js.map +0 -1
  84. package/dist/chunk-SH5S7XYD.js.map +0 -1
  85. /package/dist/{chunk-YO3AZEE5.js.map → chunk-25YQM6XW.js.map} +0 -0
  86. /package/dist/{chunk-TLM762GT.js.map → chunk-2WIPXV3Y.js.map} +0 -0
  87. /package/dist/{chunk-QOHBYVZG.js.map → chunk-3F24QTRI.js.map} +0 -0
  88. /package/dist/{chunk-5IQC4OG6.js.map → chunk-4H6DURG6.js.map} +0 -0
  89. /package/dist/{chunk-NOQ74SJN.js.map → chunk-7D6O46PF.js.map} +0 -0
  90. /package/dist/{chunk-SLKSC522.js.map → chunk-7E7SZRPP.js.map} +0 -0
  91. /package/dist/{chunk-APW7AQOJ.js.map → chunk-FHXVW3L4.js.map} +0 -0
  92. /package/dist/{chunk-FSODDMR2.js.map → chunk-IANK6Y5W.js.map} +0 -0
  93. /package/dist/{chunk-GGCJ253V.js.map → chunk-MVAOT247.js.map} +0 -0
  94. /package/dist/{chunk-SZKCBLS5.js.map → chunk-PUXCIHRL.js.map} +0 -0
  95. /package/dist/{chunk-2IRT26RZ.js.map → chunk-QYHQ2JHL.js.map} +0 -0
  96. /package/dist/{chunk-73DAPA62.js.map → chunk-RA73CTVY.js.map} +0 -0
  97. /package/dist/{chunk-CN4P6SVA.js.map → chunk-RCZRL5BE.js.map} +0 -0
  98. /package/dist/{chunk-SGIXDVSF.js.map → chunk-S27EXIHY.js.map} +0 -0
  99. /package/dist/{chunk-5ML4TH3E.js.map → chunk-TFORLO3O.js.map} +0 -0
  100. /package/dist/{chunk-TOFUTKQN.js.map → chunk-TR4DK5OH.js.map} +0 -0
  101. /package/dist/{chunk-6ORWKANA.js.map → chunk-VYU7PXUS.js.map} +0 -0
  102. /package/dist/{chunk-FFU4GMST.js.map → chunk-WNARATI3.js.map} +0 -0
  103. /package/dist/{chunk-KSFBM6TV.js.map → chunk-YITUHONZ.js.map} +0 -0
@@ -6,6 +6,7 @@ import {
6
6
  readdir,
7
7
  readFile,
8
8
  rename,
9
+ rm,
9
10
  stat,
10
11
  unlink,
11
12
  writeFile,
@@ -130,6 +131,12 @@ export interface OfflineSyncFileWriteTarget extends OfflineSyncFileTarget {
130
131
  content: Buffer;
131
132
  }
132
133
 
134
+ export interface OfflineSyncFileWriteChunksTarget extends OfflineSyncFileTarget {
135
+ chunks: AsyncIterable<Buffer>;
136
+ }
137
+
138
+ export interface OfflineSyncFileStagingWriteTarget extends OfflineSyncFileWriteTarget {}
139
+
133
140
  export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState, "sha256"> {
134
141
  sha256?: string;
135
142
  offset: number;
@@ -137,6 +144,26 @@ export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState,
137
144
  content: Buffer;
138
145
  }
139
146
 
147
+ export interface OfflineSyncApplyFileContentChunkResult {
148
+ path: string;
149
+ sha256: string;
150
+ bytes: number;
151
+ mtimeMs: number;
152
+ offset: number;
153
+ chunkBytes: number;
154
+ done: boolean;
155
+ applied: boolean;
156
+ skipped: boolean;
157
+ conflict?: OfflineSyncConflict;
158
+ currentFile?: OfflineSyncFileState;
159
+ }
160
+
161
+ interface OfflineUploadStaging {
162
+ kind: "single" | "chunks";
163
+ relPath: string;
164
+ filePath: string;
165
+ }
166
+
140
167
  interface OfflineSyncFileRecordOptions {
141
168
  root: SafeArchiveRoot;
142
169
  relPath: string;
@@ -146,36 +173,11 @@ interface OfflineSyncFileRecordOptions {
146
173
  }
147
174
 
148
175
  const SYNC_INTERNAL_DIR = ".offline-sync";
176
+ const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
149
177
  const EXCLUDED_FILE_NAMES = new Set([
150
178
  ".sync-state.json",
151
179
  ]);
152
180
 
153
- const DERIVED_RUNTIME_REL_PATHS = new Set([
154
- "state/fact-hashes.ready",
155
- "state/fact-hashes.txt",
156
- "state/buffer-surprise-ledger.jsonl",
157
- "state/buffer.json",
158
- "state/embeddings.json",
159
- "state/index_tags.json",
160
- "state/index_time.json",
161
- "state/last_graph_recall.json",
162
- "state/last_intent.json",
163
- "state/last_qmd_recall.json",
164
- "state/last_recall.json",
165
- "state/lcm.sqlite",
166
- "state/lcm.sqlite-shm",
167
- "state/lcm.sqlite-wal",
168
- "state/memory-lifecycle-ledger.jsonl",
169
- "state/memory-projection.sqlite",
170
- "state/memory-projection.sqlite-shm",
171
- "state/memory-projection.sqlite-wal",
172
- "state/recall_impressions.jsonl",
173
- ]);
174
-
175
- const EXCLUDED_REL_PATHS = new Set([
176
- ...DERIVED_RUNTIME_REL_PATHS,
177
- ]);
178
-
179
181
  const EXCLUDED_FILE_PREFIXES = [
180
182
  ".remnic-sync.",
181
183
  ".remnic-sync-state.",
@@ -432,10 +434,9 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
432
434
  const parts = relPosix.split("/");
433
435
  if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
434
436
  if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
435
- if (EXCLUDED_REL_PATHS.has(relPosix)) return true;
436
437
  if (!includeTranscripts && parts[0] === "transcripts") return true;
437
438
  const basename = parts[parts.length - 1] ?? "";
438
- if (parts[0] === "state" && basename.includes(".tmp-")) return true;
439
+ if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
439
440
  if (EXCLUDED_FILE_NAMES.has(basename)) return true;
440
441
  return EXCLUDED_FILE_PREFIXES.some((prefix) => basename.startsWith(prefix));
441
442
  }
@@ -443,7 +444,12 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
443
444
  function shouldIgnoreIncomingRuntimePath(relPosix: string): boolean {
444
445
  const parts = relPosix.split("/");
445
446
  const basename = parts[parts.length - 1] ?? "";
446
- return DERIVED_RUNTIME_REL_PATHS.has(relPosix) || (parts[0] === "state" && basename.includes(".tmp-"));
447
+ return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
448
+ }
449
+
450
+ function isCanonicalRuntimeStatePath(parts: string[]): boolean {
451
+ if (parts[0] === "state") return true;
452
+ return parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state";
447
453
  }
448
454
 
449
455
  function filterBaseFilesForMode(
@@ -674,11 +680,15 @@ export async function buildOfflineSyncChangeset(options: {
674
680
  root: string;
675
681
  sourceId: string;
676
682
  baseFiles?: readonly OfflineSyncFileState[];
683
+ excludePaths?: readonly string[];
677
684
  includeTranscripts?: boolean;
678
685
  now?: Date;
679
686
  readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
680
687
  }): Promise<OfflineSyncChangeset> {
681
688
  const includeTranscripts = options.includeTranscripts !== false;
689
+ const excludedPaths = new Set(
690
+ (options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]")),
691
+ );
682
692
  const base = byPath(filterBaseFilesForMode(
683
693
  normalizeFileStates(options.baseFiles),
684
694
  includeTranscripts,
@@ -695,6 +705,7 @@ export async function buildOfflineSyncChangeset(options: {
695
705
  const changes: OfflineSyncChange[] = [];
696
706
 
697
707
  for (const relPath of unionPaths(base, currentMap)) {
708
+ if (excludedPaths.has(relPath)) continue;
698
709
  const baseEntry = base.get(relPath);
699
710
  const currentEntry = currentMap.get(relPath);
700
711
  if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
@@ -1139,6 +1150,443 @@ async function writeSafeFile(
1139
1150
  }
1140
1151
  }
1141
1152
 
1153
+ export async function applyOfflineSyncFileContentChunk(options: {
1154
+ root: string;
1155
+ sourceId: string;
1156
+ path: string;
1157
+ sha256: string;
1158
+ bytes: number;
1159
+ mtimeMs: number;
1160
+ offset?: number;
1161
+ content: Buffer;
1162
+ baseSha256?: string;
1163
+ includeTranscripts?: boolean;
1164
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1165
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1166
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1167
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
1168
+ }): Promise<OfflineSyncApplyFileContentChunkResult> {
1169
+ const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
1170
+ const sourceId = normalizeSourceId(options.sourceId, "sourceId");
1171
+ const relPath = normalizeRelativePath(options.path, "path");
1172
+ const includeTranscripts = options.includeTranscripts !== false;
1173
+ if (shouldExcludeRelPath(relPath, includeTranscripts)) {
1174
+ throw new Error(`offline sync file content path is excluded: ${relPath}`);
1175
+ }
1176
+ const sha256 = assertSha256(options.sha256, "sha256");
1177
+ const bytes = assertNonNegativeInteger(options.bytes, "bytes");
1178
+ const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
1179
+ const offset = options.offset === undefined
1180
+ ? 0
1181
+ : assertNonNegativeInteger(options.offset, "offset");
1182
+ const baseSha256 = options.baseSha256 === undefined
1183
+ ? undefined
1184
+ : assertSha256(options.baseSha256, "baseSha256");
1185
+ if (!Buffer.isBuffer(options.content)) {
1186
+ throw new Error("content must be a Buffer");
1187
+ }
1188
+ if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
1189
+ throw new Error(
1190
+ `content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`,
1191
+ );
1192
+ }
1193
+ if (bytes > 0 && options.content.length === 0) {
1194
+ throw new Error("content chunk must be non-empty before EOF");
1195
+ }
1196
+ if (offset > bytes || offset + options.content.length > bytes) {
1197
+ throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
1198
+ }
1199
+ if (options.writeFile && !options.writeFileChunks) {
1200
+ throw new Error("offline sync upload storage hooks require writeFileChunks");
1201
+ }
1202
+ if (options.writeFile && !options.writeStagingFile) {
1203
+ throw new Error("offline sync upload storage hooks require writeStagingFile");
1204
+ }
1205
+ if (offset === 0) {
1206
+ await pruneOfflineUploadStaging(root);
1207
+ }
1208
+
1209
+ const upload = await writeOfflineUploadChunk({
1210
+ root,
1211
+ sourceId,
1212
+ relPath,
1213
+ sha256,
1214
+ bytes,
1215
+ offset,
1216
+ content: options.content,
1217
+ readFile: options.readFile,
1218
+ writeFile: options.writeFile,
1219
+ writeStagingFile: options.writeStagingFile,
1220
+ });
1221
+ const done = offset + options.content.length === bytes;
1222
+ const baseResult = {
1223
+ path: relPath,
1224
+ sha256,
1225
+ bytes,
1226
+ mtimeMs,
1227
+ offset,
1228
+ chunkBytes: options.content.length,
1229
+ done,
1230
+ };
1231
+ if (!done) {
1232
+ return {
1233
+ ...baseResult,
1234
+ applied: false,
1235
+ skipped: false,
1236
+ };
1237
+ }
1238
+
1239
+ const digest = await digestOfflineUploadStagingContent({
1240
+ root,
1241
+ upload,
1242
+ readFile: options.readFile,
1243
+ });
1244
+ if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
1245
+ await cleanupOfflineUpload(upload).catch(() => {});
1246
+ throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
1247
+ }
1248
+
1249
+ const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
1250
+ root: root.abs,
1251
+ sourceId: "local",
1252
+ paths: [relPath],
1253
+ includeContent: false,
1254
+ includeTranscripts,
1255
+ readFile: options.readFile,
1256
+ });
1257
+ const currentFile = currentSnapshot.files[0];
1258
+ const uploadedState: OfflineSyncFileState = {
1259
+ path: relPath,
1260
+ sha256,
1261
+ bytes,
1262
+ mtimeMs,
1263
+ };
1264
+
1265
+ try {
1266
+ if (currentFile?.sha256 === sha256) {
1267
+ return {
1268
+ ...baseResult,
1269
+ applied: false,
1270
+ skipped: true,
1271
+ currentFile: toFileState(currentFile),
1272
+ };
1273
+ }
1274
+ if (!baseSha256 && currentFile) {
1275
+ const conflict = await recordConflict({
1276
+ root,
1277
+ relPath,
1278
+ reason: "remote_exists_for_local_create",
1279
+ localSha256: currentFile.sha256,
1280
+ incomingSha256: sha256,
1281
+ writeConflictCopies: false,
1282
+ sourceId,
1283
+ writeFile: options.writeFile,
1284
+ });
1285
+ return {
1286
+ ...baseResult,
1287
+ applied: false,
1288
+ skipped: false,
1289
+ conflict,
1290
+ currentFile: toFileState(currentFile),
1291
+ };
1292
+ }
1293
+ if (baseSha256 && currentFile?.sha256 !== baseSha256) {
1294
+ const conflict = await recordConflict({
1295
+ root,
1296
+ relPath,
1297
+ reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
1298
+ baseSha256,
1299
+ localSha256: currentFile?.sha256,
1300
+ incomingSha256: sha256,
1301
+ writeConflictCopies: false,
1302
+ sourceId,
1303
+ writeFile: options.writeFile,
1304
+ });
1305
+ return {
1306
+ ...baseResult,
1307
+ applied: false,
1308
+ skipped: false,
1309
+ conflict,
1310
+ ...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
1311
+ };
1312
+ }
1313
+
1314
+ await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
1315
+ return {
1316
+ ...baseResult,
1317
+ applied: true,
1318
+ skipped: false,
1319
+ currentFile: uploadedState,
1320
+ };
1321
+ } finally {
1322
+ await cleanupOfflineUpload(upload).catch(() => {});
1323
+ }
1324
+ }
1325
+
1326
+ function offlineUploadRelPath(options: {
1327
+ sourceId: string;
1328
+ relPath: string;
1329
+ sha256: string;
1330
+ bytes: number;
1331
+ }): string {
1332
+ const key = hashText([
1333
+ options.sourceId,
1334
+ options.relPath,
1335
+ options.sha256,
1336
+ String(options.bytes),
1337
+ ].join("\0"));
1338
+ return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
1339
+ }
1340
+
1341
+ async function offlineUploadPath(root: SafeArchiveRoot, options: {
1342
+ sourceId: string;
1343
+ relPath: string;
1344
+ sha256: string;
1345
+ bytes: number;
1346
+ }): Promise<OfflineUploadStaging> {
1347
+ const relPath = offlineUploadRelPath(options);
1348
+ return {
1349
+ kind: "single",
1350
+ relPath,
1351
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1352
+ };
1353
+ }
1354
+
1355
+ async function offlineUploadChunkPath(root: SafeArchiveRoot, options: {
1356
+ sourceId: string;
1357
+ relPath: string;
1358
+ sha256: string;
1359
+ bytes: number;
1360
+ offset: number;
1361
+ }): Promise<OfflineUploadStaging> {
1362
+ const uploadRelPath = offlineUploadRelPath(options);
1363
+ const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
1364
+ return {
1365
+ kind: "chunks",
1366
+ relPath,
1367
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1368
+ };
1369
+ }
1370
+
1371
+ async function writeOfflineUploadChunk(options: {
1372
+ root: SafeArchiveRoot;
1373
+ sourceId: string;
1374
+ relPath: string;
1375
+ sha256: string;
1376
+ bytes: number;
1377
+ offset: number;
1378
+ content: Buffer;
1379
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1380
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1381
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1382
+ }): Promise<OfflineUploadStaging> {
1383
+ if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
1384
+ throw new Error("offline sync upload chunk storage hooks require readFile");
1385
+ }
1386
+ const uploadRoot = {
1387
+ ...(await offlineUploadPath(options.root, options)),
1388
+ kind: "chunks" as const,
1389
+ };
1390
+ if (options.offset === 0) {
1391
+ await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {});
1392
+ } else {
1393
+ const existing = await stat(uploadRoot.filePath).catch((error: unknown) => {
1394
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1395
+ throw error;
1396
+ });
1397
+ if (!existing || !existing.isDirectory()) {
1398
+ throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
1399
+ }
1400
+ }
1401
+ const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
1402
+
1403
+ const writeStagingFile = options.writeStagingFile ?? options.writeFile;
1404
+ if (writeStagingFile) {
1405
+ // Storage-backed services provide these hooks so secure-store deployments
1406
+ // keep staged partial uploads encrypted at rest without mutating indexes.
1407
+ await writeOfflineUploadContent({
1408
+ root: options.root,
1409
+ relPath: chunk.relPath,
1410
+ filePath: chunk.filePath,
1411
+ content: options.content,
1412
+ writeFile: writeStagingFile,
1413
+ });
1414
+ return uploadRoot;
1415
+ }
1416
+
1417
+ await mkdir(path.dirname(chunk.filePath), { recursive: true });
1418
+ const existingChunk = await lstat(chunk.filePath).catch((error: unknown) => {
1419
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1420
+ throw error;
1421
+ });
1422
+ if (existingChunk?.isSymbolicLink()) {
1423
+ throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
1424
+ }
1425
+ await writeFile(chunk.filePath, options.content, { mode: 0o600 });
1426
+ return uploadRoot;
1427
+ }
1428
+
1429
+ async function pruneOfflineUploadStaging(root: SafeArchiveRoot): Promise<void> {
1430
+ const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
1431
+ const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
1432
+ const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error: unknown) => {
1433
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
1434
+ throw error;
1435
+ });
1436
+ const now = Date.now();
1437
+ await Promise.all(entries.map(async (entry) => {
1438
+ if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
1439
+ const relPath = `${uploadsRelPath}/${entry.name}`;
1440
+ const filePath = await resolveSafeArchiveTarget(root, relPath);
1441
+ const info = await lstat(filePath).catch((error: unknown) => {
1442
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1443
+ throw error;
1444
+ });
1445
+ if (!info) return;
1446
+ if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
1447
+ await rm(filePath, { recursive: true, force: true });
1448
+ }));
1449
+ }
1450
+
1451
+ async function* readOfflineUploadStagingChunks(options: {
1452
+ root: SafeArchiveRoot;
1453
+ upload: OfflineUploadStaging;
1454
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1455
+ }): AsyncGenerator<Buffer> {
1456
+ if (options.upload.kind === "single") {
1457
+ yield await readOfflineUploadContent({
1458
+ root: options.root,
1459
+ relPath: options.upload.relPath,
1460
+ filePath: options.upload.filePath,
1461
+ readFile: options.readFile,
1462
+ });
1463
+ return;
1464
+ }
1465
+
1466
+ const entries = await readdir(options.upload.filePath);
1467
+ const chunkNames = entries
1468
+ .filter((entry) => /^\d{20}\.part$/.test(entry))
1469
+ .sort();
1470
+ if (chunkNames.length === 0) {
1471
+ throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
1472
+ }
1473
+ let expectedOffset = 0;
1474
+ for (const chunkName of chunkNames) {
1475
+ const offset = Number(chunkName.slice(0, 20));
1476
+ if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
1477
+ throw new Error(
1478
+ `offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`,
1479
+ );
1480
+ }
1481
+ const relPath = `${options.upload.relPath}/${chunkName}`;
1482
+ const filePath = await resolveSafeArchiveTarget(options.root, relPath);
1483
+ const content = await readOfflineUploadContent({
1484
+ root: options.root,
1485
+ relPath,
1486
+ filePath,
1487
+ readFile: options.readFile,
1488
+ });
1489
+ expectedOffset += content.length;
1490
+ yield content;
1491
+ }
1492
+ }
1493
+
1494
+ async function digestOfflineUploadStagingContent(options: {
1495
+ root: SafeArchiveRoot;
1496
+ upload: OfflineUploadStaging;
1497
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1498
+ }): Promise<{ sha256: string; bytes: number }> {
1499
+ const hash = createHash("sha256");
1500
+ let bytes = 0;
1501
+ for await (const chunk of readOfflineUploadStagingChunks(options)) {
1502
+ hash.update(chunk);
1503
+ bytes += chunk.length;
1504
+ }
1505
+ return { sha256: hash.digest("hex"), bytes };
1506
+ }
1507
+
1508
+ async function writeSafeFileFromUpload(
1509
+ root: SafeArchiveRoot,
1510
+ relPath: string,
1511
+ upload: OfflineUploadStaging,
1512
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
1513
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
1514
+ ): Promise<void> {
1515
+ const target = await resolveSafeArchiveTarget(root, relPath);
1516
+ const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
1517
+ if (writeFileChunks) {
1518
+ await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
1519
+ return;
1520
+ }
1521
+
1522
+ await mkdir(path.dirname(target), { recursive: true });
1523
+ const tmp = path.join(
1524
+ path.dirname(target),
1525
+ `.remnic-sync.${process.pid}.${randomUUID()}.tmp`,
1526
+ );
1527
+ const handle = await open(tmp, "w", 0o600);
1528
+ try {
1529
+ for await (const chunk of chunks) {
1530
+ if (chunk.length > 0) await handle.write(chunk);
1531
+ }
1532
+ await handle.close();
1533
+ const targetStat = await lstat(target).catch((error: unknown) => {
1534
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1535
+ throw error;
1536
+ });
1537
+ if (targetStat?.isSymbolicLink()) {
1538
+ throw new Error(`offline sync target is a symlink: ${relPath}`);
1539
+ }
1540
+ await rename(tmp, target);
1541
+ } catch (error) {
1542
+ await handle.close().catch(() => {});
1543
+ await unlink(tmp).catch(() => {});
1544
+ throw error;
1545
+ }
1546
+ }
1547
+
1548
+ async function cleanupOfflineUpload(upload: OfflineUploadStaging): Promise<void> {
1549
+ if (upload.kind === "chunks") {
1550
+ await rm(upload.filePath, { recursive: true, force: true });
1551
+ return;
1552
+ }
1553
+ await unlink(upload.filePath).catch((error: unknown) => {
1554
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
1555
+ throw error;
1556
+ });
1557
+ }
1558
+
1559
+ async function readOfflineUploadContent(options: {
1560
+ root: SafeArchiveRoot;
1561
+ relPath: string;
1562
+ filePath: string;
1563
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1564
+ }): Promise<Buffer> {
1565
+ if (options.readFile) {
1566
+ return options.readFile({
1567
+ root: options.root.abs,
1568
+ path: options.relPath,
1569
+ filePath: options.filePath,
1570
+ });
1571
+ }
1572
+ return readFile(options.filePath);
1573
+ }
1574
+
1575
+ async function writeOfflineUploadContent(options: {
1576
+ root: SafeArchiveRoot;
1577
+ relPath: string;
1578
+ filePath: string;
1579
+ content: Buffer;
1580
+ writeFile: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1581
+ }): Promise<void> {
1582
+ await options.writeFile({
1583
+ root: options.root.abs,
1584
+ path: options.relPath,
1585
+ filePath: options.filePath,
1586
+ content: options.content,
1587
+ });
1588
+ }
1589
+
1142
1590
  async function deleteSafeFile(
1143
1591
  root: SafeArchiveRoot,
1144
1592
  relPath: string,
@@ -43,10 +43,21 @@
43
43
  * Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
44
44
  */
45
45
 
46
- import { lstat, mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
46
+ import { createCipheriv, randomBytes } from "node:crypto";
47
+ import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
47
48
  import path from "node:path";
48
49
 
49
- import { generateSalt, open, seal } from "./cipher.js";
50
+ import {
51
+ AUTH_TAG_LENGTH,
52
+ ENVELOPE_HEADER_SIZE,
53
+ ENVELOPE_LAYOUT,
54
+ ENVELOPE_SALT_LENGTH,
55
+ ENVELOPE_VERSION,
56
+ IV_LENGTH,
57
+ generateSalt,
58
+ open as openEnvelope,
59
+ seal,
60
+ } from "./cipher.js";
50
61
 
51
62
  // ---------------------------------------------------------------------------
52
63
  // Error classes
@@ -157,7 +168,7 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
157
168
  }
158
169
  const envelope = buf.subarray(MAGIC_HEADER_SIZE);
159
170
  try {
160
- return open(key, envelope, aad ? { aad } : {});
171
+ return openEnvelope(key, envelope, aad ? { aad } : {});
161
172
  } catch (err) {
162
173
  const msg = err instanceof Error ? err.message : String(err);
163
174
  throw new SecureStoreDecryptError(
@@ -166,6 +177,13 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
166
177
  }
167
178
  }
168
179
 
180
+ function buildHeaderAad(salt: Uint8Array): Buffer {
181
+ const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
182
+ out.writeUInt8(ENVELOPE_VERSION, 0);
183
+ Buffer.from(salt).copy(out, 1);
184
+ return out;
185
+ }
186
+
169
187
  // ---------------------------------------------------------------------------
170
188
  // Path → AAD helper
171
189
  // ---------------------------------------------------------------------------
@@ -293,6 +311,69 @@ export async function writeMaybeEncryptedFile(
293
311
  }
294
312
  }
295
313
 
314
+ export async function writeMaybeEncryptedFileFromChunks(
315
+ filePath: string,
316
+ chunks: AsyncIterable<Buffer>,
317
+ key: Buffer | null,
318
+ options: WriteMaybeEncryptedFileOptions = {},
319
+ memoryDir?: string,
320
+ ): Promise<void> {
321
+ const { mode = 0o600, atomic = true } = options;
322
+ await mkdir(path.dirname(filePath), { recursive: true });
323
+ const writePath = atomic ? `${filePath}.tmp-${process.pid}-${Date.now()}` : filePath;
324
+ let completed = false;
325
+ try {
326
+ const handle = await openFile(writePath, "w", mode);
327
+ try {
328
+ if (key !== null) {
329
+ const salt = generateSalt();
330
+ const iv = randomBytes(IV_LENGTH);
331
+ const header = Buffer.alloc(MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
332
+ MAGIC_BYTES.copy(header, 0);
333
+ header.writeUInt8(FILE_FORMAT_VERSION, MAGIC_BYTES.length);
334
+ header.writeUInt8(FILE_FORMAT_FLAGS, MAGIC_BYTES.length + 1);
335
+ const envelopeOffset = MAGIC_HEADER_SIZE;
336
+ header.writeUInt8(ENVELOPE_VERSION, envelopeOffset + ENVELOPE_LAYOUT.version);
337
+ salt.copy(header, envelopeOffset + ENVELOPE_LAYOUT.salt);
338
+ iv.copy(header, envelopeOffset + ENVELOPE_LAYOUT.iv);
339
+ await handle.write(header);
340
+
341
+ const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
342
+ const aad = filePathAad(filePath, memoryDir);
343
+ cipher.setAAD(Buffer.concat([buildHeaderAad(salt), aad]));
344
+ for await (const chunk of chunks) {
345
+ if (chunk.length === 0) continue;
346
+ const encrypted = cipher.update(chunk);
347
+ if (encrypted.length > 0) await handle.write(encrypted);
348
+ }
349
+ const final = cipher.final();
350
+ if (final.length > 0) await handle.write(final);
351
+ const authTag = cipher.getAuthTag();
352
+ await handle.write(
353
+ authTag,
354
+ 0,
355
+ authTag.length,
356
+ MAGIC_HEADER_SIZE + ENVELOPE_LAYOUT.authTag,
357
+ );
358
+ } else {
359
+ for await (const chunk of chunks) {
360
+ if (chunk.length > 0) await handle.write(chunk);
361
+ }
362
+ }
363
+ } finally {
364
+ await handle.close();
365
+ }
366
+ if (atomic) {
367
+ await rename(writePath, filePath);
368
+ }
369
+ completed = true;
370
+ } finally {
371
+ if (!completed && atomic) {
372
+ await unlink(writePath).catch(() => {});
373
+ }
374
+ }
375
+ }
376
+
296
377
  // ---------------------------------------------------------------------------
297
378
  // Migration
298
379
  // ---------------------------------------------------------------------------