@remnic/core 1.1.22 → 1.1.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/access-cli.js +15 -15
  2. package/dist/access-http.d.ts +9 -1
  3. package/dist/access-http.js +9 -9
  4. package/dist/access-mcp.d.ts +1 -1
  5. package/dist/access-mcp.js +8 -8
  6. package/dist/access-schema.js +3 -3
  7. package/dist/{access-service-DT9L2DW4.d.ts → access-service-CEyV8XJ5.d.ts} +19 -2
  8. package/dist/access-service.d.ts +1 -1
  9. package/dist/access-service.js +6 -6
  10. package/dist/briefing.js +3 -3
  11. package/dist/causal-consolidation.js +4 -4
  12. package/dist/{chunk-YO3AZEE5.js → chunk-25YQM6XW.js} +3 -3
  13. package/dist/{chunk-LDJANWTK.js → chunk-2DM72JF3.js} +12 -12
  14. package/dist/{chunk-TLM762GT.js → chunk-2WIPXV3Y.js} +2 -2
  15. package/dist/{chunk-QOHBYVZG.js → chunk-3F24QTRI.js} +2 -2
  16. package/dist/{chunk-5IQC4OG6.js → chunk-4H6DURG6.js} +2 -2
  17. package/dist/{chunk-26OQECWH.js → chunk-6CB4E7ZV.js} +4 -4
  18. package/dist/{chunk-NOQ74SJN.js → chunk-7D6O46PF.js} +2 -2
  19. package/dist/{chunk-FF46Q3SN.js → chunk-AMVN77EU.js} +360 -32
  20. package/dist/chunk-AMVN77EU.js.map +1 -0
  21. package/dist/{chunk-7Q2P774N.js → chunk-F33CJ5CH.js} +13 -3
  22. package/dist/chunk-F33CJ5CH.js.map +1 -0
  23. package/dist/{chunk-FSODDMR2.js → chunk-IANK6Y5W.js} +2 -2
  24. package/dist/{chunk-UA6OCL6S.js → chunk-JUYT2J3K.js} +106 -11
  25. package/dist/chunk-JUYT2J3K.js.map +1 -0
  26. package/dist/{chunk-NGPO6S3M.js → chunk-LCTP7YRU.js} +42 -5
  27. package/dist/chunk-LCTP7YRU.js.map +1 -0
  28. package/dist/{chunk-GGCJ253V.js → chunk-MVAOT247.js} +8 -8
  29. package/dist/{chunk-SH5S7XYD.js → chunk-MXFBBHJU.js} +72 -2
  30. package/dist/chunk-MXFBBHJU.js.map +1 -0
  31. package/dist/{chunk-VMQRBXJ5.js → chunk-NW7JW5GA.js} +2 -2
  32. package/dist/{chunk-SZKCBLS5.js → chunk-PUXCIHRL.js} +2 -2
  33. package/dist/{chunk-2IRT26RZ.js → chunk-QYHQ2JHL.js} +2 -2
  34. package/dist/{chunk-CN4P6SVA.js → chunk-RCZRL5BE.js} +2 -2
  35. package/dist/{chunk-SGIXDVSF.js → chunk-S27EXIHY.js} +2 -2
  36. package/dist/{chunk-5ML4TH3E.js → chunk-TFORLO3O.js} +4 -4
  37. package/dist/{chunk-TOFUTKQN.js → chunk-TR4DK5OH.js} +2 -2
  38. package/dist/{chunk-6ORWKANA.js → chunk-VYU7PXUS.js} +2 -2
  39. package/dist/{chunk-FFU4GMST.js → chunk-WNARATI3.js} +2 -2
  40. package/dist/{chunk-KSFBM6TV.js → chunk-YITUHONZ.js} +2 -2
  41. package/dist/{cli-BN0CkYzI.d.ts → cli-BguVmIwO.d.ts} +1 -1
  42. package/dist/cli.d.ts +2 -2
  43. package/dist/cli.js +18 -18
  44. package/dist/compounding/engine.js +3 -3
  45. package/dist/connectors/codex-materialize-runner.js +3 -3
  46. package/dist/connectors/index.js +3 -3
  47. package/dist/entity-retrieval.js +3 -3
  48. package/dist/index.d.ts +4 -4
  49. package/dist/index.js +30 -24
  50. package/dist/index.js.map +1 -1
  51. package/dist/maintenance/memory-governance.js +3 -3
  52. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
  53. package/dist/maintenance/rebuild-memory-projection.js +4 -4
  54. package/dist/mcp-memory-inspector-app.d.ts +1 -1
  55. package/dist/namespaces/migrate.js +4 -4
  56. package/dist/namespaces/storage.js +3 -3
  57. package/dist/offline-sync.d.ts +38 -1
  58. package/dist/offline-sync.js +8 -2
  59. package/dist/operator-toolkit.js +6 -6
  60. package/dist/orchestrator.js +11 -11
  61. package/dist/schemas.d.ts +22 -22
  62. package/dist/secure-store/index.js +2 -2
  63. package/dist/semantic-consolidation.js +4 -4
  64. package/dist/semantic-rule-promotion.js +3 -3
  65. package/dist/semantic-rule-verifier.js +3 -3
  66. package/dist/storage.d.ts +2 -0
  67. package/dist/storage.js +2 -2
  68. package/dist/transfer/types.d.ts +12 -12
  69. package/dist/verified-recall.js +3 -3
  70. package/package.json +1 -1
  71. package/src/access-http.test.ts +239 -0
  72. package/src/access-http.ts +128 -7
  73. package/src/access-service-offline-file-content.test.ts +37 -0
  74. package/src/access-service.ts +70 -0
  75. package/src/index.ts +4 -0
  76. package/src/offline-sync.test.ts +395 -79
  77. package/src/offline-sync.ts +473 -32
  78. package/src/secure-store/secure-fs.ts +84 -3
  79. package/src/storage.ts +12 -0
  80. package/dist/chunk-7Q2P774N.js.map +0 -1
  81. package/dist/chunk-FF46Q3SN.js.map +0 -1
  82. package/dist/chunk-NGPO6S3M.js.map +0 -1
  83. package/dist/chunk-SH5S7XYD.js.map +0 -1
  84. package/dist/chunk-UA6OCL6S.js.map +0 -1
  85. /package/dist/{chunk-YO3AZEE5.js.map → chunk-25YQM6XW.js.map} +0 -0
  86. /package/dist/{chunk-LDJANWTK.js.map → chunk-2DM72JF3.js.map} +0 -0
  87. /package/dist/{chunk-TLM762GT.js.map → chunk-2WIPXV3Y.js.map} +0 -0
  88. /package/dist/{chunk-QOHBYVZG.js.map → chunk-3F24QTRI.js.map} +0 -0
  89. /package/dist/{chunk-5IQC4OG6.js.map → chunk-4H6DURG6.js.map} +0 -0
  90. /package/dist/{chunk-26OQECWH.js.map → chunk-6CB4E7ZV.js.map} +0 -0
  91. /package/dist/{chunk-NOQ74SJN.js.map → chunk-7D6O46PF.js.map} +0 -0
  92. /package/dist/{chunk-FSODDMR2.js.map → chunk-IANK6Y5W.js.map} +0 -0
  93. /package/dist/{chunk-GGCJ253V.js.map → chunk-MVAOT247.js.map} +0 -0
  94. /package/dist/{chunk-VMQRBXJ5.js.map → chunk-NW7JW5GA.js.map} +0 -0
  95. /package/dist/{chunk-SZKCBLS5.js.map → chunk-PUXCIHRL.js.map} +0 -0
  96. /package/dist/{chunk-2IRT26RZ.js.map → chunk-QYHQ2JHL.js.map} +0 -0
  97. /package/dist/{chunk-CN4P6SVA.js.map → chunk-RCZRL5BE.js.map} +0 -0
  98. /package/dist/{chunk-SGIXDVSF.js.map → chunk-S27EXIHY.js.map} +0 -0
  99. /package/dist/{chunk-5ML4TH3E.js.map → chunk-TFORLO3O.js.map} +0 -0
  100. /package/dist/{chunk-TOFUTKQN.js.map → chunk-TR4DK5OH.js.map} +0 -0
  101. /package/dist/{chunk-6ORWKANA.js.map → chunk-VYU7PXUS.js.map} +0 -0
  102. /package/dist/{chunk-FFU4GMST.js.map → chunk-WNARATI3.js.map} +0 -0
  103. /package/dist/{chunk-KSFBM6TV.js.map → chunk-YITUHONZ.js.map} +0 -0
@@ -6,6 +6,7 @@ import {
6
6
  readdir,
7
7
  readFile,
8
8
  rename,
9
+ rm,
9
10
  stat,
10
11
  unlink,
11
12
  writeFile,
@@ -28,6 +29,8 @@ export const OFFLINE_SYNC_SNAPSHOT_FORMAT = "remnic.offline-sync.snapshot.v1";
28
29
  export const OFFLINE_SYNC_CHANGESET_FORMAT = "remnic.offline-sync.changeset.v1";
29
30
  export const OFFLINE_SYNC_STATE_VERSION = 1;
30
31
  export const OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES = 64 * 1024 * 1024;
32
+ export const OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024;
33
+ export const OFFLINE_SYNC_APPLY_MAX_BODY_BYTES = 16 * 1024 * 1024;
31
34
 
32
35
  export interface OfflineSyncFileState {
33
36
  path: string;
@@ -130,6 +133,12 @@ export interface OfflineSyncFileWriteTarget extends OfflineSyncFileTarget {
130
133
  content: Buffer;
131
134
  }
132
135
 
136
+ export interface OfflineSyncFileWriteChunksTarget extends OfflineSyncFileTarget {
137
+ chunks: AsyncIterable<Buffer>;
138
+ }
139
+
140
+ export interface OfflineSyncFileStagingWriteTarget extends OfflineSyncFileWriteTarget {}
141
+
133
142
  export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState, "sha256"> {
134
143
  sha256?: string;
135
144
  offset: number;
@@ -137,6 +146,26 @@ export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState,
137
146
  content: Buffer;
138
147
  }
139
148
 
149
+ export interface OfflineSyncApplyFileContentChunkResult {
150
+ path: string;
151
+ sha256: string;
152
+ bytes: number;
153
+ mtimeMs: number;
154
+ offset: number;
155
+ chunkBytes: number;
156
+ done: boolean;
157
+ applied: boolean;
158
+ skipped: boolean;
159
+ conflict?: OfflineSyncConflict;
160
+ currentFile?: OfflineSyncFileState;
161
+ }
162
+
163
+ interface OfflineUploadStaging {
164
+ kind: "single" | "chunks";
165
+ relPath: string;
166
+ filePath: string;
167
+ }
168
+
140
169
  interface OfflineSyncFileRecordOptions {
141
170
  root: SafeArchiveRoot;
142
171
  relPath: string;
@@ -146,35 +175,11 @@ interface OfflineSyncFileRecordOptions {
146
175
  }
147
176
 
148
177
  const SYNC_INTERNAL_DIR = ".offline-sync";
178
+ const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
149
179
  const EXCLUDED_FILE_NAMES = new Set([
150
180
  ".sync-state.json",
151
181
  ]);
152
182
 
153
- const DERIVED_RUNTIME_STATE_BASENAMES = new Set([
154
- ".artifact-write-version.log",
155
- ".memory-status-version.log",
156
- "fact-hashes.ready",
157
- "fact-hashes.txt",
158
- "buffer-surprise-ledger.jsonl",
159
- "buffer.json",
160
- "embeddings.json",
161
- "entity-mention-index.json",
162
- "index_tags.json",
163
- "index_time.json",
164
- "last_graph_recall.json",
165
- "last_intent.json",
166
- "last_qmd_recall.json",
167
- "last_recall.json",
168
- "lcm.sqlite",
169
- "lcm.sqlite-shm",
170
- "lcm.sqlite-wal",
171
- "memory-lifecycle-ledger.jsonl",
172
- "memory-projection.sqlite",
173
- "memory-projection.sqlite-shm",
174
- "memory-projection.sqlite-wal",
175
- "recall_impressions.jsonl",
176
- ]);
177
-
178
183
  const EXCLUDED_FILE_PREFIXES = [
179
184
  ".remnic-sync.",
180
185
  ".remnic-sync-state.",
@@ -431,7 +436,6 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
431
436
  const parts = relPosix.split("/");
432
437
  if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
433
438
  if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
434
- if (isDerivedRuntimeStatePath(parts)) return true;
435
439
  if (!includeTranscripts && parts[0] === "transcripts") return true;
436
440
  const basename = parts[parts.length - 1] ?? "";
437
441
  if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
@@ -442,12 +446,7 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
442
446
  function shouldIgnoreIncomingRuntimePath(relPosix: string): boolean {
443
447
  const parts = relPosix.split("/");
444
448
  const basename = parts[parts.length - 1] ?? "";
445
- return isDerivedRuntimeStatePath(parts) || (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-"));
446
- }
447
-
448
- function isDerivedRuntimeStatePath(parts: string[]): boolean {
449
- const basename = parts[parts.length - 1] ?? "";
450
- return isCanonicalRuntimeStatePath(parts) && DERIVED_RUNTIME_STATE_BASENAMES.has(basename);
449
+ return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
451
450
  }
452
451
 
453
452
  function isCanonicalRuntimeStatePath(parts: string[]): boolean {
@@ -683,11 +682,15 @@ export async function buildOfflineSyncChangeset(options: {
683
682
  root: string;
684
683
  sourceId: string;
685
684
  baseFiles?: readonly OfflineSyncFileState[];
685
+ excludePaths?: readonly string[];
686
686
  includeTranscripts?: boolean;
687
687
  now?: Date;
688
688
  readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
689
689
  }): Promise<OfflineSyncChangeset> {
690
690
  const includeTranscripts = options.includeTranscripts !== false;
691
+ const excludedPaths = new Set(
692
+ (options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]")),
693
+ );
691
694
  const base = byPath(filterBaseFilesForMode(
692
695
  normalizeFileStates(options.baseFiles),
693
696
  includeTranscripts,
@@ -704,6 +707,7 @@ export async function buildOfflineSyncChangeset(options: {
704
707
  const changes: OfflineSyncChange[] = [];
705
708
 
706
709
  for (const relPath of unionPaths(base, currentMap)) {
710
+ if (excludedPaths.has(relPath)) continue;
707
711
  const baseEntry = base.get(relPath);
708
712
  const currentEntry = currentMap.get(relPath);
709
713
  if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
@@ -1148,6 +1152,443 @@ async function writeSafeFile(
1148
1152
  }
1149
1153
  }
1150
1154
 
1155
+ export async function applyOfflineSyncFileContentChunk(options: {
1156
+ root: string;
1157
+ sourceId: string;
1158
+ path: string;
1159
+ sha256: string;
1160
+ bytes: number;
1161
+ mtimeMs: number;
1162
+ offset?: number;
1163
+ content: Buffer;
1164
+ baseSha256?: string;
1165
+ includeTranscripts?: boolean;
1166
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1167
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1168
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1169
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
1170
+ }): Promise<OfflineSyncApplyFileContentChunkResult> {
1171
+ const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
1172
+ const sourceId = normalizeSourceId(options.sourceId, "sourceId");
1173
+ const relPath = normalizeRelativePath(options.path, "path");
1174
+ const includeTranscripts = options.includeTranscripts !== false;
1175
+ if (shouldExcludeRelPath(relPath, includeTranscripts)) {
1176
+ throw new Error(`offline sync file content path is excluded: ${relPath}`);
1177
+ }
1178
+ const sha256 = assertSha256(options.sha256, "sha256");
1179
+ const bytes = assertNonNegativeInteger(options.bytes, "bytes");
1180
+ const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
1181
+ const offset = options.offset === undefined
1182
+ ? 0
1183
+ : assertNonNegativeInteger(options.offset, "offset");
1184
+ const baseSha256 = options.baseSha256 === undefined
1185
+ ? undefined
1186
+ : assertSha256(options.baseSha256, "baseSha256");
1187
+ if (!Buffer.isBuffer(options.content)) {
1188
+ throw new Error("content must be a Buffer");
1189
+ }
1190
+ if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
1191
+ throw new Error(
1192
+ `content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`,
1193
+ );
1194
+ }
1195
+ if (bytes > 0 && options.content.length === 0) {
1196
+ throw new Error("content chunk must be non-empty before EOF");
1197
+ }
1198
+ if (offset > bytes || offset + options.content.length > bytes) {
1199
+ throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
1200
+ }
1201
+ if (options.writeFile && !options.writeFileChunks) {
1202
+ throw new Error("offline sync upload storage hooks require writeFileChunks");
1203
+ }
1204
+ if (options.writeFile && !options.writeStagingFile) {
1205
+ throw new Error("offline sync upload storage hooks require writeStagingFile");
1206
+ }
1207
+ if (offset === 0) {
1208
+ await pruneOfflineUploadStaging(root);
1209
+ }
1210
+
1211
+ const upload = await writeOfflineUploadChunk({
1212
+ root,
1213
+ sourceId,
1214
+ relPath,
1215
+ sha256,
1216
+ bytes,
1217
+ offset,
1218
+ content: options.content,
1219
+ readFile: options.readFile,
1220
+ writeFile: options.writeFile,
1221
+ writeStagingFile: options.writeStagingFile,
1222
+ });
1223
+ const done = offset + options.content.length === bytes;
1224
+ const baseResult = {
1225
+ path: relPath,
1226
+ sha256,
1227
+ bytes,
1228
+ mtimeMs,
1229
+ offset,
1230
+ chunkBytes: options.content.length,
1231
+ done,
1232
+ };
1233
+ if (!done) {
1234
+ return {
1235
+ ...baseResult,
1236
+ applied: false,
1237
+ skipped: false,
1238
+ };
1239
+ }
1240
+
1241
+ const digest = await digestOfflineUploadStagingContent({
1242
+ root,
1243
+ upload,
1244
+ readFile: options.readFile,
1245
+ });
1246
+ if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
1247
+ await cleanupOfflineUpload(upload).catch(() => {});
1248
+ throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
1249
+ }
1250
+
1251
+ const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
1252
+ root: root.abs,
1253
+ sourceId: "local",
1254
+ paths: [relPath],
1255
+ includeContent: false,
1256
+ includeTranscripts,
1257
+ readFile: options.readFile,
1258
+ });
1259
+ const currentFile = currentSnapshot.files[0];
1260
+ const uploadedState: OfflineSyncFileState = {
1261
+ path: relPath,
1262
+ sha256,
1263
+ bytes,
1264
+ mtimeMs,
1265
+ };
1266
+
1267
+ try {
1268
+ if (currentFile?.sha256 === sha256) {
1269
+ return {
1270
+ ...baseResult,
1271
+ applied: false,
1272
+ skipped: true,
1273
+ currentFile: toFileState(currentFile),
1274
+ };
1275
+ }
1276
+ if (!baseSha256 && currentFile) {
1277
+ const conflict = await recordConflict({
1278
+ root,
1279
+ relPath,
1280
+ reason: "remote_exists_for_local_create",
1281
+ localSha256: currentFile.sha256,
1282
+ incomingSha256: sha256,
1283
+ writeConflictCopies: false,
1284
+ sourceId,
1285
+ writeFile: options.writeFile,
1286
+ });
1287
+ return {
1288
+ ...baseResult,
1289
+ applied: false,
1290
+ skipped: false,
1291
+ conflict,
1292
+ currentFile: toFileState(currentFile),
1293
+ };
1294
+ }
1295
+ if (baseSha256 && currentFile?.sha256 !== baseSha256) {
1296
+ const conflict = await recordConflict({
1297
+ root,
1298
+ relPath,
1299
+ reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
1300
+ baseSha256,
1301
+ localSha256: currentFile?.sha256,
1302
+ incomingSha256: sha256,
1303
+ writeConflictCopies: false,
1304
+ sourceId,
1305
+ writeFile: options.writeFile,
1306
+ });
1307
+ return {
1308
+ ...baseResult,
1309
+ applied: false,
1310
+ skipped: false,
1311
+ conflict,
1312
+ ...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
1313
+ };
1314
+ }
1315
+
1316
+ await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
1317
+ return {
1318
+ ...baseResult,
1319
+ applied: true,
1320
+ skipped: false,
1321
+ currentFile: uploadedState,
1322
+ };
1323
+ } finally {
1324
+ await cleanupOfflineUpload(upload).catch(() => {});
1325
+ }
1326
+ }
1327
+
1328
+ function offlineUploadRelPath(options: {
1329
+ sourceId: string;
1330
+ relPath: string;
1331
+ sha256: string;
1332
+ bytes: number;
1333
+ }): string {
1334
+ const key = hashText([
1335
+ options.sourceId,
1336
+ options.relPath,
1337
+ options.sha256,
1338
+ String(options.bytes),
1339
+ ].join("\0"));
1340
+ return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
1341
+ }
1342
+
1343
+ async function offlineUploadPath(root: SafeArchiveRoot, options: {
1344
+ sourceId: string;
1345
+ relPath: string;
1346
+ sha256: string;
1347
+ bytes: number;
1348
+ }): Promise<OfflineUploadStaging> {
1349
+ const relPath = offlineUploadRelPath(options);
1350
+ return {
1351
+ kind: "single",
1352
+ relPath,
1353
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1354
+ };
1355
+ }
1356
+
1357
+ async function offlineUploadChunkPath(root: SafeArchiveRoot, options: {
1358
+ sourceId: string;
1359
+ relPath: string;
1360
+ sha256: string;
1361
+ bytes: number;
1362
+ offset: number;
1363
+ }): Promise<OfflineUploadStaging> {
1364
+ const uploadRelPath = offlineUploadRelPath(options);
1365
+ const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
1366
+ return {
1367
+ kind: "chunks",
1368
+ relPath,
1369
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1370
+ };
1371
+ }
1372
+
1373
+ async function writeOfflineUploadChunk(options: {
1374
+ root: SafeArchiveRoot;
1375
+ sourceId: string;
1376
+ relPath: string;
1377
+ sha256: string;
1378
+ bytes: number;
1379
+ offset: number;
1380
+ content: Buffer;
1381
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1382
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1383
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1384
+ }): Promise<OfflineUploadStaging> {
1385
+ if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
1386
+ throw new Error("offline sync upload chunk storage hooks require readFile");
1387
+ }
1388
+ const uploadRoot = {
1389
+ ...(await offlineUploadPath(options.root, options)),
1390
+ kind: "chunks" as const,
1391
+ };
1392
+ if (options.offset === 0) {
1393
+ await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {});
1394
+ } else {
1395
+ const existing = await stat(uploadRoot.filePath).catch((error: unknown) => {
1396
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1397
+ throw error;
1398
+ });
1399
+ if (!existing || !existing.isDirectory()) {
1400
+ throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
1401
+ }
1402
+ }
1403
+ const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
1404
+
1405
+ const writeStagingFile = options.writeStagingFile ?? options.writeFile;
1406
+ if (writeStagingFile) {
1407
+ // Storage-backed services provide these hooks so secure-store deployments
1408
+ // keep staged partial uploads encrypted at rest without mutating indexes.
1409
+ await writeOfflineUploadContent({
1410
+ root: options.root,
1411
+ relPath: chunk.relPath,
1412
+ filePath: chunk.filePath,
1413
+ content: options.content,
1414
+ writeFile: writeStagingFile,
1415
+ });
1416
+ return uploadRoot;
1417
+ }
1418
+
1419
+ await mkdir(path.dirname(chunk.filePath), { recursive: true });
1420
+ const existingChunk = await lstat(chunk.filePath).catch((error: unknown) => {
1421
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1422
+ throw error;
1423
+ });
1424
+ if (existingChunk?.isSymbolicLink()) {
1425
+ throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
1426
+ }
1427
+ await writeFile(chunk.filePath, options.content, { mode: 0o600 });
1428
+ return uploadRoot;
1429
+ }
1430
+
1431
+ async function pruneOfflineUploadStaging(root: SafeArchiveRoot): Promise<void> {
1432
+ const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
1433
+ const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
1434
+ const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error: unknown) => {
1435
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
1436
+ throw error;
1437
+ });
1438
+ const now = Date.now();
1439
+ await Promise.all(entries.map(async (entry) => {
1440
+ if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
1441
+ const relPath = `${uploadsRelPath}/${entry.name}`;
1442
+ const filePath = await resolveSafeArchiveTarget(root, relPath);
1443
+ const info = await lstat(filePath).catch((error: unknown) => {
1444
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1445
+ throw error;
1446
+ });
1447
+ if (!info) return;
1448
+ if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
1449
+ await rm(filePath, { recursive: true, force: true });
1450
+ }));
1451
+ }
1452
+
1453
+ async function* readOfflineUploadStagingChunks(options: {
1454
+ root: SafeArchiveRoot;
1455
+ upload: OfflineUploadStaging;
1456
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1457
+ }): AsyncGenerator<Buffer> {
1458
+ if (options.upload.kind === "single") {
1459
+ yield await readOfflineUploadContent({
1460
+ root: options.root,
1461
+ relPath: options.upload.relPath,
1462
+ filePath: options.upload.filePath,
1463
+ readFile: options.readFile,
1464
+ });
1465
+ return;
1466
+ }
1467
+
1468
+ const entries = await readdir(options.upload.filePath);
1469
+ const chunkNames = entries
1470
+ .filter((entry) => /^\d{20}\.part$/.test(entry))
1471
+ .sort();
1472
+ if (chunkNames.length === 0) {
1473
+ throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
1474
+ }
1475
+ let expectedOffset = 0;
1476
+ for (const chunkName of chunkNames) {
1477
+ const offset = Number(chunkName.slice(0, 20));
1478
+ if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
1479
+ throw new Error(
1480
+ `offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`,
1481
+ );
1482
+ }
1483
+ const relPath = `${options.upload.relPath}/${chunkName}`;
1484
+ const filePath = await resolveSafeArchiveTarget(options.root, relPath);
1485
+ const content = await readOfflineUploadContent({
1486
+ root: options.root,
1487
+ relPath,
1488
+ filePath,
1489
+ readFile: options.readFile,
1490
+ });
1491
+ expectedOffset += content.length;
1492
+ yield content;
1493
+ }
1494
+ }
1495
+
1496
+ async function digestOfflineUploadStagingContent(options: {
1497
+ root: SafeArchiveRoot;
1498
+ upload: OfflineUploadStaging;
1499
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1500
+ }): Promise<{ sha256: string; bytes: number }> {
1501
+ const hash = createHash("sha256");
1502
+ let bytes = 0;
1503
+ for await (const chunk of readOfflineUploadStagingChunks(options)) {
1504
+ hash.update(chunk);
1505
+ bytes += chunk.length;
1506
+ }
1507
+ return { sha256: hash.digest("hex"), bytes };
1508
+ }
1509
+
1510
+ async function writeSafeFileFromUpload(
1511
+ root: SafeArchiveRoot,
1512
+ relPath: string,
1513
+ upload: OfflineUploadStaging,
1514
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
1515
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
1516
+ ): Promise<void> {
1517
+ const target = await resolveSafeArchiveTarget(root, relPath);
1518
+ const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
1519
+ if (writeFileChunks) {
1520
+ await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
1521
+ return;
1522
+ }
1523
+
1524
+ await mkdir(path.dirname(target), { recursive: true });
1525
+ const tmp = path.join(
1526
+ path.dirname(target),
1527
+ `.remnic-sync.${process.pid}.${randomUUID()}.tmp`,
1528
+ );
1529
+ const handle = await open(tmp, "w", 0o600);
1530
+ try {
1531
+ for await (const chunk of chunks) {
1532
+ if (chunk.length > 0) await handle.write(chunk);
1533
+ }
1534
+ await handle.close();
1535
+ const targetStat = await lstat(target).catch((error: unknown) => {
1536
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1537
+ throw error;
1538
+ });
1539
+ if (targetStat?.isSymbolicLink()) {
1540
+ throw new Error(`offline sync target is a symlink: ${relPath}`);
1541
+ }
1542
+ await rename(tmp, target);
1543
+ } catch (error) {
1544
+ await handle.close().catch(() => {});
1545
+ await unlink(tmp).catch(() => {});
1546
+ throw error;
1547
+ }
1548
+ }
1549
+
1550
+ async function cleanupOfflineUpload(upload: OfflineUploadStaging): Promise<void> {
1551
+ if (upload.kind === "chunks") {
1552
+ await rm(upload.filePath, { recursive: true, force: true });
1553
+ return;
1554
+ }
1555
+ await unlink(upload.filePath).catch((error: unknown) => {
1556
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
1557
+ throw error;
1558
+ });
1559
+ }
1560
+
1561
+ async function readOfflineUploadContent(options: {
1562
+ root: SafeArchiveRoot;
1563
+ relPath: string;
1564
+ filePath: string;
1565
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1566
+ }): Promise<Buffer> {
1567
+ if (options.readFile) {
1568
+ return options.readFile({
1569
+ root: options.root.abs,
1570
+ path: options.relPath,
1571
+ filePath: options.filePath,
1572
+ });
1573
+ }
1574
+ return readFile(options.filePath);
1575
+ }
1576
+
1577
+ async function writeOfflineUploadContent(options: {
1578
+ root: SafeArchiveRoot;
1579
+ relPath: string;
1580
+ filePath: string;
1581
+ content: Buffer;
1582
+ writeFile: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1583
+ }): Promise<void> {
1584
+ await options.writeFile({
1585
+ root: options.root.abs,
1586
+ path: options.relPath,
1587
+ filePath: options.filePath,
1588
+ content: options.content,
1589
+ });
1590
+ }
1591
+
1151
1592
  async function deleteSafeFile(
1152
1593
  root: SafeArchiveRoot,
1153
1594
  relPath: string,
@@ -43,10 +43,21 @@
43
43
  * Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
44
44
  */
45
45
 
46
- import { lstat, mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
46
+ import { createCipheriv, randomBytes } from "node:crypto";
47
+ import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
47
48
  import path from "node:path";
48
49
 
49
- import { generateSalt, open, seal } from "./cipher.js";
50
+ import {
51
+ AUTH_TAG_LENGTH,
52
+ ENVELOPE_HEADER_SIZE,
53
+ ENVELOPE_LAYOUT,
54
+ ENVELOPE_SALT_LENGTH,
55
+ ENVELOPE_VERSION,
56
+ IV_LENGTH,
57
+ generateSalt,
58
+ open as openEnvelope,
59
+ seal,
60
+ } from "./cipher.js";
50
61
 
51
62
  // ---------------------------------------------------------------------------
52
63
  // Error classes
@@ -157,7 +168,7 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
157
168
  }
158
169
  const envelope = buf.subarray(MAGIC_HEADER_SIZE);
159
170
  try {
160
- return open(key, envelope, aad ? { aad } : {});
171
+ return openEnvelope(key, envelope, aad ? { aad } : {});
161
172
  } catch (err) {
162
173
  const msg = err instanceof Error ? err.message : String(err);
163
174
  throw new SecureStoreDecryptError(
@@ -166,6 +177,13 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
166
177
  }
167
178
  }
168
179
 
180
+ function buildHeaderAad(salt: Uint8Array): Buffer {
181
+ const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
182
+ out.writeUInt8(ENVELOPE_VERSION, 0);
183
+ Buffer.from(salt).copy(out, 1);
184
+ return out;
185
+ }
186
+
169
187
  // ---------------------------------------------------------------------------
170
188
  // Path → AAD helper
171
189
  // ---------------------------------------------------------------------------
@@ -293,6 +311,69 @@ export async function writeMaybeEncryptedFile(
293
311
  }
294
312
  }
295
313
 
314
+ export async function writeMaybeEncryptedFileFromChunks(
315
+ filePath: string,
316
+ chunks: AsyncIterable<Buffer>,
317
+ key: Buffer | null,
318
+ options: WriteMaybeEncryptedFileOptions = {},
319
+ memoryDir?: string,
320
+ ): Promise<void> {
321
+ const { mode = 0o600, atomic = true } = options;
322
+ await mkdir(path.dirname(filePath), { recursive: true });
323
+ const writePath = atomic ? `${filePath}.tmp-${process.pid}-${Date.now()}` : filePath;
324
+ let completed = false;
325
+ try {
326
+ const handle = await openFile(writePath, "w", mode);
327
+ try {
328
+ if (key !== null) {
329
+ const salt = generateSalt();
330
+ const iv = randomBytes(IV_LENGTH);
331
+ const header = Buffer.alloc(MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
332
+ MAGIC_BYTES.copy(header, 0);
333
+ header.writeUInt8(FILE_FORMAT_VERSION, MAGIC_BYTES.length);
334
+ header.writeUInt8(FILE_FORMAT_FLAGS, MAGIC_BYTES.length + 1);
335
+ const envelopeOffset = MAGIC_HEADER_SIZE;
336
+ header.writeUInt8(ENVELOPE_VERSION, envelopeOffset + ENVELOPE_LAYOUT.version);
337
+ salt.copy(header, envelopeOffset + ENVELOPE_LAYOUT.salt);
338
+ iv.copy(header, envelopeOffset + ENVELOPE_LAYOUT.iv);
339
+ await handle.write(header);
340
+
341
+ const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
342
+ const aad = filePathAad(filePath, memoryDir);
343
+ cipher.setAAD(Buffer.concat([buildHeaderAad(salt), aad]));
344
+ for await (const chunk of chunks) {
345
+ if (chunk.length === 0) continue;
346
+ const encrypted = cipher.update(chunk);
347
+ if (encrypted.length > 0) await handle.write(encrypted);
348
+ }
349
+ const final = cipher.final();
350
+ if (final.length > 0) await handle.write(final);
351
+ const authTag = cipher.getAuthTag();
352
+ await handle.write(
353
+ authTag,
354
+ 0,
355
+ authTag.length,
356
+ MAGIC_HEADER_SIZE + ENVELOPE_LAYOUT.authTag,
357
+ );
358
+ } else {
359
+ for await (const chunk of chunks) {
360
+ if (chunk.length > 0) await handle.write(chunk);
361
+ }
362
+ }
363
+ } finally {
364
+ await handle.close();
365
+ }
366
+ if (atomic) {
367
+ await rename(writePath, filePath);
368
+ }
369
+ completed = true;
370
+ } finally {
371
+ if (!completed && atomic) {
372
+ await unlink(writePath).catch(() => {});
373
+ }
374
+ }
375
+ }
376
+
296
377
  // ---------------------------------------------------------------------------
297
378
  // Migration
298
379
  // ---------------------------------------------------------------------------