@remnic/core 1.1.22 → 1.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/access-cli.js +15 -15
  2. package/dist/access-http.d.ts +9 -1
  3. package/dist/access-http.js +9 -9
  4. package/dist/access-mcp.d.ts +1 -1
  5. package/dist/access-mcp.js +8 -8
  6. package/dist/access-schema.js +3 -3
  7. package/dist/{access-service-DT9L2DW4.d.ts → access-service-CEyV8XJ5.d.ts} +19 -2
  8. package/dist/access-service.d.ts +1 -1
  9. package/dist/access-service.js +6 -6
  10. package/dist/briefing.js +3 -3
  11. package/dist/causal-consolidation.js +4 -4
  12. package/dist/{chunk-YO3AZEE5.js → chunk-25YQM6XW.js} +3 -3
  13. package/dist/{chunk-TLM762GT.js → chunk-2WIPXV3Y.js} +2 -2
  14. package/dist/{chunk-QOHBYVZG.js → chunk-3F24QTRI.js} +2 -2
  15. package/dist/{chunk-5IQC4OG6.js → chunk-4H6DURG6.js} +2 -2
  16. package/dist/{chunk-NOQ74SJN.js → chunk-7D6O46PF.js} +2 -2
  17. package/dist/{chunk-VMQRBXJ5.js → chunk-7E7SZRPP.js} +2 -2
  18. package/dist/{chunk-7Q2P774N.js → chunk-F33CJ5CH.js} +13 -3
  19. package/dist/chunk-F33CJ5CH.js.map +1 -0
  20. package/dist/{chunk-26OQECWH.js → chunk-FHXVW3L4.js} +4 -4
  21. package/dist/{chunk-UA6OCL6S.js → chunk-HWF42K6J.js} +103 -4
  22. package/dist/chunk-HWF42K6J.js.map +1 -0
  23. package/dist/{chunk-FSODDMR2.js → chunk-IANK6Y5W.js} +2 -2
  24. package/dist/{chunk-FF46Q3SN.js → chunk-JKXFF3NT.js} +356 -32
  25. package/dist/chunk-JKXFF3NT.js.map +1 -0
  26. package/dist/{chunk-NGPO6S3M.js → chunk-MM5EBZVW.js} +42 -5
  27. package/dist/chunk-MM5EBZVW.js.map +1 -0
  28. package/dist/{chunk-GGCJ253V.js → chunk-MVAOT247.js} +8 -8
  29. package/dist/{chunk-SH5S7XYD.js → chunk-MXFBBHJU.js} +72 -2
  30. package/dist/chunk-MXFBBHJU.js.map +1 -0
  31. package/dist/{chunk-SZKCBLS5.js → chunk-PUXCIHRL.js} +2 -2
  32. package/dist/{chunk-2IRT26RZ.js → chunk-QYHQ2JHL.js} +2 -2
  33. package/dist/{chunk-LDJANWTK.js → chunk-RA73CTVY.js} +12 -12
  34. package/dist/{chunk-CN4P6SVA.js → chunk-RCZRL5BE.js} +2 -2
  35. package/dist/{chunk-SGIXDVSF.js → chunk-S27EXIHY.js} +2 -2
  36. package/dist/{chunk-5ML4TH3E.js → chunk-TFORLO3O.js} +4 -4
  37. package/dist/{chunk-TOFUTKQN.js → chunk-TR4DK5OH.js} +2 -2
  38. package/dist/{chunk-6ORWKANA.js → chunk-VYU7PXUS.js} +2 -2
  39. package/dist/{chunk-FFU4GMST.js → chunk-WNARATI3.js} +2 -2
  40. package/dist/{chunk-KSFBM6TV.js → chunk-YITUHONZ.js} +2 -2
  41. package/dist/{cli-BN0CkYzI.d.ts → cli-BguVmIwO.d.ts} +1 -1
  42. package/dist/cli.d.ts +2 -2
  43. package/dist/cli.js +18 -18
  44. package/dist/compounding/engine.js +3 -3
  45. package/dist/connectors/codex-materialize-runner.js +3 -3
  46. package/dist/connectors/index.js +3 -3
  47. package/dist/entity-retrieval.js +3 -3
  48. package/dist/index.d.ts +4 -4
  49. package/dist/index.js +26 -24
  50. package/dist/index.js.map +1 -1
  51. package/dist/maintenance/memory-governance.js +3 -3
  52. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
  53. package/dist/maintenance/rebuild-memory-projection.js +4 -4
  54. package/dist/mcp-memory-inspector-app.d.ts +1 -1
  55. package/dist/namespaces/migrate.js +4 -4
  56. package/dist/namespaces/storage.js +3 -3
  57. package/dist/offline-sync.d.ts +36 -1
  58. package/dist/offline-sync.js +4 -2
  59. package/dist/operator-toolkit.js +6 -6
  60. package/dist/orchestrator.js +11 -11
  61. package/dist/schemas.d.ts +22 -22
  62. package/dist/secure-store/index.js +2 -2
  63. package/dist/semantic-consolidation.js +4 -4
  64. package/dist/semantic-rule-promotion.js +3 -3
  65. package/dist/semantic-rule-verifier.js +3 -3
  66. package/dist/storage.d.ts +2 -0
  67. package/dist/storage.js +2 -2
  68. package/dist/transfer/types.d.ts +12 -12
  69. package/dist/verified-recall.js +3 -3
  70. package/package.json +1 -1
  71. package/src/access-http.test.ts +176 -0
  72. package/src/access-http.ts +116 -0
  73. package/src/access-service-offline-file-content.test.ts +37 -0
  74. package/src/access-service.ts +70 -0
  75. package/src/index.ts +2 -0
  76. package/src/offline-sync.test.ts +395 -79
  77. package/src/offline-sync.ts +471 -32
  78. package/src/secure-store/secure-fs.ts +84 -3
  79. package/src/storage.ts +12 -0
  80. package/dist/chunk-7Q2P774N.js.map +0 -1
  81. package/dist/chunk-FF46Q3SN.js.map +0 -1
  82. package/dist/chunk-NGPO6S3M.js.map +0 -1
  83. package/dist/chunk-SH5S7XYD.js.map +0 -1
  84. package/dist/chunk-UA6OCL6S.js.map +0 -1
  85. /package/dist/{chunk-YO3AZEE5.js.map → chunk-25YQM6XW.js.map} +0 -0
  86. /package/dist/{chunk-TLM762GT.js.map → chunk-2WIPXV3Y.js.map} +0 -0
  87. /package/dist/{chunk-QOHBYVZG.js.map → chunk-3F24QTRI.js.map} +0 -0
  88. /package/dist/{chunk-5IQC4OG6.js.map → chunk-4H6DURG6.js.map} +0 -0
  89. /package/dist/{chunk-NOQ74SJN.js.map → chunk-7D6O46PF.js.map} +0 -0
  90. /package/dist/{chunk-VMQRBXJ5.js.map → chunk-7E7SZRPP.js.map} +0 -0
  91. /package/dist/{chunk-26OQECWH.js.map → chunk-FHXVW3L4.js.map} +0 -0
  92. /package/dist/{chunk-FSODDMR2.js.map → chunk-IANK6Y5W.js.map} +0 -0
  93. /package/dist/{chunk-GGCJ253V.js.map → chunk-MVAOT247.js.map} +0 -0
  94. /package/dist/{chunk-SZKCBLS5.js.map → chunk-PUXCIHRL.js.map} +0 -0
  95. /package/dist/{chunk-2IRT26RZ.js.map → chunk-QYHQ2JHL.js.map} +0 -0
  96. /package/dist/{chunk-LDJANWTK.js.map → chunk-RA73CTVY.js.map} +0 -0
  97. /package/dist/{chunk-CN4P6SVA.js.map → chunk-RCZRL5BE.js.map} +0 -0
  98. /package/dist/{chunk-SGIXDVSF.js.map → chunk-S27EXIHY.js.map} +0 -0
  99. /package/dist/{chunk-5ML4TH3E.js.map → chunk-TFORLO3O.js.map} +0 -0
  100. /package/dist/{chunk-TOFUTKQN.js.map → chunk-TR4DK5OH.js.map} +0 -0
  101. /package/dist/{chunk-6ORWKANA.js.map → chunk-VYU7PXUS.js.map} +0 -0
  102. /package/dist/{chunk-FFU4GMST.js.map → chunk-WNARATI3.js.map} +0 -0
  103. /package/dist/{chunk-KSFBM6TV.js.map → chunk-YITUHONZ.js.map} +0 -0
@@ -6,6 +6,7 @@ import {
6
6
  readdir,
7
7
  readFile,
8
8
  rename,
9
+ rm,
9
10
  stat,
10
11
  unlink,
11
12
  writeFile,
@@ -130,6 +131,12 @@ export interface OfflineSyncFileWriteTarget extends OfflineSyncFileTarget {
130
131
  content: Buffer;
131
132
  }
132
133
 
134
+ export interface OfflineSyncFileWriteChunksTarget extends OfflineSyncFileTarget {
135
+ chunks: AsyncIterable<Buffer>;
136
+ }
137
+
138
+ export interface OfflineSyncFileStagingWriteTarget extends OfflineSyncFileWriteTarget {}
139
+
133
140
  export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState, "sha256"> {
134
141
  sha256?: string;
135
142
  offset: number;
@@ -137,6 +144,26 @@ export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState,
137
144
  content: Buffer;
138
145
  }
139
146
 
147
+ export interface OfflineSyncApplyFileContentChunkResult {
148
+ path: string;
149
+ sha256: string;
150
+ bytes: number;
151
+ mtimeMs: number;
152
+ offset: number;
153
+ chunkBytes: number;
154
+ done: boolean;
155
+ applied: boolean;
156
+ skipped: boolean;
157
+ conflict?: OfflineSyncConflict;
158
+ currentFile?: OfflineSyncFileState;
159
+ }
160
+
161
+ interface OfflineUploadStaging {
162
+ kind: "single" | "chunks";
163
+ relPath: string;
164
+ filePath: string;
165
+ }
166
+
140
167
  interface OfflineSyncFileRecordOptions {
141
168
  root: SafeArchiveRoot;
142
169
  relPath: string;
@@ -146,35 +173,11 @@ interface OfflineSyncFileRecordOptions {
146
173
  }
147
174
 
148
175
  const SYNC_INTERNAL_DIR = ".offline-sync";
176
+ const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
149
177
  const EXCLUDED_FILE_NAMES = new Set([
150
178
  ".sync-state.json",
151
179
  ]);
152
180
 
153
- const DERIVED_RUNTIME_STATE_BASENAMES = new Set([
154
- ".artifact-write-version.log",
155
- ".memory-status-version.log",
156
- "fact-hashes.ready",
157
- "fact-hashes.txt",
158
- "buffer-surprise-ledger.jsonl",
159
- "buffer.json",
160
- "embeddings.json",
161
- "entity-mention-index.json",
162
- "index_tags.json",
163
- "index_time.json",
164
- "last_graph_recall.json",
165
- "last_intent.json",
166
- "last_qmd_recall.json",
167
- "last_recall.json",
168
- "lcm.sqlite",
169
- "lcm.sqlite-shm",
170
- "lcm.sqlite-wal",
171
- "memory-lifecycle-ledger.jsonl",
172
- "memory-projection.sqlite",
173
- "memory-projection.sqlite-shm",
174
- "memory-projection.sqlite-wal",
175
- "recall_impressions.jsonl",
176
- ]);
177
-
178
181
  const EXCLUDED_FILE_PREFIXES = [
179
182
  ".remnic-sync.",
180
183
  ".remnic-sync-state.",
@@ -431,7 +434,6 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
431
434
  const parts = relPosix.split("/");
432
435
  if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
433
436
  if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
434
- if (isDerivedRuntimeStatePath(parts)) return true;
435
437
  if (!includeTranscripts && parts[0] === "transcripts") return true;
436
438
  const basename = parts[parts.length - 1] ?? "";
437
439
  if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
@@ -442,12 +444,7 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
442
444
  function shouldIgnoreIncomingRuntimePath(relPosix: string): boolean {
443
445
  const parts = relPosix.split("/");
444
446
  const basename = parts[parts.length - 1] ?? "";
445
- return isDerivedRuntimeStatePath(parts) || (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-"));
446
- }
447
-
448
- function isDerivedRuntimeStatePath(parts: string[]): boolean {
449
- const basename = parts[parts.length - 1] ?? "";
450
- return isCanonicalRuntimeStatePath(parts) && DERIVED_RUNTIME_STATE_BASENAMES.has(basename);
447
+ return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
451
448
  }
452
449
 
453
450
  function isCanonicalRuntimeStatePath(parts: string[]): boolean {
@@ -683,11 +680,15 @@ export async function buildOfflineSyncChangeset(options: {
683
680
  root: string;
684
681
  sourceId: string;
685
682
  baseFiles?: readonly OfflineSyncFileState[];
683
+ excludePaths?: readonly string[];
686
684
  includeTranscripts?: boolean;
687
685
  now?: Date;
688
686
  readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
689
687
  }): Promise<OfflineSyncChangeset> {
690
688
  const includeTranscripts = options.includeTranscripts !== false;
689
+ const excludedPaths = new Set(
690
+ (options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]")),
691
+ );
691
692
  const base = byPath(filterBaseFilesForMode(
692
693
  normalizeFileStates(options.baseFiles),
693
694
  includeTranscripts,
@@ -704,6 +705,7 @@ export async function buildOfflineSyncChangeset(options: {
704
705
  const changes: OfflineSyncChange[] = [];
705
706
 
706
707
  for (const relPath of unionPaths(base, currentMap)) {
708
+ if (excludedPaths.has(relPath)) continue;
707
709
  const baseEntry = base.get(relPath);
708
710
  const currentEntry = currentMap.get(relPath);
709
711
  if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
@@ -1148,6 +1150,443 @@ async function writeSafeFile(
1148
1150
  }
1149
1151
  }
1150
1152
 
1153
+ export async function applyOfflineSyncFileContentChunk(options: {
1154
+ root: string;
1155
+ sourceId: string;
1156
+ path: string;
1157
+ sha256: string;
1158
+ bytes: number;
1159
+ mtimeMs: number;
1160
+ offset?: number;
1161
+ content: Buffer;
1162
+ baseSha256?: string;
1163
+ includeTranscripts?: boolean;
1164
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1165
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1166
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1167
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
1168
+ }): Promise<OfflineSyncApplyFileContentChunkResult> {
1169
+ const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
1170
+ const sourceId = normalizeSourceId(options.sourceId, "sourceId");
1171
+ const relPath = normalizeRelativePath(options.path, "path");
1172
+ const includeTranscripts = options.includeTranscripts !== false;
1173
+ if (shouldExcludeRelPath(relPath, includeTranscripts)) {
1174
+ throw new Error(`offline sync file content path is excluded: ${relPath}`);
1175
+ }
1176
+ const sha256 = assertSha256(options.sha256, "sha256");
1177
+ const bytes = assertNonNegativeInteger(options.bytes, "bytes");
1178
+ const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
1179
+ const offset = options.offset === undefined
1180
+ ? 0
1181
+ : assertNonNegativeInteger(options.offset, "offset");
1182
+ const baseSha256 = options.baseSha256 === undefined
1183
+ ? undefined
1184
+ : assertSha256(options.baseSha256, "baseSha256");
1185
+ if (!Buffer.isBuffer(options.content)) {
1186
+ throw new Error("content must be a Buffer");
1187
+ }
1188
+ if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
1189
+ throw new Error(
1190
+ `content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`,
1191
+ );
1192
+ }
1193
+ if (bytes > 0 && options.content.length === 0) {
1194
+ throw new Error("content chunk must be non-empty before EOF");
1195
+ }
1196
+ if (offset > bytes || offset + options.content.length > bytes) {
1197
+ throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
1198
+ }
1199
+ if (options.writeFile && !options.writeFileChunks) {
1200
+ throw new Error("offline sync upload storage hooks require writeFileChunks");
1201
+ }
1202
+ if (options.writeFile && !options.writeStagingFile) {
1203
+ throw new Error("offline sync upload storage hooks require writeStagingFile");
1204
+ }
1205
+ if (offset === 0) {
1206
+ await pruneOfflineUploadStaging(root);
1207
+ }
1208
+
1209
+ const upload = await writeOfflineUploadChunk({
1210
+ root,
1211
+ sourceId,
1212
+ relPath,
1213
+ sha256,
1214
+ bytes,
1215
+ offset,
1216
+ content: options.content,
1217
+ readFile: options.readFile,
1218
+ writeFile: options.writeFile,
1219
+ writeStagingFile: options.writeStagingFile,
1220
+ });
1221
+ const done = offset + options.content.length === bytes;
1222
+ const baseResult = {
1223
+ path: relPath,
1224
+ sha256,
1225
+ bytes,
1226
+ mtimeMs,
1227
+ offset,
1228
+ chunkBytes: options.content.length,
1229
+ done,
1230
+ };
1231
+ if (!done) {
1232
+ return {
1233
+ ...baseResult,
1234
+ applied: false,
1235
+ skipped: false,
1236
+ };
1237
+ }
1238
+
1239
+ const digest = await digestOfflineUploadStagingContent({
1240
+ root,
1241
+ upload,
1242
+ readFile: options.readFile,
1243
+ });
1244
+ if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
1245
+ await cleanupOfflineUpload(upload).catch(() => {});
1246
+ throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
1247
+ }
1248
+
1249
+ const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
1250
+ root: root.abs,
1251
+ sourceId: "local",
1252
+ paths: [relPath],
1253
+ includeContent: false,
1254
+ includeTranscripts,
1255
+ readFile: options.readFile,
1256
+ });
1257
+ const currentFile = currentSnapshot.files[0];
1258
+ const uploadedState: OfflineSyncFileState = {
1259
+ path: relPath,
1260
+ sha256,
1261
+ bytes,
1262
+ mtimeMs,
1263
+ };
1264
+
1265
+ try {
1266
+ if (currentFile?.sha256 === sha256) {
1267
+ return {
1268
+ ...baseResult,
1269
+ applied: false,
1270
+ skipped: true,
1271
+ currentFile: toFileState(currentFile),
1272
+ };
1273
+ }
1274
+ if (!baseSha256 && currentFile) {
1275
+ const conflict = await recordConflict({
1276
+ root,
1277
+ relPath,
1278
+ reason: "remote_exists_for_local_create",
1279
+ localSha256: currentFile.sha256,
1280
+ incomingSha256: sha256,
1281
+ writeConflictCopies: false,
1282
+ sourceId,
1283
+ writeFile: options.writeFile,
1284
+ });
1285
+ return {
1286
+ ...baseResult,
1287
+ applied: false,
1288
+ skipped: false,
1289
+ conflict,
1290
+ currentFile: toFileState(currentFile),
1291
+ };
1292
+ }
1293
+ if (baseSha256 && currentFile?.sha256 !== baseSha256) {
1294
+ const conflict = await recordConflict({
1295
+ root,
1296
+ relPath,
1297
+ reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
1298
+ baseSha256,
1299
+ localSha256: currentFile?.sha256,
1300
+ incomingSha256: sha256,
1301
+ writeConflictCopies: false,
1302
+ sourceId,
1303
+ writeFile: options.writeFile,
1304
+ });
1305
+ return {
1306
+ ...baseResult,
1307
+ applied: false,
1308
+ skipped: false,
1309
+ conflict,
1310
+ ...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
1311
+ };
1312
+ }
1313
+
1314
+ await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
1315
+ return {
1316
+ ...baseResult,
1317
+ applied: true,
1318
+ skipped: false,
1319
+ currentFile: uploadedState,
1320
+ };
1321
+ } finally {
1322
+ await cleanupOfflineUpload(upload).catch(() => {});
1323
+ }
1324
+ }
1325
+
1326
+ function offlineUploadRelPath(options: {
1327
+ sourceId: string;
1328
+ relPath: string;
1329
+ sha256: string;
1330
+ bytes: number;
1331
+ }): string {
1332
+ const key = hashText([
1333
+ options.sourceId,
1334
+ options.relPath,
1335
+ options.sha256,
1336
+ String(options.bytes),
1337
+ ].join("\0"));
1338
+ return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
1339
+ }
1340
+
1341
+ async function offlineUploadPath(root: SafeArchiveRoot, options: {
1342
+ sourceId: string;
1343
+ relPath: string;
1344
+ sha256: string;
1345
+ bytes: number;
1346
+ }): Promise<OfflineUploadStaging> {
1347
+ const relPath = offlineUploadRelPath(options);
1348
+ return {
1349
+ kind: "single",
1350
+ relPath,
1351
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1352
+ };
1353
+ }
1354
+
1355
+ async function offlineUploadChunkPath(root: SafeArchiveRoot, options: {
1356
+ sourceId: string;
1357
+ relPath: string;
1358
+ sha256: string;
1359
+ bytes: number;
1360
+ offset: number;
1361
+ }): Promise<OfflineUploadStaging> {
1362
+ const uploadRelPath = offlineUploadRelPath(options);
1363
+ const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
1364
+ return {
1365
+ kind: "chunks",
1366
+ relPath,
1367
+ filePath: await resolveSafeArchiveTarget(root, relPath),
1368
+ };
1369
+ }
1370
+
1371
+ async function writeOfflineUploadChunk(options: {
1372
+ root: SafeArchiveRoot;
1373
+ sourceId: string;
1374
+ relPath: string;
1375
+ sha256: string;
1376
+ bytes: number;
1377
+ offset: number;
1378
+ content: Buffer;
1379
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1380
+ writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1381
+ writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
1382
+ }): Promise<OfflineUploadStaging> {
1383
+ if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
1384
+ throw new Error("offline sync upload chunk storage hooks require readFile");
1385
+ }
1386
+ const uploadRoot = {
1387
+ ...(await offlineUploadPath(options.root, options)),
1388
+ kind: "chunks" as const,
1389
+ };
1390
+ if (options.offset === 0) {
1391
+ await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {});
1392
+ } else {
1393
+ const existing = await stat(uploadRoot.filePath).catch((error: unknown) => {
1394
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1395
+ throw error;
1396
+ });
1397
+ if (!existing || !existing.isDirectory()) {
1398
+ throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
1399
+ }
1400
+ }
1401
+ const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
1402
+
1403
+ const writeStagingFile = options.writeStagingFile ?? options.writeFile;
1404
+ if (writeStagingFile) {
1405
+ // Storage-backed services provide these hooks so secure-store deployments
1406
+ // keep staged partial uploads encrypted at rest without mutating indexes.
1407
+ await writeOfflineUploadContent({
1408
+ root: options.root,
1409
+ relPath: chunk.relPath,
1410
+ filePath: chunk.filePath,
1411
+ content: options.content,
1412
+ writeFile: writeStagingFile,
1413
+ });
1414
+ return uploadRoot;
1415
+ }
1416
+
1417
+ await mkdir(path.dirname(chunk.filePath), { recursive: true });
1418
+ const existingChunk = await lstat(chunk.filePath).catch((error: unknown) => {
1419
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1420
+ throw error;
1421
+ });
1422
+ if (existingChunk?.isSymbolicLink()) {
1423
+ throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
1424
+ }
1425
+ await writeFile(chunk.filePath, options.content, { mode: 0o600 });
1426
+ return uploadRoot;
1427
+ }
1428
+
1429
+ async function pruneOfflineUploadStaging(root: SafeArchiveRoot): Promise<void> {
1430
+ const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
1431
+ const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
1432
+ const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error: unknown) => {
1433
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
1434
+ throw error;
1435
+ });
1436
+ const now = Date.now();
1437
+ await Promise.all(entries.map(async (entry) => {
1438
+ if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
1439
+ const relPath = `${uploadsRelPath}/${entry.name}`;
1440
+ const filePath = await resolveSafeArchiveTarget(root, relPath);
1441
+ const info = await lstat(filePath).catch((error: unknown) => {
1442
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1443
+ throw error;
1444
+ });
1445
+ if (!info) return;
1446
+ if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
1447
+ await rm(filePath, { recursive: true, force: true });
1448
+ }));
1449
+ }
1450
+
1451
+ async function* readOfflineUploadStagingChunks(options: {
1452
+ root: SafeArchiveRoot;
1453
+ upload: OfflineUploadStaging;
1454
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1455
+ }): AsyncGenerator<Buffer> {
1456
+ if (options.upload.kind === "single") {
1457
+ yield await readOfflineUploadContent({
1458
+ root: options.root,
1459
+ relPath: options.upload.relPath,
1460
+ filePath: options.upload.filePath,
1461
+ readFile: options.readFile,
1462
+ });
1463
+ return;
1464
+ }
1465
+
1466
+ const entries = await readdir(options.upload.filePath);
1467
+ const chunkNames = entries
1468
+ .filter((entry) => /^\d{20}\.part$/.test(entry))
1469
+ .sort();
1470
+ if (chunkNames.length === 0) {
1471
+ throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
1472
+ }
1473
+ let expectedOffset = 0;
1474
+ for (const chunkName of chunkNames) {
1475
+ const offset = Number(chunkName.slice(0, 20));
1476
+ if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
1477
+ throw new Error(
1478
+ `offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`,
1479
+ );
1480
+ }
1481
+ const relPath = `${options.upload.relPath}/${chunkName}`;
1482
+ const filePath = await resolveSafeArchiveTarget(options.root, relPath);
1483
+ const content = await readOfflineUploadContent({
1484
+ root: options.root,
1485
+ relPath,
1486
+ filePath,
1487
+ readFile: options.readFile,
1488
+ });
1489
+ expectedOffset += content.length;
1490
+ yield content;
1491
+ }
1492
+ }
1493
+
1494
+ async function digestOfflineUploadStagingContent(options: {
1495
+ root: SafeArchiveRoot;
1496
+ upload: OfflineUploadStaging;
1497
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1498
+ }): Promise<{ sha256: string; bytes: number }> {
1499
+ const hash = createHash("sha256");
1500
+ let bytes = 0;
1501
+ for await (const chunk of readOfflineUploadStagingChunks(options)) {
1502
+ hash.update(chunk);
1503
+ bytes += chunk.length;
1504
+ }
1505
+ return { sha256: hash.digest("hex"), bytes };
1506
+ }
1507
+
1508
+ async function writeSafeFileFromUpload(
1509
+ root: SafeArchiveRoot,
1510
+ relPath: string,
1511
+ upload: OfflineUploadStaging,
1512
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
1513
+ writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
1514
+ ): Promise<void> {
1515
+ const target = await resolveSafeArchiveTarget(root, relPath);
1516
+ const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
1517
+ if (writeFileChunks) {
1518
+ await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
1519
+ return;
1520
+ }
1521
+
1522
+ await mkdir(path.dirname(target), { recursive: true });
1523
+ const tmp = path.join(
1524
+ path.dirname(target),
1525
+ `.remnic-sync.${process.pid}.${randomUUID()}.tmp`,
1526
+ );
1527
+ const handle = await open(tmp, "w", 0o600);
1528
+ try {
1529
+ for await (const chunk of chunks) {
1530
+ if (chunk.length > 0) await handle.write(chunk);
1531
+ }
1532
+ await handle.close();
1533
+ const targetStat = await lstat(target).catch((error: unknown) => {
1534
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
1535
+ throw error;
1536
+ });
1537
+ if (targetStat?.isSymbolicLink()) {
1538
+ throw new Error(`offline sync target is a symlink: ${relPath}`);
1539
+ }
1540
+ await rename(tmp, target);
1541
+ } catch (error) {
1542
+ await handle.close().catch(() => {});
1543
+ await unlink(tmp).catch(() => {});
1544
+ throw error;
1545
+ }
1546
+ }
1547
+
1548
+ async function cleanupOfflineUpload(upload: OfflineUploadStaging): Promise<void> {
1549
+ if (upload.kind === "chunks") {
1550
+ await rm(upload.filePath, { recursive: true, force: true });
1551
+ return;
1552
+ }
1553
+ await unlink(upload.filePath).catch((error: unknown) => {
1554
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
1555
+ throw error;
1556
+ });
1557
+ }
1558
+
1559
+ async function readOfflineUploadContent(options: {
1560
+ root: SafeArchiveRoot;
1561
+ relPath: string;
1562
+ filePath: string;
1563
+ readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
1564
+ }): Promise<Buffer> {
1565
+ if (options.readFile) {
1566
+ return options.readFile({
1567
+ root: options.root.abs,
1568
+ path: options.relPath,
1569
+ filePath: options.filePath,
1570
+ });
1571
+ }
1572
+ return readFile(options.filePath);
1573
+ }
1574
+
1575
+ async function writeOfflineUploadContent(options: {
1576
+ root: SafeArchiveRoot;
1577
+ relPath: string;
1578
+ filePath: string;
1579
+ content: Buffer;
1580
+ writeFile: (target: OfflineSyncFileWriteTarget) => Promise<void>;
1581
+ }): Promise<void> {
1582
+ await options.writeFile({
1583
+ root: options.root.abs,
1584
+ path: options.relPath,
1585
+ filePath: options.filePath,
1586
+ content: options.content,
1587
+ });
1588
+ }
1589
+
1151
1590
  async function deleteSafeFile(
1152
1591
  root: SafeArchiveRoot,
1153
1592
  relPath: string,
@@ -43,10 +43,21 @@
43
43
  * Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
44
44
  */
45
45
 
46
- import { lstat, mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
46
+ import { createCipheriv, randomBytes } from "node:crypto";
47
+ import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
47
48
  import path from "node:path";
48
49
 
49
- import { generateSalt, open, seal } from "./cipher.js";
50
+ import {
51
+ AUTH_TAG_LENGTH,
52
+ ENVELOPE_HEADER_SIZE,
53
+ ENVELOPE_LAYOUT,
54
+ ENVELOPE_SALT_LENGTH,
55
+ ENVELOPE_VERSION,
56
+ IV_LENGTH,
57
+ generateSalt,
58
+ open as openEnvelope,
59
+ seal,
60
+ } from "./cipher.js";
50
61
 
51
62
  // ---------------------------------------------------------------------------
52
63
  // Error classes
@@ -157,7 +168,7 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
157
168
  }
158
169
  const envelope = buf.subarray(MAGIC_HEADER_SIZE);
159
170
  try {
160
- return open(key, envelope, aad ? { aad } : {});
171
+ return openEnvelope(key, envelope, aad ? { aad } : {});
161
172
  } catch (err) {
162
173
  const msg = err instanceof Error ? err.message : String(err);
163
174
  throw new SecureStoreDecryptError(
@@ -166,6 +177,13 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
166
177
  }
167
178
  }
168
179
 
180
+ function buildHeaderAad(salt: Uint8Array): Buffer {
181
+ const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
182
+ out.writeUInt8(ENVELOPE_VERSION, 0);
183
+ Buffer.from(salt).copy(out, 1);
184
+ return out;
185
+ }
186
+
169
187
  // ---------------------------------------------------------------------------
170
188
  // Path → AAD helper
171
189
  // ---------------------------------------------------------------------------
@@ -293,6 +311,69 @@ export async function writeMaybeEncryptedFile(
293
311
  }
294
312
  }
295
313
 
314
+ export async function writeMaybeEncryptedFileFromChunks(
315
+ filePath: string,
316
+ chunks: AsyncIterable<Buffer>,
317
+ key: Buffer | null,
318
+ options: WriteMaybeEncryptedFileOptions = {},
319
+ memoryDir?: string,
320
+ ): Promise<void> {
321
+ const { mode = 0o600, atomic = true } = options;
322
+ await mkdir(path.dirname(filePath), { recursive: true });
323
+ const writePath = atomic ? `${filePath}.tmp-${process.pid}-${Date.now()}` : filePath;
324
+ let completed = false;
325
+ try {
326
+ const handle = await openFile(writePath, "w", mode);
327
+ try {
328
+ if (key !== null) {
329
+ const salt = generateSalt();
330
+ const iv = randomBytes(IV_LENGTH);
331
+ const header = Buffer.alloc(MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
332
+ MAGIC_BYTES.copy(header, 0);
333
+ header.writeUInt8(FILE_FORMAT_VERSION, MAGIC_BYTES.length);
334
+ header.writeUInt8(FILE_FORMAT_FLAGS, MAGIC_BYTES.length + 1);
335
+ const envelopeOffset = MAGIC_HEADER_SIZE;
336
+ header.writeUInt8(ENVELOPE_VERSION, envelopeOffset + ENVELOPE_LAYOUT.version);
337
+ salt.copy(header, envelopeOffset + ENVELOPE_LAYOUT.salt);
338
+ iv.copy(header, envelopeOffset + ENVELOPE_LAYOUT.iv);
339
+ await handle.write(header);
340
+
341
+ const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
342
+ const aad = filePathAad(filePath, memoryDir);
343
+ cipher.setAAD(Buffer.concat([buildHeaderAad(salt), aad]));
344
+ for await (const chunk of chunks) {
345
+ if (chunk.length === 0) continue;
346
+ const encrypted = cipher.update(chunk);
347
+ if (encrypted.length > 0) await handle.write(encrypted);
348
+ }
349
+ const final = cipher.final();
350
+ if (final.length > 0) await handle.write(final);
351
+ const authTag = cipher.getAuthTag();
352
+ await handle.write(
353
+ authTag,
354
+ 0,
355
+ authTag.length,
356
+ MAGIC_HEADER_SIZE + ENVELOPE_LAYOUT.authTag,
357
+ );
358
+ } else {
359
+ for await (const chunk of chunks) {
360
+ if (chunk.length > 0) await handle.write(chunk);
361
+ }
362
+ }
363
+ } finally {
364
+ await handle.close();
365
+ }
366
+ if (atomic) {
367
+ await rename(writePath, filePath);
368
+ }
369
+ completed = true;
370
+ } finally {
371
+ if (!completed && atomic) {
372
+ await unlink(writePath).catch(() => {});
373
+ }
374
+ }
375
+ }
376
+
296
377
  // ---------------------------------------------------------------------------
297
378
  // Migration
298
379
  // ---------------------------------------------------------------------------