@remnic/core 1.1.22 → 1.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +15 -15
- package/dist/access-http.d.ts +9 -1
- package/dist/access-http.js +9 -9
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-mcp.js +8 -8
- package/dist/access-schema.js +3 -3
- package/dist/{access-service-DT9L2DW4.d.ts → access-service-CEyV8XJ5.d.ts} +19 -2
- package/dist/access-service.d.ts +1 -1
- package/dist/access-service.js +6 -6
- package/dist/briefing.js +3 -3
- package/dist/causal-consolidation.js +4 -4
- package/dist/{chunk-YO3AZEE5.js → chunk-25YQM6XW.js} +3 -3
- package/dist/{chunk-TLM762GT.js → chunk-2WIPXV3Y.js} +2 -2
- package/dist/{chunk-QOHBYVZG.js → chunk-3F24QTRI.js} +2 -2
- package/dist/{chunk-5IQC4OG6.js → chunk-4H6DURG6.js} +2 -2
- package/dist/{chunk-NOQ74SJN.js → chunk-7D6O46PF.js} +2 -2
- package/dist/{chunk-VMQRBXJ5.js → chunk-7E7SZRPP.js} +2 -2
- package/dist/{chunk-7Q2P774N.js → chunk-F33CJ5CH.js} +13 -3
- package/dist/chunk-F33CJ5CH.js.map +1 -0
- package/dist/{chunk-26OQECWH.js → chunk-FHXVW3L4.js} +4 -4
- package/dist/{chunk-UA6OCL6S.js → chunk-HWF42K6J.js} +103 -4
- package/dist/chunk-HWF42K6J.js.map +1 -0
- package/dist/{chunk-FSODDMR2.js → chunk-IANK6Y5W.js} +2 -2
- package/dist/{chunk-FF46Q3SN.js → chunk-JKXFF3NT.js} +356 -32
- package/dist/chunk-JKXFF3NT.js.map +1 -0
- package/dist/{chunk-NGPO6S3M.js → chunk-MM5EBZVW.js} +42 -5
- package/dist/chunk-MM5EBZVW.js.map +1 -0
- package/dist/{chunk-GGCJ253V.js → chunk-MVAOT247.js} +8 -8
- package/dist/{chunk-SH5S7XYD.js → chunk-MXFBBHJU.js} +72 -2
- package/dist/chunk-MXFBBHJU.js.map +1 -0
- package/dist/{chunk-SZKCBLS5.js → chunk-PUXCIHRL.js} +2 -2
- package/dist/{chunk-2IRT26RZ.js → chunk-QYHQ2JHL.js} +2 -2
- package/dist/{chunk-LDJANWTK.js → chunk-RA73CTVY.js} +12 -12
- package/dist/{chunk-CN4P6SVA.js → chunk-RCZRL5BE.js} +2 -2
- package/dist/{chunk-SGIXDVSF.js → chunk-S27EXIHY.js} +2 -2
- package/dist/{chunk-5ML4TH3E.js → chunk-TFORLO3O.js} +4 -4
- package/dist/{chunk-TOFUTKQN.js → chunk-TR4DK5OH.js} +2 -2
- package/dist/{chunk-6ORWKANA.js → chunk-VYU7PXUS.js} +2 -2
- package/dist/{chunk-FFU4GMST.js → chunk-WNARATI3.js} +2 -2
- package/dist/{chunk-KSFBM6TV.js → chunk-YITUHONZ.js} +2 -2
- package/dist/{cli-BN0CkYzI.d.ts → cli-BguVmIwO.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +18 -18
- package/dist/compounding/engine.js +3 -3
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/index.js +3 -3
- package/dist/entity-retrieval.js +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.js +26 -24
- package/dist/index.js.map +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/storage.js +3 -3
- package/dist/offline-sync.d.ts +36 -1
- package/dist/offline-sync.js +4 -2
- package/dist/operator-toolkit.js +6 -6
- package/dist/orchestrator.js +11 -11
- package/dist/schemas.d.ts +22 -22
- package/dist/secure-store/index.js +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/storage.d.ts +2 -0
- package/dist/storage.js +2 -2
- package/dist/transfer/types.d.ts +12 -12
- package/dist/verified-recall.js +3 -3
- package/package.json +1 -1
- package/src/access-http.test.ts +176 -0
- package/src/access-http.ts +116 -0
- package/src/access-service-offline-file-content.test.ts +37 -0
- package/src/access-service.ts +70 -0
- package/src/index.ts +2 -0
- package/src/offline-sync.test.ts +395 -79
- package/src/offline-sync.ts +471 -32
- package/src/secure-store/secure-fs.ts +84 -3
- package/src/storage.ts +12 -0
- package/dist/chunk-7Q2P774N.js.map +0 -1
- package/dist/chunk-FF46Q3SN.js.map +0 -1
- package/dist/chunk-NGPO6S3M.js.map +0 -1
- package/dist/chunk-SH5S7XYD.js.map +0 -1
- package/dist/chunk-UA6OCL6S.js.map +0 -1
- /package/dist/{chunk-YO3AZEE5.js.map → chunk-25YQM6XW.js.map} +0 -0
- /package/dist/{chunk-TLM762GT.js.map → chunk-2WIPXV3Y.js.map} +0 -0
- /package/dist/{chunk-QOHBYVZG.js.map → chunk-3F24QTRI.js.map} +0 -0
- /package/dist/{chunk-5IQC4OG6.js.map → chunk-4H6DURG6.js.map} +0 -0
- /package/dist/{chunk-NOQ74SJN.js.map → chunk-7D6O46PF.js.map} +0 -0
- /package/dist/{chunk-VMQRBXJ5.js.map → chunk-7E7SZRPP.js.map} +0 -0
- /package/dist/{chunk-26OQECWH.js.map → chunk-FHXVW3L4.js.map} +0 -0
- /package/dist/{chunk-FSODDMR2.js.map → chunk-IANK6Y5W.js.map} +0 -0
- /package/dist/{chunk-GGCJ253V.js.map → chunk-MVAOT247.js.map} +0 -0
- /package/dist/{chunk-SZKCBLS5.js.map → chunk-PUXCIHRL.js.map} +0 -0
- /package/dist/{chunk-2IRT26RZ.js.map → chunk-QYHQ2JHL.js.map} +0 -0
- /package/dist/{chunk-LDJANWTK.js.map → chunk-RA73CTVY.js.map} +0 -0
- /package/dist/{chunk-CN4P6SVA.js.map → chunk-RCZRL5BE.js.map} +0 -0
- /package/dist/{chunk-SGIXDVSF.js.map → chunk-S27EXIHY.js.map} +0 -0
- /package/dist/{chunk-5ML4TH3E.js.map → chunk-TFORLO3O.js.map} +0 -0
- /package/dist/{chunk-TOFUTKQN.js.map → chunk-TR4DK5OH.js.map} +0 -0
- /package/dist/{chunk-6ORWKANA.js.map → chunk-VYU7PXUS.js.map} +0 -0
- /package/dist/{chunk-FFU4GMST.js.map → chunk-WNARATI3.js.map} +0 -0
- /package/dist/{chunk-KSFBM6TV.js.map → chunk-YITUHONZ.js.map} +0 -0
package/src/offline-sync.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
readdir,
|
|
7
7
|
readFile,
|
|
8
8
|
rename,
|
|
9
|
+
rm,
|
|
9
10
|
stat,
|
|
10
11
|
unlink,
|
|
11
12
|
writeFile,
|
|
@@ -130,6 +131,12 @@ export interface OfflineSyncFileWriteTarget extends OfflineSyncFileTarget {
|
|
|
130
131
|
content: Buffer;
|
|
131
132
|
}
|
|
132
133
|
|
|
134
|
+
export interface OfflineSyncFileWriteChunksTarget extends OfflineSyncFileTarget {
|
|
135
|
+
chunks: AsyncIterable<Buffer>;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface OfflineSyncFileStagingWriteTarget extends OfflineSyncFileWriteTarget {}
|
|
139
|
+
|
|
133
140
|
export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState, "sha256"> {
|
|
134
141
|
sha256?: string;
|
|
135
142
|
offset: number;
|
|
@@ -137,6 +144,26 @@ export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState,
|
|
|
137
144
|
content: Buffer;
|
|
138
145
|
}
|
|
139
146
|
|
|
147
|
+
export interface OfflineSyncApplyFileContentChunkResult {
|
|
148
|
+
path: string;
|
|
149
|
+
sha256: string;
|
|
150
|
+
bytes: number;
|
|
151
|
+
mtimeMs: number;
|
|
152
|
+
offset: number;
|
|
153
|
+
chunkBytes: number;
|
|
154
|
+
done: boolean;
|
|
155
|
+
applied: boolean;
|
|
156
|
+
skipped: boolean;
|
|
157
|
+
conflict?: OfflineSyncConflict;
|
|
158
|
+
currentFile?: OfflineSyncFileState;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
interface OfflineUploadStaging {
|
|
162
|
+
kind: "single" | "chunks";
|
|
163
|
+
relPath: string;
|
|
164
|
+
filePath: string;
|
|
165
|
+
}
|
|
166
|
+
|
|
140
167
|
interface OfflineSyncFileRecordOptions {
|
|
141
168
|
root: SafeArchiveRoot;
|
|
142
169
|
relPath: string;
|
|
@@ -146,35 +173,11 @@ interface OfflineSyncFileRecordOptions {
|
|
|
146
173
|
}
|
|
147
174
|
|
|
148
175
|
const SYNC_INTERNAL_DIR = ".offline-sync";
|
|
176
|
+
const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
149
177
|
const EXCLUDED_FILE_NAMES = new Set([
|
|
150
178
|
".sync-state.json",
|
|
151
179
|
]);
|
|
152
180
|
|
|
153
|
-
const DERIVED_RUNTIME_STATE_BASENAMES = new Set([
|
|
154
|
-
".artifact-write-version.log",
|
|
155
|
-
".memory-status-version.log",
|
|
156
|
-
"fact-hashes.ready",
|
|
157
|
-
"fact-hashes.txt",
|
|
158
|
-
"buffer-surprise-ledger.jsonl",
|
|
159
|
-
"buffer.json",
|
|
160
|
-
"embeddings.json",
|
|
161
|
-
"entity-mention-index.json",
|
|
162
|
-
"index_tags.json",
|
|
163
|
-
"index_time.json",
|
|
164
|
-
"last_graph_recall.json",
|
|
165
|
-
"last_intent.json",
|
|
166
|
-
"last_qmd_recall.json",
|
|
167
|
-
"last_recall.json",
|
|
168
|
-
"lcm.sqlite",
|
|
169
|
-
"lcm.sqlite-shm",
|
|
170
|
-
"lcm.sqlite-wal",
|
|
171
|
-
"memory-lifecycle-ledger.jsonl",
|
|
172
|
-
"memory-projection.sqlite",
|
|
173
|
-
"memory-projection.sqlite-shm",
|
|
174
|
-
"memory-projection.sqlite-wal",
|
|
175
|
-
"recall_impressions.jsonl",
|
|
176
|
-
]);
|
|
177
|
-
|
|
178
181
|
const EXCLUDED_FILE_PREFIXES = [
|
|
179
182
|
".remnic-sync.",
|
|
180
183
|
".remnic-sync-state.",
|
|
@@ -431,7 +434,6 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
|
|
|
431
434
|
const parts = relPosix.split("/");
|
|
432
435
|
if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
|
|
433
436
|
if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
|
|
434
|
-
if (isDerivedRuntimeStatePath(parts)) return true;
|
|
435
437
|
if (!includeTranscripts && parts[0] === "transcripts") return true;
|
|
436
438
|
const basename = parts[parts.length - 1] ?? "";
|
|
437
439
|
if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
|
|
@@ -442,12 +444,7 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
|
|
|
442
444
|
function shouldIgnoreIncomingRuntimePath(relPosix: string): boolean {
|
|
443
445
|
const parts = relPosix.split("/");
|
|
444
446
|
const basename = parts[parts.length - 1] ?? "";
|
|
445
|
-
return
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
function isDerivedRuntimeStatePath(parts: string[]): boolean {
|
|
449
|
-
const basename = parts[parts.length - 1] ?? "";
|
|
450
|
-
return isCanonicalRuntimeStatePath(parts) && DERIVED_RUNTIME_STATE_BASENAMES.has(basename);
|
|
447
|
+
return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
|
|
451
448
|
}
|
|
452
449
|
|
|
453
450
|
function isCanonicalRuntimeStatePath(parts: string[]): boolean {
|
|
@@ -683,11 +680,15 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
683
680
|
root: string;
|
|
684
681
|
sourceId: string;
|
|
685
682
|
baseFiles?: readonly OfflineSyncFileState[];
|
|
683
|
+
excludePaths?: readonly string[];
|
|
686
684
|
includeTranscripts?: boolean;
|
|
687
685
|
now?: Date;
|
|
688
686
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
689
687
|
}): Promise<OfflineSyncChangeset> {
|
|
690
688
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
689
|
+
const excludedPaths = new Set(
|
|
690
|
+
(options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]")),
|
|
691
|
+
);
|
|
691
692
|
const base = byPath(filterBaseFilesForMode(
|
|
692
693
|
normalizeFileStates(options.baseFiles),
|
|
693
694
|
includeTranscripts,
|
|
@@ -704,6 +705,7 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
704
705
|
const changes: OfflineSyncChange[] = [];
|
|
705
706
|
|
|
706
707
|
for (const relPath of unionPaths(base, currentMap)) {
|
|
708
|
+
if (excludedPaths.has(relPath)) continue;
|
|
707
709
|
const baseEntry = base.get(relPath);
|
|
708
710
|
const currentEntry = currentMap.get(relPath);
|
|
709
711
|
if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
|
|
@@ -1148,6 +1150,443 @@ async function writeSafeFile(
|
|
|
1148
1150
|
}
|
|
1149
1151
|
}
|
|
1150
1152
|
|
|
1153
|
+
export async function applyOfflineSyncFileContentChunk(options: {
|
|
1154
|
+
root: string;
|
|
1155
|
+
sourceId: string;
|
|
1156
|
+
path: string;
|
|
1157
|
+
sha256: string;
|
|
1158
|
+
bytes: number;
|
|
1159
|
+
mtimeMs: number;
|
|
1160
|
+
offset?: number;
|
|
1161
|
+
content: Buffer;
|
|
1162
|
+
baseSha256?: string;
|
|
1163
|
+
includeTranscripts?: boolean;
|
|
1164
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1165
|
+
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1166
|
+
writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
|
|
1167
|
+
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
|
|
1168
|
+
}): Promise<OfflineSyncApplyFileContentChunkResult> {
|
|
1169
|
+
const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
|
|
1170
|
+
const sourceId = normalizeSourceId(options.sourceId, "sourceId");
|
|
1171
|
+
const relPath = normalizeRelativePath(options.path, "path");
|
|
1172
|
+
const includeTranscripts = options.includeTranscripts !== false;
|
|
1173
|
+
if (shouldExcludeRelPath(relPath, includeTranscripts)) {
|
|
1174
|
+
throw new Error(`offline sync file content path is excluded: ${relPath}`);
|
|
1175
|
+
}
|
|
1176
|
+
const sha256 = assertSha256(options.sha256, "sha256");
|
|
1177
|
+
const bytes = assertNonNegativeInteger(options.bytes, "bytes");
|
|
1178
|
+
const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
|
|
1179
|
+
const offset = options.offset === undefined
|
|
1180
|
+
? 0
|
|
1181
|
+
: assertNonNegativeInteger(options.offset, "offset");
|
|
1182
|
+
const baseSha256 = options.baseSha256 === undefined
|
|
1183
|
+
? undefined
|
|
1184
|
+
: assertSha256(options.baseSha256, "baseSha256");
|
|
1185
|
+
if (!Buffer.isBuffer(options.content)) {
|
|
1186
|
+
throw new Error("content must be a Buffer");
|
|
1187
|
+
}
|
|
1188
|
+
if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
|
|
1189
|
+
throw new Error(
|
|
1190
|
+
`content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`,
|
|
1191
|
+
);
|
|
1192
|
+
}
|
|
1193
|
+
if (bytes > 0 && options.content.length === 0) {
|
|
1194
|
+
throw new Error("content chunk must be non-empty before EOF");
|
|
1195
|
+
}
|
|
1196
|
+
if (offset > bytes || offset + options.content.length > bytes) {
|
|
1197
|
+
throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
|
|
1198
|
+
}
|
|
1199
|
+
if (options.writeFile && !options.writeFileChunks) {
|
|
1200
|
+
throw new Error("offline sync upload storage hooks require writeFileChunks");
|
|
1201
|
+
}
|
|
1202
|
+
if (options.writeFile && !options.writeStagingFile) {
|
|
1203
|
+
throw new Error("offline sync upload storage hooks require writeStagingFile");
|
|
1204
|
+
}
|
|
1205
|
+
if (offset === 0) {
|
|
1206
|
+
await pruneOfflineUploadStaging(root);
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
const upload = await writeOfflineUploadChunk({
|
|
1210
|
+
root,
|
|
1211
|
+
sourceId,
|
|
1212
|
+
relPath,
|
|
1213
|
+
sha256,
|
|
1214
|
+
bytes,
|
|
1215
|
+
offset,
|
|
1216
|
+
content: options.content,
|
|
1217
|
+
readFile: options.readFile,
|
|
1218
|
+
writeFile: options.writeFile,
|
|
1219
|
+
writeStagingFile: options.writeStagingFile,
|
|
1220
|
+
});
|
|
1221
|
+
const done = offset + options.content.length === bytes;
|
|
1222
|
+
const baseResult = {
|
|
1223
|
+
path: relPath,
|
|
1224
|
+
sha256,
|
|
1225
|
+
bytes,
|
|
1226
|
+
mtimeMs,
|
|
1227
|
+
offset,
|
|
1228
|
+
chunkBytes: options.content.length,
|
|
1229
|
+
done,
|
|
1230
|
+
};
|
|
1231
|
+
if (!done) {
|
|
1232
|
+
return {
|
|
1233
|
+
...baseResult,
|
|
1234
|
+
applied: false,
|
|
1235
|
+
skipped: false,
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
const digest = await digestOfflineUploadStagingContent({
|
|
1240
|
+
root,
|
|
1241
|
+
upload,
|
|
1242
|
+
readFile: options.readFile,
|
|
1243
|
+
});
|
|
1244
|
+
if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
|
|
1245
|
+
await cleanupOfflineUpload(upload).catch(() => {});
|
|
1246
|
+
throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
|
|
1250
|
+
root: root.abs,
|
|
1251
|
+
sourceId: "local",
|
|
1252
|
+
paths: [relPath],
|
|
1253
|
+
includeContent: false,
|
|
1254
|
+
includeTranscripts,
|
|
1255
|
+
readFile: options.readFile,
|
|
1256
|
+
});
|
|
1257
|
+
const currentFile = currentSnapshot.files[0];
|
|
1258
|
+
const uploadedState: OfflineSyncFileState = {
|
|
1259
|
+
path: relPath,
|
|
1260
|
+
sha256,
|
|
1261
|
+
bytes,
|
|
1262
|
+
mtimeMs,
|
|
1263
|
+
};
|
|
1264
|
+
|
|
1265
|
+
try {
|
|
1266
|
+
if (currentFile?.sha256 === sha256) {
|
|
1267
|
+
return {
|
|
1268
|
+
...baseResult,
|
|
1269
|
+
applied: false,
|
|
1270
|
+
skipped: true,
|
|
1271
|
+
currentFile: toFileState(currentFile),
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1274
|
+
if (!baseSha256 && currentFile) {
|
|
1275
|
+
const conflict = await recordConflict({
|
|
1276
|
+
root,
|
|
1277
|
+
relPath,
|
|
1278
|
+
reason: "remote_exists_for_local_create",
|
|
1279
|
+
localSha256: currentFile.sha256,
|
|
1280
|
+
incomingSha256: sha256,
|
|
1281
|
+
writeConflictCopies: false,
|
|
1282
|
+
sourceId,
|
|
1283
|
+
writeFile: options.writeFile,
|
|
1284
|
+
});
|
|
1285
|
+
return {
|
|
1286
|
+
...baseResult,
|
|
1287
|
+
applied: false,
|
|
1288
|
+
skipped: false,
|
|
1289
|
+
conflict,
|
|
1290
|
+
currentFile: toFileState(currentFile),
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
if (baseSha256 && currentFile?.sha256 !== baseSha256) {
|
|
1294
|
+
const conflict = await recordConflict({
|
|
1295
|
+
root,
|
|
1296
|
+
relPath,
|
|
1297
|
+
reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
|
|
1298
|
+
baseSha256,
|
|
1299
|
+
localSha256: currentFile?.sha256,
|
|
1300
|
+
incomingSha256: sha256,
|
|
1301
|
+
writeConflictCopies: false,
|
|
1302
|
+
sourceId,
|
|
1303
|
+
writeFile: options.writeFile,
|
|
1304
|
+
});
|
|
1305
|
+
return {
|
|
1306
|
+
...baseResult,
|
|
1307
|
+
applied: false,
|
|
1308
|
+
skipped: false,
|
|
1309
|
+
conflict,
|
|
1310
|
+
...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
|
|
1315
|
+
return {
|
|
1316
|
+
...baseResult,
|
|
1317
|
+
applied: true,
|
|
1318
|
+
skipped: false,
|
|
1319
|
+
currentFile: uploadedState,
|
|
1320
|
+
};
|
|
1321
|
+
} finally {
|
|
1322
|
+
await cleanupOfflineUpload(upload).catch(() => {});
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function offlineUploadRelPath(options: {
|
|
1327
|
+
sourceId: string;
|
|
1328
|
+
relPath: string;
|
|
1329
|
+
sha256: string;
|
|
1330
|
+
bytes: number;
|
|
1331
|
+
}): string {
|
|
1332
|
+
const key = hashText([
|
|
1333
|
+
options.sourceId,
|
|
1334
|
+
options.relPath,
|
|
1335
|
+
options.sha256,
|
|
1336
|
+
String(options.bytes),
|
|
1337
|
+
].join("\0"));
|
|
1338
|
+
return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
async function offlineUploadPath(root: SafeArchiveRoot, options: {
|
|
1342
|
+
sourceId: string;
|
|
1343
|
+
relPath: string;
|
|
1344
|
+
sha256: string;
|
|
1345
|
+
bytes: number;
|
|
1346
|
+
}): Promise<OfflineUploadStaging> {
|
|
1347
|
+
const relPath = offlineUploadRelPath(options);
|
|
1348
|
+
return {
|
|
1349
|
+
kind: "single",
|
|
1350
|
+
relPath,
|
|
1351
|
+
filePath: await resolveSafeArchiveTarget(root, relPath),
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
async function offlineUploadChunkPath(root: SafeArchiveRoot, options: {
|
|
1356
|
+
sourceId: string;
|
|
1357
|
+
relPath: string;
|
|
1358
|
+
sha256: string;
|
|
1359
|
+
bytes: number;
|
|
1360
|
+
offset: number;
|
|
1361
|
+
}): Promise<OfflineUploadStaging> {
|
|
1362
|
+
const uploadRelPath = offlineUploadRelPath(options);
|
|
1363
|
+
const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
|
|
1364
|
+
return {
|
|
1365
|
+
kind: "chunks",
|
|
1366
|
+
relPath,
|
|
1367
|
+
filePath: await resolveSafeArchiveTarget(root, relPath),
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
async function writeOfflineUploadChunk(options: {
|
|
1372
|
+
root: SafeArchiveRoot;
|
|
1373
|
+
sourceId: string;
|
|
1374
|
+
relPath: string;
|
|
1375
|
+
sha256: string;
|
|
1376
|
+
bytes: number;
|
|
1377
|
+
offset: number;
|
|
1378
|
+
content: Buffer;
|
|
1379
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1380
|
+
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1381
|
+
writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
|
|
1382
|
+
}): Promise<OfflineUploadStaging> {
|
|
1383
|
+
if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
|
|
1384
|
+
throw new Error("offline sync upload chunk storage hooks require readFile");
|
|
1385
|
+
}
|
|
1386
|
+
const uploadRoot = {
|
|
1387
|
+
...(await offlineUploadPath(options.root, options)),
|
|
1388
|
+
kind: "chunks" as const,
|
|
1389
|
+
};
|
|
1390
|
+
if (options.offset === 0) {
|
|
1391
|
+
await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {});
|
|
1392
|
+
} else {
|
|
1393
|
+
const existing = await stat(uploadRoot.filePath).catch((error: unknown) => {
|
|
1394
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1395
|
+
throw error;
|
|
1396
|
+
});
|
|
1397
|
+
if (!existing || !existing.isDirectory()) {
|
|
1398
|
+
throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
|
|
1402
|
+
|
|
1403
|
+
const writeStagingFile = options.writeStagingFile ?? options.writeFile;
|
|
1404
|
+
if (writeStagingFile) {
|
|
1405
|
+
// Storage-backed services provide these hooks so secure-store deployments
|
|
1406
|
+
// keep staged partial uploads encrypted at rest without mutating indexes.
|
|
1407
|
+
await writeOfflineUploadContent({
|
|
1408
|
+
root: options.root,
|
|
1409
|
+
relPath: chunk.relPath,
|
|
1410
|
+
filePath: chunk.filePath,
|
|
1411
|
+
content: options.content,
|
|
1412
|
+
writeFile: writeStagingFile,
|
|
1413
|
+
});
|
|
1414
|
+
return uploadRoot;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
await mkdir(path.dirname(chunk.filePath), { recursive: true });
|
|
1418
|
+
const existingChunk = await lstat(chunk.filePath).catch((error: unknown) => {
|
|
1419
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1420
|
+
throw error;
|
|
1421
|
+
});
|
|
1422
|
+
if (existingChunk?.isSymbolicLink()) {
|
|
1423
|
+
throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
|
|
1424
|
+
}
|
|
1425
|
+
await writeFile(chunk.filePath, options.content, { mode: 0o600 });
|
|
1426
|
+
return uploadRoot;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
async function pruneOfflineUploadStaging(root: SafeArchiveRoot): Promise<void> {
|
|
1430
|
+
const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
|
|
1431
|
+
const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
|
|
1432
|
+
const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error: unknown) => {
|
|
1433
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
1434
|
+
throw error;
|
|
1435
|
+
});
|
|
1436
|
+
const now = Date.now();
|
|
1437
|
+
await Promise.all(entries.map(async (entry) => {
|
|
1438
|
+
if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
|
|
1439
|
+
const relPath = `${uploadsRelPath}/${entry.name}`;
|
|
1440
|
+
const filePath = await resolveSafeArchiveTarget(root, relPath);
|
|
1441
|
+
const info = await lstat(filePath).catch((error: unknown) => {
|
|
1442
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1443
|
+
throw error;
|
|
1444
|
+
});
|
|
1445
|
+
if (!info) return;
|
|
1446
|
+
if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
|
|
1447
|
+
await rm(filePath, { recursive: true, force: true });
|
|
1448
|
+
}));
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
async function* readOfflineUploadStagingChunks(options: {
|
|
1452
|
+
root: SafeArchiveRoot;
|
|
1453
|
+
upload: OfflineUploadStaging;
|
|
1454
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1455
|
+
}): AsyncGenerator<Buffer> {
|
|
1456
|
+
if (options.upload.kind === "single") {
|
|
1457
|
+
yield await readOfflineUploadContent({
|
|
1458
|
+
root: options.root,
|
|
1459
|
+
relPath: options.upload.relPath,
|
|
1460
|
+
filePath: options.upload.filePath,
|
|
1461
|
+
readFile: options.readFile,
|
|
1462
|
+
});
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
const entries = await readdir(options.upload.filePath);
|
|
1467
|
+
const chunkNames = entries
|
|
1468
|
+
.filter((entry) => /^\d{20}\.part$/.test(entry))
|
|
1469
|
+
.sort();
|
|
1470
|
+
if (chunkNames.length === 0) {
|
|
1471
|
+
throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
|
|
1472
|
+
}
|
|
1473
|
+
let expectedOffset = 0;
|
|
1474
|
+
for (const chunkName of chunkNames) {
|
|
1475
|
+
const offset = Number(chunkName.slice(0, 20));
|
|
1476
|
+
if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
|
|
1477
|
+
throw new Error(
|
|
1478
|
+
`offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`,
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
const relPath = `${options.upload.relPath}/${chunkName}`;
|
|
1482
|
+
const filePath = await resolveSafeArchiveTarget(options.root, relPath);
|
|
1483
|
+
const content = await readOfflineUploadContent({
|
|
1484
|
+
root: options.root,
|
|
1485
|
+
relPath,
|
|
1486
|
+
filePath,
|
|
1487
|
+
readFile: options.readFile,
|
|
1488
|
+
});
|
|
1489
|
+
expectedOffset += content.length;
|
|
1490
|
+
yield content;
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
async function digestOfflineUploadStagingContent(options: {
|
|
1495
|
+
root: SafeArchiveRoot;
|
|
1496
|
+
upload: OfflineUploadStaging;
|
|
1497
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1498
|
+
}): Promise<{ sha256: string; bytes: number }> {
|
|
1499
|
+
const hash = createHash("sha256");
|
|
1500
|
+
let bytes = 0;
|
|
1501
|
+
for await (const chunk of readOfflineUploadStagingChunks(options)) {
|
|
1502
|
+
hash.update(chunk);
|
|
1503
|
+
bytes += chunk.length;
|
|
1504
|
+
}
|
|
1505
|
+
return { sha256: hash.digest("hex"), bytes };
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
async function writeSafeFileFromUpload(
|
|
1509
|
+
root: SafeArchiveRoot,
|
|
1510
|
+
relPath: string,
|
|
1511
|
+
upload: OfflineUploadStaging,
|
|
1512
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
|
|
1513
|
+
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
|
|
1514
|
+
): Promise<void> {
|
|
1515
|
+
const target = await resolveSafeArchiveTarget(root, relPath);
|
|
1516
|
+
const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
|
|
1517
|
+
if (writeFileChunks) {
|
|
1518
|
+
await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
1523
|
+
const tmp = path.join(
|
|
1524
|
+
path.dirname(target),
|
|
1525
|
+
`.remnic-sync.${process.pid}.${randomUUID()}.tmp`,
|
|
1526
|
+
);
|
|
1527
|
+
const handle = await open(tmp, "w", 0o600);
|
|
1528
|
+
try {
|
|
1529
|
+
for await (const chunk of chunks) {
|
|
1530
|
+
if (chunk.length > 0) await handle.write(chunk);
|
|
1531
|
+
}
|
|
1532
|
+
await handle.close();
|
|
1533
|
+
const targetStat = await lstat(target).catch((error: unknown) => {
|
|
1534
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1535
|
+
throw error;
|
|
1536
|
+
});
|
|
1537
|
+
if (targetStat?.isSymbolicLink()) {
|
|
1538
|
+
throw new Error(`offline sync target is a symlink: ${relPath}`);
|
|
1539
|
+
}
|
|
1540
|
+
await rename(tmp, target);
|
|
1541
|
+
} catch (error) {
|
|
1542
|
+
await handle.close().catch(() => {});
|
|
1543
|
+
await unlink(tmp).catch(() => {});
|
|
1544
|
+
throw error;
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
async function cleanupOfflineUpload(upload: OfflineUploadStaging): Promise<void> {
|
|
1549
|
+
if (upload.kind === "chunks") {
|
|
1550
|
+
await rm(upload.filePath, { recursive: true, force: true });
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
await unlink(upload.filePath).catch((error: unknown) => {
|
|
1554
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
1555
|
+
throw error;
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
async function readOfflineUploadContent(options: {
|
|
1560
|
+
root: SafeArchiveRoot;
|
|
1561
|
+
relPath: string;
|
|
1562
|
+
filePath: string;
|
|
1563
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1564
|
+
}): Promise<Buffer> {
|
|
1565
|
+
if (options.readFile) {
|
|
1566
|
+
return options.readFile({
|
|
1567
|
+
root: options.root.abs,
|
|
1568
|
+
path: options.relPath,
|
|
1569
|
+
filePath: options.filePath,
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
return readFile(options.filePath);
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
async function writeOfflineUploadContent(options: {
|
|
1576
|
+
root: SafeArchiveRoot;
|
|
1577
|
+
relPath: string;
|
|
1578
|
+
filePath: string;
|
|
1579
|
+
content: Buffer;
|
|
1580
|
+
writeFile: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1581
|
+
}): Promise<void> {
|
|
1582
|
+
await options.writeFile({
|
|
1583
|
+
root: options.root.abs,
|
|
1584
|
+
path: options.relPath,
|
|
1585
|
+
filePath: options.filePath,
|
|
1586
|
+
content: options.content,
|
|
1587
|
+
});
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1151
1590
|
async function deleteSafeFile(
|
|
1152
1591
|
root: SafeArchiveRoot,
|
|
1153
1592
|
relPath: string,
|
|
@@ -43,10 +43,21 @@
|
|
|
43
43
|
* Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
|
|
44
44
|
*/
|
|
45
45
|
|
|
46
|
-
import {
|
|
46
|
+
import { createCipheriv, randomBytes } from "node:crypto";
|
|
47
|
+
import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
47
48
|
import path from "node:path";
|
|
48
49
|
|
|
49
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
AUTH_TAG_LENGTH,
|
|
52
|
+
ENVELOPE_HEADER_SIZE,
|
|
53
|
+
ENVELOPE_LAYOUT,
|
|
54
|
+
ENVELOPE_SALT_LENGTH,
|
|
55
|
+
ENVELOPE_VERSION,
|
|
56
|
+
IV_LENGTH,
|
|
57
|
+
generateSalt,
|
|
58
|
+
open as openEnvelope,
|
|
59
|
+
seal,
|
|
60
|
+
} from "./cipher.js";
|
|
50
61
|
|
|
51
62
|
// ---------------------------------------------------------------------------
|
|
52
63
|
// Error classes
|
|
@@ -157,7 +168,7 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
|
|
|
157
168
|
}
|
|
158
169
|
const envelope = buf.subarray(MAGIC_HEADER_SIZE);
|
|
159
170
|
try {
|
|
160
|
-
return
|
|
171
|
+
return openEnvelope(key, envelope, aad ? { aad } : {});
|
|
161
172
|
} catch (err) {
|
|
162
173
|
const msg = err instanceof Error ? err.message : String(err);
|
|
163
174
|
throw new SecureStoreDecryptError(
|
|
@@ -166,6 +177,13 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
|
|
|
166
177
|
}
|
|
167
178
|
}
|
|
168
179
|
|
|
180
|
+
function buildHeaderAad(salt: Uint8Array): Buffer {
|
|
181
|
+
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
182
|
+
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
183
|
+
Buffer.from(salt).copy(out, 1);
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
|
|
169
187
|
// ---------------------------------------------------------------------------
|
|
170
188
|
// Path → AAD helper
|
|
171
189
|
// ---------------------------------------------------------------------------
|
|
@@ -293,6 +311,69 @@ export async function writeMaybeEncryptedFile(
|
|
|
293
311
|
}
|
|
294
312
|
}
|
|
295
313
|
|
|
314
|
+
export async function writeMaybeEncryptedFileFromChunks(
|
|
315
|
+
filePath: string,
|
|
316
|
+
chunks: AsyncIterable<Buffer>,
|
|
317
|
+
key: Buffer | null,
|
|
318
|
+
options: WriteMaybeEncryptedFileOptions = {},
|
|
319
|
+
memoryDir?: string,
|
|
320
|
+
): Promise<void> {
|
|
321
|
+
const { mode = 0o600, atomic = true } = options;
|
|
322
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
323
|
+
const writePath = atomic ? `${filePath}.tmp-${process.pid}-${Date.now()}` : filePath;
|
|
324
|
+
let completed = false;
|
|
325
|
+
try {
|
|
326
|
+
const handle = await openFile(writePath, "w", mode);
|
|
327
|
+
try {
|
|
328
|
+
if (key !== null) {
|
|
329
|
+
const salt = generateSalt();
|
|
330
|
+
const iv = randomBytes(IV_LENGTH);
|
|
331
|
+
const header = Buffer.alloc(MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
332
|
+
MAGIC_BYTES.copy(header, 0);
|
|
333
|
+
header.writeUInt8(FILE_FORMAT_VERSION, MAGIC_BYTES.length);
|
|
334
|
+
header.writeUInt8(FILE_FORMAT_FLAGS, MAGIC_BYTES.length + 1);
|
|
335
|
+
const envelopeOffset = MAGIC_HEADER_SIZE;
|
|
336
|
+
header.writeUInt8(ENVELOPE_VERSION, envelopeOffset + ENVELOPE_LAYOUT.version);
|
|
337
|
+
salt.copy(header, envelopeOffset + ENVELOPE_LAYOUT.salt);
|
|
338
|
+
iv.copy(header, envelopeOffset + ENVELOPE_LAYOUT.iv);
|
|
339
|
+
await handle.write(header);
|
|
340
|
+
|
|
341
|
+
const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
|
|
342
|
+
const aad = filePathAad(filePath, memoryDir);
|
|
343
|
+
cipher.setAAD(Buffer.concat([buildHeaderAad(salt), aad]));
|
|
344
|
+
for await (const chunk of chunks) {
|
|
345
|
+
if (chunk.length === 0) continue;
|
|
346
|
+
const encrypted = cipher.update(chunk);
|
|
347
|
+
if (encrypted.length > 0) await handle.write(encrypted);
|
|
348
|
+
}
|
|
349
|
+
const final = cipher.final();
|
|
350
|
+
if (final.length > 0) await handle.write(final);
|
|
351
|
+
const authTag = cipher.getAuthTag();
|
|
352
|
+
await handle.write(
|
|
353
|
+
authTag,
|
|
354
|
+
0,
|
|
355
|
+
authTag.length,
|
|
356
|
+
MAGIC_HEADER_SIZE + ENVELOPE_LAYOUT.authTag,
|
|
357
|
+
);
|
|
358
|
+
} else {
|
|
359
|
+
for await (const chunk of chunks) {
|
|
360
|
+
if (chunk.length > 0) await handle.write(chunk);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
} finally {
|
|
364
|
+
await handle.close();
|
|
365
|
+
}
|
|
366
|
+
if (atomic) {
|
|
367
|
+
await rename(writePath, filePath);
|
|
368
|
+
}
|
|
369
|
+
completed = true;
|
|
370
|
+
} finally {
|
|
371
|
+
if (!completed && atomic) {
|
|
372
|
+
await unlink(writePath).catch(() => {});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
296
377
|
// ---------------------------------------------------------------------------
|
|
297
378
|
// Migration
|
|
298
379
|
// ---------------------------------------------------------------------------
|