@remnic/core 1.1.21 → 1.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +15 -15
- package/dist/access-http.d.ts +9 -1
- package/dist/access-http.js +9 -9
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-mcp.js +8 -8
- package/dist/access-schema.js +3 -3
- package/dist/{access-service-DT9L2DW4.d.ts → access-service-CEyV8XJ5.d.ts} +19 -2
- package/dist/access-service.d.ts +1 -1
- package/dist/access-service.js +6 -6
- package/dist/briefing.js +3 -3
- package/dist/causal-consolidation.js +4 -4
- package/dist/{chunk-YO3AZEE5.js → chunk-25YQM6XW.js} +3 -3
- package/dist/{chunk-TLM762GT.js → chunk-2WIPXV3Y.js} +2 -2
- package/dist/{chunk-QOHBYVZG.js → chunk-3F24QTRI.js} +2 -2
- package/dist/{chunk-5IQC4OG6.js → chunk-4H6DURG6.js} +2 -2
- package/dist/{chunk-NOQ74SJN.js → chunk-7D6O46PF.js} +2 -2
- package/dist/{chunk-SLKSC522.js → chunk-7E7SZRPP.js} +2 -2
- package/dist/{chunk-7Q2P774N.js → chunk-F33CJ5CH.js} +13 -3
- package/dist/chunk-F33CJ5CH.js.map +1 -0
- package/dist/{chunk-APW7AQOJ.js → chunk-FHXVW3L4.js} +4 -4
- package/dist/{chunk-PFFKUJM2.js → chunk-HWF42K6J.js} +103 -4
- package/dist/chunk-HWF42K6J.js.map +1 -0
- package/dist/{chunk-FSODDMR2.js → chunk-IANK6Y5W.js} +2 -2
- package/dist/{chunk-BYYIIXIJ.js → chunk-JKXFF3NT.js} +361 -29
- package/dist/chunk-JKXFF3NT.js.map +1 -0
- package/dist/{chunk-P7F6DJPA.js → chunk-MM5EBZVW.js} +42 -5
- package/dist/chunk-MM5EBZVW.js.map +1 -0
- package/dist/{chunk-GGCJ253V.js → chunk-MVAOT247.js} +8 -8
- package/dist/{chunk-SH5S7XYD.js → chunk-MXFBBHJU.js} +72 -2
- package/dist/chunk-MXFBBHJU.js.map +1 -0
- package/dist/{chunk-SZKCBLS5.js → chunk-PUXCIHRL.js} +2 -2
- package/dist/{chunk-2IRT26RZ.js → chunk-QYHQ2JHL.js} +2 -2
- package/dist/{chunk-73DAPA62.js → chunk-RA73CTVY.js} +12 -12
- package/dist/{chunk-CN4P6SVA.js → chunk-RCZRL5BE.js} +2 -2
- package/dist/{chunk-SGIXDVSF.js → chunk-S27EXIHY.js} +2 -2
- package/dist/{chunk-5ML4TH3E.js → chunk-TFORLO3O.js} +4 -4
- package/dist/{chunk-TOFUTKQN.js → chunk-TR4DK5OH.js} +2 -2
- package/dist/{chunk-6ORWKANA.js → chunk-VYU7PXUS.js} +2 -2
- package/dist/{chunk-FFU4GMST.js → chunk-WNARATI3.js} +2 -2
- package/dist/{chunk-KSFBM6TV.js → chunk-YITUHONZ.js} +2 -2
- package/dist/{cli-BN0CkYzI.d.ts → cli-BguVmIwO.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +18 -18
- package/dist/compounding/engine.js +3 -3
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/index.js +3 -3
- package/dist/entity-retrieval.js +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.js +26 -24
- package/dist/index.js.map +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/storage.js +3 -3
- package/dist/offline-sync.d.ts +36 -1
- package/dist/offline-sync.js +4 -2
- package/dist/operator-toolkit.js +6 -6
- package/dist/orchestrator.js +11 -11
- package/dist/schemas.d.ts +22 -22
- package/dist/secure-store/index.js +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/storage.d.ts +2 -0
- package/dist/storage.js +2 -2
- package/dist/transfer/types.d.ts +12 -12
- package/dist/verified-recall.js +3 -3
- package/package.json +1 -1
- package/src/access-http.test.ts +176 -0
- package/src/access-http.ts +116 -0
- package/src/access-service-offline-file-content.test.ts +37 -0
- package/src/access-service.ts +70 -0
- package/src/index.ts +2 -0
- package/src/offline-sync.test.ts +448 -64
- package/src/offline-sync.ts +477 -29
- package/src/secure-store/secure-fs.ts +84 -3
- package/src/storage.ts +12 -0
- package/dist/chunk-7Q2P774N.js.map +0 -1
- package/dist/chunk-BYYIIXIJ.js.map +0 -1
- package/dist/chunk-P7F6DJPA.js.map +0 -1
- package/dist/chunk-PFFKUJM2.js.map +0 -1
- package/dist/chunk-SH5S7XYD.js.map +0 -1
- /package/dist/{chunk-YO3AZEE5.js.map → chunk-25YQM6XW.js.map} +0 -0
- /package/dist/{chunk-TLM762GT.js.map → chunk-2WIPXV3Y.js.map} +0 -0
- /package/dist/{chunk-QOHBYVZG.js.map → chunk-3F24QTRI.js.map} +0 -0
- /package/dist/{chunk-5IQC4OG6.js.map → chunk-4H6DURG6.js.map} +0 -0
- /package/dist/{chunk-NOQ74SJN.js.map → chunk-7D6O46PF.js.map} +0 -0
- /package/dist/{chunk-SLKSC522.js.map → chunk-7E7SZRPP.js.map} +0 -0
- /package/dist/{chunk-APW7AQOJ.js.map → chunk-FHXVW3L4.js.map} +0 -0
- /package/dist/{chunk-FSODDMR2.js.map → chunk-IANK6Y5W.js.map} +0 -0
- /package/dist/{chunk-GGCJ253V.js.map → chunk-MVAOT247.js.map} +0 -0
- /package/dist/{chunk-SZKCBLS5.js.map → chunk-PUXCIHRL.js.map} +0 -0
- /package/dist/{chunk-2IRT26RZ.js.map → chunk-QYHQ2JHL.js.map} +0 -0
- /package/dist/{chunk-73DAPA62.js.map → chunk-RA73CTVY.js.map} +0 -0
- /package/dist/{chunk-CN4P6SVA.js.map → chunk-RCZRL5BE.js.map} +0 -0
- /package/dist/{chunk-SGIXDVSF.js.map → chunk-S27EXIHY.js.map} +0 -0
- /package/dist/{chunk-5ML4TH3E.js.map → chunk-TFORLO3O.js.map} +0 -0
- /package/dist/{chunk-TOFUTKQN.js.map → chunk-TR4DK5OH.js.map} +0 -0
- /package/dist/{chunk-6ORWKANA.js.map → chunk-VYU7PXUS.js.map} +0 -0
- /package/dist/{chunk-FFU4GMST.js.map → chunk-WNARATI3.js.map} +0 -0
- /package/dist/{chunk-KSFBM6TV.js.map → chunk-YITUHONZ.js.map} +0 -0
package/src/offline-sync.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
readdir,
|
|
7
7
|
readFile,
|
|
8
8
|
rename,
|
|
9
|
+
rm,
|
|
9
10
|
stat,
|
|
10
11
|
unlink,
|
|
11
12
|
writeFile,
|
|
@@ -130,6 +131,12 @@ export interface OfflineSyncFileWriteTarget extends OfflineSyncFileTarget {
|
|
|
130
131
|
content: Buffer;
|
|
131
132
|
}
|
|
132
133
|
|
|
134
|
+
export interface OfflineSyncFileWriteChunksTarget extends OfflineSyncFileTarget {
|
|
135
|
+
chunks: AsyncIterable<Buffer>;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface OfflineSyncFileStagingWriteTarget extends OfflineSyncFileWriteTarget {}
|
|
139
|
+
|
|
133
140
|
export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState, "sha256"> {
|
|
134
141
|
sha256?: string;
|
|
135
142
|
offset: number;
|
|
@@ -137,6 +144,26 @@ export interface OfflineSyncFileContentChunk extends Omit<OfflineSyncFileState,
|
|
|
137
144
|
content: Buffer;
|
|
138
145
|
}
|
|
139
146
|
|
|
147
|
+
export interface OfflineSyncApplyFileContentChunkResult {
|
|
148
|
+
path: string;
|
|
149
|
+
sha256: string;
|
|
150
|
+
bytes: number;
|
|
151
|
+
mtimeMs: number;
|
|
152
|
+
offset: number;
|
|
153
|
+
chunkBytes: number;
|
|
154
|
+
done: boolean;
|
|
155
|
+
applied: boolean;
|
|
156
|
+
skipped: boolean;
|
|
157
|
+
conflict?: OfflineSyncConflict;
|
|
158
|
+
currentFile?: OfflineSyncFileState;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
interface OfflineUploadStaging {
|
|
162
|
+
kind: "single" | "chunks";
|
|
163
|
+
relPath: string;
|
|
164
|
+
filePath: string;
|
|
165
|
+
}
|
|
166
|
+
|
|
140
167
|
interface OfflineSyncFileRecordOptions {
|
|
141
168
|
root: SafeArchiveRoot;
|
|
142
169
|
relPath: string;
|
|
@@ -146,36 +173,11 @@ interface OfflineSyncFileRecordOptions {
|
|
|
146
173
|
}
|
|
147
174
|
|
|
148
175
|
const SYNC_INTERNAL_DIR = ".offline-sync";
|
|
176
|
+
const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
149
177
|
const EXCLUDED_FILE_NAMES = new Set([
|
|
150
178
|
".sync-state.json",
|
|
151
179
|
]);
|
|
152
180
|
|
|
153
|
-
const DERIVED_RUNTIME_REL_PATHS = new Set([
|
|
154
|
-
"state/fact-hashes.ready",
|
|
155
|
-
"state/fact-hashes.txt",
|
|
156
|
-
"state/buffer-surprise-ledger.jsonl",
|
|
157
|
-
"state/buffer.json",
|
|
158
|
-
"state/embeddings.json",
|
|
159
|
-
"state/index_tags.json",
|
|
160
|
-
"state/index_time.json",
|
|
161
|
-
"state/last_graph_recall.json",
|
|
162
|
-
"state/last_intent.json",
|
|
163
|
-
"state/last_qmd_recall.json",
|
|
164
|
-
"state/last_recall.json",
|
|
165
|
-
"state/lcm.sqlite",
|
|
166
|
-
"state/lcm.sqlite-shm",
|
|
167
|
-
"state/lcm.sqlite-wal",
|
|
168
|
-
"state/memory-lifecycle-ledger.jsonl",
|
|
169
|
-
"state/memory-projection.sqlite",
|
|
170
|
-
"state/memory-projection.sqlite-shm",
|
|
171
|
-
"state/memory-projection.sqlite-wal",
|
|
172
|
-
"state/recall_impressions.jsonl",
|
|
173
|
-
]);
|
|
174
|
-
|
|
175
|
-
const EXCLUDED_REL_PATHS = new Set([
|
|
176
|
-
...DERIVED_RUNTIME_REL_PATHS,
|
|
177
|
-
]);
|
|
178
|
-
|
|
179
181
|
const EXCLUDED_FILE_PREFIXES = [
|
|
180
182
|
".remnic-sync.",
|
|
181
183
|
".remnic-sync-state.",
|
|
@@ -432,10 +434,9 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
|
|
|
432
434
|
const parts = relPosix.split("/");
|
|
433
435
|
if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
|
|
434
436
|
if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
|
|
435
|
-
if (EXCLUDED_REL_PATHS.has(relPosix)) return true;
|
|
436
437
|
if (!includeTranscripts && parts[0] === "transcripts") return true;
|
|
437
438
|
const basename = parts[parts.length - 1] ?? "";
|
|
438
|
-
if (parts
|
|
439
|
+
if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
|
|
439
440
|
if (EXCLUDED_FILE_NAMES.has(basename)) return true;
|
|
440
441
|
return EXCLUDED_FILE_PREFIXES.some((prefix) => basename.startsWith(prefix));
|
|
441
442
|
}
|
|
@@ -443,7 +444,12 @@ function shouldExcludeRelPath(relPosix: string, includeTranscripts: boolean): bo
|
|
|
443
444
|
function shouldIgnoreIncomingRuntimePath(relPosix: string): boolean {
|
|
444
445
|
const parts = relPosix.split("/");
|
|
445
446
|
const basename = parts[parts.length - 1] ?? "";
|
|
446
|
-
return
|
|
447
|
+
return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function isCanonicalRuntimeStatePath(parts: string[]): boolean {
|
|
451
|
+
if (parts[0] === "state") return true;
|
|
452
|
+
return parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state";
|
|
447
453
|
}
|
|
448
454
|
|
|
449
455
|
function filterBaseFilesForMode(
|
|
@@ -674,11 +680,15 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
674
680
|
root: string;
|
|
675
681
|
sourceId: string;
|
|
676
682
|
baseFiles?: readonly OfflineSyncFileState[];
|
|
683
|
+
excludePaths?: readonly string[];
|
|
677
684
|
includeTranscripts?: boolean;
|
|
678
685
|
now?: Date;
|
|
679
686
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
680
687
|
}): Promise<OfflineSyncChangeset> {
|
|
681
688
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
689
|
+
const excludedPaths = new Set(
|
|
690
|
+
(options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]")),
|
|
691
|
+
);
|
|
682
692
|
const base = byPath(filterBaseFilesForMode(
|
|
683
693
|
normalizeFileStates(options.baseFiles),
|
|
684
694
|
includeTranscripts,
|
|
@@ -695,6 +705,7 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
695
705
|
const changes: OfflineSyncChange[] = [];
|
|
696
706
|
|
|
697
707
|
for (const relPath of unionPaths(base, currentMap)) {
|
|
708
|
+
if (excludedPaths.has(relPath)) continue;
|
|
698
709
|
const baseEntry = base.get(relPath);
|
|
699
710
|
const currentEntry = currentMap.get(relPath);
|
|
700
711
|
if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
|
|
@@ -1139,6 +1150,443 @@ async function writeSafeFile(
|
|
|
1139
1150
|
}
|
|
1140
1151
|
}
|
|
1141
1152
|
|
|
1153
|
+
export async function applyOfflineSyncFileContentChunk(options: {
|
|
1154
|
+
root: string;
|
|
1155
|
+
sourceId: string;
|
|
1156
|
+
path: string;
|
|
1157
|
+
sha256: string;
|
|
1158
|
+
bytes: number;
|
|
1159
|
+
mtimeMs: number;
|
|
1160
|
+
offset?: number;
|
|
1161
|
+
content: Buffer;
|
|
1162
|
+
baseSha256?: string;
|
|
1163
|
+
includeTranscripts?: boolean;
|
|
1164
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1165
|
+
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1166
|
+
writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
|
|
1167
|
+
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
|
|
1168
|
+
}): Promise<OfflineSyncApplyFileContentChunkResult> {
|
|
1169
|
+
const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
|
|
1170
|
+
const sourceId = normalizeSourceId(options.sourceId, "sourceId");
|
|
1171
|
+
const relPath = normalizeRelativePath(options.path, "path");
|
|
1172
|
+
const includeTranscripts = options.includeTranscripts !== false;
|
|
1173
|
+
if (shouldExcludeRelPath(relPath, includeTranscripts)) {
|
|
1174
|
+
throw new Error(`offline sync file content path is excluded: ${relPath}`);
|
|
1175
|
+
}
|
|
1176
|
+
const sha256 = assertSha256(options.sha256, "sha256");
|
|
1177
|
+
const bytes = assertNonNegativeInteger(options.bytes, "bytes");
|
|
1178
|
+
const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
|
|
1179
|
+
const offset = options.offset === undefined
|
|
1180
|
+
? 0
|
|
1181
|
+
: assertNonNegativeInteger(options.offset, "offset");
|
|
1182
|
+
const baseSha256 = options.baseSha256 === undefined
|
|
1183
|
+
? undefined
|
|
1184
|
+
: assertSha256(options.baseSha256, "baseSha256");
|
|
1185
|
+
if (!Buffer.isBuffer(options.content)) {
|
|
1186
|
+
throw new Error("content must be a Buffer");
|
|
1187
|
+
}
|
|
1188
|
+
if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
|
|
1189
|
+
throw new Error(
|
|
1190
|
+
`content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`,
|
|
1191
|
+
);
|
|
1192
|
+
}
|
|
1193
|
+
if (bytes > 0 && options.content.length === 0) {
|
|
1194
|
+
throw new Error("content chunk must be non-empty before EOF");
|
|
1195
|
+
}
|
|
1196
|
+
if (offset > bytes || offset + options.content.length > bytes) {
|
|
1197
|
+
throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
|
|
1198
|
+
}
|
|
1199
|
+
if (options.writeFile && !options.writeFileChunks) {
|
|
1200
|
+
throw new Error("offline sync upload storage hooks require writeFileChunks");
|
|
1201
|
+
}
|
|
1202
|
+
if (options.writeFile && !options.writeStagingFile) {
|
|
1203
|
+
throw new Error("offline sync upload storage hooks require writeStagingFile");
|
|
1204
|
+
}
|
|
1205
|
+
if (offset === 0) {
|
|
1206
|
+
await pruneOfflineUploadStaging(root);
|
|
1207
|
+
}
|
|
1208
|
+
|
|
1209
|
+
const upload = await writeOfflineUploadChunk({
|
|
1210
|
+
root,
|
|
1211
|
+
sourceId,
|
|
1212
|
+
relPath,
|
|
1213
|
+
sha256,
|
|
1214
|
+
bytes,
|
|
1215
|
+
offset,
|
|
1216
|
+
content: options.content,
|
|
1217
|
+
readFile: options.readFile,
|
|
1218
|
+
writeFile: options.writeFile,
|
|
1219
|
+
writeStagingFile: options.writeStagingFile,
|
|
1220
|
+
});
|
|
1221
|
+
const done = offset + options.content.length === bytes;
|
|
1222
|
+
const baseResult = {
|
|
1223
|
+
path: relPath,
|
|
1224
|
+
sha256,
|
|
1225
|
+
bytes,
|
|
1226
|
+
mtimeMs,
|
|
1227
|
+
offset,
|
|
1228
|
+
chunkBytes: options.content.length,
|
|
1229
|
+
done,
|
|
1230
|
+
};
|
|
1231
|
+
if (!done) {
|
|
1232
|
+
return {
|
|
1233
|
+
...baseResult,
|
|
1234
|
+
applied: false,
|
|
1235
|
+
skipped: false,
|
|
1236
|
+
};
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
const digest = await digestOfflineUploadStagingContent({
|
|
1240
|
+
root,
|
|
1241
|
+
upload,
|
|
1242
|
+
readFile: options.readFile,
|
|
1243
|
+
});
|
|
1244
|
+
if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
|
|
1245
|
+
await cleanupOfflineUpload(upload).catch(() => {});
|
|
1246
|
+
throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
|
|
1250
|
+
root: root.abs,
|
|
1251
|
+
sourceId: "local",
|
|
1252
|
+
paths: [relPath],
|
|
1253
|
+
includeContent: false,
|
|
1254
|
+
includeTranscripts,
|
|
1255
|
+
readFile: options.readFile,
|
|
1256
|
+
});
|
|
1257
|
+
const currentFile = currentSnapshot.files[0];
|
|
1258
|
+
const uploadedState: OfflineSyncFileState = {
|
|
1259
|
+
path: relPath,
|
|
1260
|
+
sha256,
|
|
1261
|
+
bytes,
|
|
1262
|
+
mtimeMs,
|
|
1263
|
+
};
|
|
1264
|
+
|
|
1265
|
+
try {
|
|
1266
|
+
if (currentFile?.sha256 === sha256) {
|
|
1267
|
+
return {
|
|
1268
|
+
...baseResult,
|
|
1269
|
+
applied: false,
|
|
1270
|
+
skipped: true,
|
|
1271
|
+
currentFile: toFileState(currentFile),
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1274
|
+
if (!baseSha256 && currentFile) {
|
|
1275
|
+
const conflict = await recordConflict({
|
|
1276
|
+
root,
|
|
1277
|
+
relPath,
|
|
1278
|
+
reason: "remote_exists_for_local_create",
|
|
1279
|
+
localSha256: currentFile.sha256,
|
|
1280
|
+
incomingSha256: sha256,
|
|
1281
|
+
writeConflictCopies: false,
|
|
1282
|
+
sourceId,
|
|
1283
|
+
writeFile: options.writeFile,
|
|
1284
|
+
});
|
|
1285
|
+
return {
|
|
1286
|
+
...baseResult,
|
|
1287
|
+
applied: false,
|
|
1288
|
+
skipped: false,
|
|
1289
|
+
conflict,
|
|
1290
|
+
currentFile: toFileState(currentFile),
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
if (baseSha256 && currentFile?.sha256 !== baseSha256) {
|
|
1294
|
+
const conflict = await recordConflict({
|
|
1295
|
+
root,
|
|
1296
|
+
relPath,
|
|
1297
|
+
reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
|
|
1298
|
+
baseSha256,
|
|
1299
|
+
localSha256: currentFile?.sha256,
|
|
1300
|
+
incomingSha256: sha256,
|
|
1301
|
+
writeConflictCopies: false,
|
|
1302
|
+
sourceId,
|
|
1303
|
+
writeFile: options.writeFile,
|
|
1304
|
+
});
|
|
1305
|
+
return {
|
|
1306
|
+
...baseResult,
|
|
1307
|
+
applied: false,
|
|
1308
|
+
skipped: false,
|
|
1309
|
+
conflict,
|
|
1310
|
+
...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
|
|
1315
|
+
return {
|
|
1316
|
+
...baseResult,
|
|
1317
|
+
applied: true,
|
|
1318
|
+
skipped: false,
|
|
1319
|
+
currentFile: uploadedState,
|
|
1320
|
+
};
|
|
1321
|
+
} finally {
|
|
1322
|
+
await cleanupOfflineUpload(upload).catch(() => {});
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function offlineUploadRelPath(options: {
|
|
1327
|
+
sourceId: string;
|
|
1328
|
+
relPath: string;
|
|
1329
|
+
sha256: string;
|
|
1330
|
+
bytes: number;
|
|
1331
|
+
}): string {
|
|
1332
|
+
const key = hashText([
|
|
1333
|
+
options.sourceId,
|
|
1334
|
+
options.relPath,
|
|
1335
|
+
options.sha256,
|
|
1336
|
+
String(options.bytes),
|
|
1337
|
+
].join("\0"));
|
|
1338
|
+
return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
async function offlineUploadPath(root: SafeArchiveRoot, options: {
|
|
1342
|
+
sourceId: string;
|
|
1343
|
+
relPath: string;
|
|
1344
|
+
sha256: string;
|
|
1345
|
+
bytes: number;
|
|
1346
|
+
}): Promise<OfflineUploadStaging> {
|
|
1347
|
+
const relPath = offlineUploadRelPath(options);
|
|
1348
|
+
return {
|
|
1349
|
+
kind: "single",
|
|
1350
|
+
relPath,
|
|
1351
|
+
filePath: await resolveSafeArchiveTarget(root, relPath),
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
async function offlineUploadChunkPath(root: SafeArchiveRoot, options: {
|
|
1356
|
+
sourceId: string;
|
|
1357
|
+
relPath: string;
|
|
1358
|
+
sha256: string;
|
|
1359
|
+
bytes: number;
|
|
1360
|
+
offset: number;
|
|
1361
|
+
}): Promise<OfflineUploadStaging> {
|
|
1362
|
+
const uploadRelPath = offlineUploadRelPath(options);
|
|
1363
|
+
const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
|
|
1364
|
+
return {
|
|
1365
|
+
kind: "chunks",
|
|
1366
|
+
relPath,
|
|
1367
|
+
filePath: await resolveSafeArchiveTarget(root, relPath),
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
async function writeOfflineUploadChunk(options: {
|
|
1372
|
+
root: SafeArchiveRoot;
|
|
1373
|
+
sourceId: string;
|
|
1374
|
+
relPath: string;
|
|
1375
|
+
sha256: string;
|
|
1376
|
+
bytes: number;
|
|
1377
|
+
offset: number;
|
|
1378
|
+
content: Buffer;
|
|
1379
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1380
|
+
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1381
|
+
writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
|
|
1382
|
+
}): Promise<OfflineUploadStaging> {
|
|
1383
|
+
if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
|
|
1384
|
+
throw new Error("offline sync upload chunk storage hooks require readFile");
|
|
1385
|
+
}
|
|
1386
|
+
const uploadRoot = {
|
|
1387
|
+
...(await offlineUploadPath(options.root, options)),
|
|
1388
|
+
kind: "chunks" as const,
|
|
1389
|
+
};
|
|
1390
|
+
if (options.offset === 0) {
|
|
1391
|
+
await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {});
|
|
1392
|
+
} else {
|
|
1393
|
+
const existing = await stat(uploadRoot.filePath).catch((error: unknown) => {
|
|
1394
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1395
|
+
throw error;
|
|
1396
|
+
});
|
|
1397
|
+
if (!existing || !existing.isDirectory()) {
|
|
1398
|
+
throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
|
|
1402
|
+
|
|
1403
|
+
const writeStagingFile = options.writeStagingFile ?? options.writeFile;
|
|
1404
|
+
if (writeStagingFile) {
|
|
1405
|
+
// Storage-backed services provide these hooks so secure-store deployments
|
|
1406
|
+
// keep staged partial uploads encrypted at rest without mutating indexes.
|
|
1407
|
+
await writeOfflineUploadContent({
|
|
1408
|
+
root: options.root,
|
|
1409
|
+
relPath: chunk.relPath,
|
|
1410
|
+
filePath: chunk.filePath,
|
|
1411
|
+
content: options.content,
|
|
1412
|
+
writeFile: writeStagingFile,
|
|
1413
|
+
});
|
|
1414
|
+
return uploadRoot;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
await mkdir(path.dirname(chunk.filePath), { recursive: true });
|
|
1418
|
+
const existingChunk = await lstat(chunk.filePath).catch((error: unknown) => {
|
|
1419
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1420
|
+
throw error;
|
|
1421
|
+
});
|
|
1422
|
+
if (existingChunk?.isSymbolicLink()) {
|
|
1423
|
+
throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
|
|
1424
|
+
}
|
|
1425
|
+
await writeFile(chunk.filePath, options.content, { mode: 0o600 });
|
|
1426
|
+
return uploadRoot;
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
async function pruneOfflineUploadStaging(root: SafeArchiveRoot): Promise<void> {
|
|
1430
|
+
const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
|
|
1431
|
+
const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
|
|
1432
|
+
const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error: unknown) => {
|
|
1433
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
1434
|
+
throw error;
|
|
1435
|
+
});
|
|
1436
|
+
const now = Date.now();
|
|
1437
|
+
await Promise.all(entries.map(async (entry) => {
|
|
1438
|
+
if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
|
|
1439
|
+
const relPath = `${uploadsRelPath}/${entry.name}`;
|
|
1440
|
+
const filePath = await resolveSafeArchiveTarget(root, relPath);
|
|
1441
|
+
const info = await lstat(filePath).catch((error: unknown) => {
|
|
1442
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1443
|
+
throw error;
|
|
1444
|
+
});
|
|
1445
|
+
if (!info) return;
|
|
1446
|
+
if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
|
|
1447
|
+
await rm(filePath, { recursive: true, force: true });
|
|
1448
|
+
}));
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
async function* readOfflineUploadStagingChunks(options: {
|
|
1452
|
+
root: SafeArchiveRoot;
|
|
1453
|
+
upload: OfflineUploadStaging;
|
|
1454
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1455
|
+
}): AsyncGenerator<Buffer> {
|
|
1456
|
+
if (options.upload.kind === "single") {
|
|
1457
|
+
yield await readOfflineUploadContent({
|
|
1458
|
+
root: options.root,
|
|
1459
|
+
relPath: options.upload.relPath,
|
|
1460
|
+
filePath: options.upload.filePath,
|
|
1461
|
+
readFile: options.readFile,
|
|
1462
|
+
});
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
const entries = await readdir(options.upload.filePath);
|
|
1467
|
+
const chunkNames = entries
|
|
1468
|
+
.filter((entry) => /^\d{20}\.part$/.test(entry))
|
|
1469
|
+
.sort();
|
|
1470
|
+
if (chunkNames.length === 0) {
|
|
1471
|
+
throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
|
|
1472
|
+
}
|
|
1473
|
+
let expectedOffset = 0;
|
|
1474
|
+
for (const chunkName of chunkNames) {
|
|
1475
|
+
const offset = Number(chunkName.slice(0, 20));
|
|
1476
|
+
if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
|
|
1477
|
+
throw new Error(
|
|
1478
|
+
`offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`,
|
|
1479
|
+
);
|
|
1480
|
+
}
|
|
1481
|
+
const relPath = `${options.upload.relPath}/${chunkName}`;
|
|
1482
|
+
const filePath = await resolveSafeArchiveTarget(options.root, relPath);
|
|
1483
|
+
const content = await readOfflineUploadContent({
|
|
1484
|
+
root: options.root,
|
|
1485
|
+
relPath,
|
|
1486
|
+
filePath,
|
|
1487
|
+
readFile: options.readFile,
|
|
1488
|
+
});
|
|
1489
|
+
expectedOffset += content.length;
|
|
1490
|
+
yield content;
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
async function digestOfflineUploadStagingContent(options: {
|
|
1495
|
+
root: SafeArchiveRoot;
|
|
1496
|
+
upload: OfflineUploadStaging;
|
|
1497
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1498
|
+
}): Promise<{ sha256: string; bytes: number }> {
|
|
1499
|
+
const hash = createHash("sha256");
|
|
1500
|
+
let bytes = 0;
|
|
1501
|
+
for await (const chunk of readOfflineUploadStagingChunks(options)) {
|
|
1502
|
+
hash.update(chunk);
|
|
1503
|
+
bytes += chunk.length;
|
|
1504
|
+
}
|
|
1505
|
+
return { sha256: hash.digest("hex"), bytes };
|
|
1506
|
+
}
|
|
1507
|
+
|
|
1508
|
+
async function writeSafeFileFromUpload(
|
|
1509
|
+
root: SafeArchiveRoot,
|
|
1510
|
+
relPath: string,
|
|
1511
|
+
upload: OfflineUploadStaging,
|
|
1512
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
|
|
1513
|
+
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
|
|
1514
|
+
): Promise<void> {
|
|
1515
|
+
const target = await resolveSafeArchiveTarget(root, relPath);
|
|
1516
|
+
const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
|
|
1517
|
+
if (writeFileChunks) {
|
|
1518
|
+
await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
|
|
1522
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
1523
|
+
const tmp = path.join(
|
|
1524
|
+
path.dirname(target),
|
|
1525
|
+
`.remnic-sync.${process.pid}.${randomUUID()}.tmp`,
|
|
1526
|
+
);
|
|
1527
|
+
const handle = await open(tmp, "w", 0o600);
|
|
1528
|
+
try {
|
|
1529
|
+
for await (const chunk of chunks) {
|
|
1530
|
+
if (chunk.length > 0) await handle.write(chunk);
|
|
1531
|
+
}
|
|
1532
|
+
await handle.close();
|
|
1533
|
+
const targetStat = await lstat(target).catch((error: unknown) => {
|
|
1534
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1535
|
+
throw error;
|
|
1536
|
+
});
|
|
1537
|
+
if (targetStat?.isSymbolicLink()) {
|
|
1538
|
+
throw new Error(`offline sync target is a symlink: ${relPath}`);
|
|
1539
|
+
}
|
|
1540
|
+
await rename(tmp, target);
|
|
1541
|
+
} catch (error) {
|
|
1542
|
+
await handle.close().catch(() => {});
|
|
1543
|
+
await unlink(tmp).catch(() => {});
|
|
1544
|
+
throw error;
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
async function cleanupOfflineUpload(upload: OfflineUploadStaging): Promise<void> {
|
|
1549
|
+
if (upload.kind === "chunks") {
|
|
1550
|
+
await rm(upload.filePath, { recursive: true, force: true });
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
await unlink(upload.filePath).catch((error: unknown) => {
|
|
1554
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
1555
|
+
throw error;
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
async function readOfflineUploadContent(options: {
|
|
1560
|
+
root: SafeArchiveRoot;
|
|
1561
|
+
relPath: string;
|
|
1562
|
+
filePath: string;
|
|
1563
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1564
|
+
}): Promise<Buffer> {
|
|
1565
|
+
if (options.readFile) {
|
|
1566
|
+
return options.readFile({
|
|
1567
|
+
root: options.root.abs,
|
|
1568
|
+
path: options.relPath,
|
|
1569
|
+
filePath: options.filePath,
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
return readFile(options.filePath);
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
async function writeOfflineUploadContent(options: {
|
|
1576
|
+
root: SafeArchiveRoot;
|
|
1577
|
+
relPath: string;
|
|
1578
|
+
filePath: string;
|
|
1579
|
+
content: Buffer;
|
|
1580
|
+
writeFile: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1581
|
+
}): Promise<void> {
|
|
1582
|
+
await options.writeFile({
|
|
1583
|
+
root: options.root.abs,
|
|
1584
|
+
path: options.relPath,
|
|
1585
|
+
filePath: options.filePath,
|
|
1586
|
+
content: options.content,
|
|
1587
|
+
});
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1142
1590
|
async function deleteSafeFile(
|
|
1143
1591
|
root: SafeArchiveRoot,
|
|
1144
1592
|
relPath: string,
|
|
@@ -43,10 +43,21 @@
|
|
|
43
43
|
* Naming: `secure-fs.ts` (not `vault-fs.ts`) — see `kdf.ts` naming note.
|
|
44
44
|
*/
|
|
45
45
|
|
|
46
|
-
import {
|
|
46
|
+
import { createCipheriv, randomBytes } from "node:crypto";
|
|
47
|
+
import { lstat, mkdir, open as openFile, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
47
48
|
import path from "node:path";
|
|
48
49
|
|
|
49
|
-
import {
|
|
50
|
+
import {
|
|
51
|
+
AUTH_TAG_LENGTH,
|
|
52
|
+
ENVELOPE_HEADER_SIZE,
|
|
53
|
+
ENVELOPE_LAYOUT,
|
|
54
|
+
ENVELOPE_SALT_LENGTH,
|
|
55
|
+
ENVELOPE_VERSION,
|
|
56
|
+
IV_LENGTH,
|
|
57
|
+
generateSalt,
|
|
58
|
+
open as openEnvelope,
|
|
59
|
+
seal,
|
|
60
|
+
} from "./cipher.js";
|
|
50
61
|
|
|
51
62
|
// ---------------------------------------------------------------------------
|
|
52
63
|
// Error classes
|
|
@@ -157,7 +168,7 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
|
|
|
157
168
|
}
|
|
158
169
|
const envelope = buf.subarray(MAGIC_HEADER_SIZE);
|
|
159
170
|
try {
|
|
160
|
-
return
|
|
171
|
+
return openEnvelope(key, envelope, aad ? { aad } : {});
|
|
161
172
|
} catch (err) {
|
|
162
173
|
const msg = err instanceof Error ? err.message : String(err);
|
|
163
174
|
throw new SecureStoreDecryptError(
|
|
@@ -166,6 +177,13 @@ export function decryptFileBody(buf: Buffer, key: Buffer, aad?: Buffer): Buffer
|
|
|
166
177
|
}
|
|
167
178
|
}
|
|
168
179
|
|
|
180
|
+
function buildHeaderAad(salt: Uint8Array): Buffer {
|
|
181
|
+
const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
|
|
182
|
+
out.writeUInt8(ENVELOPE_VERSION, 0);
|
|
183
|
+
Buffer.from(salt).copy(out, 1);
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
|
|
169
187
|
// ---------------------------------------------------------------------------
|
|
170
188
|
// Path → AAD helper
|
|
171
189
|
// ---------------------------------------------------------------------------
|
|
@@ -293,6 +311,69 @@ export async function writeMaybeEncryptedFile(
|
|
|
293
311
|
}
|
|
294
312
|
}
|
|
295
313
|
|
|
314
|
+
export async function writeMaybeEncryptedFileFromChunks(
|
|
315
|
+
filePath: string,
|
|
316
|
+
chunks: AsyncIterable<Buffer>,
|
|
317
|
+
key: Buffer | null,
|
|
318
|
+
options: WriteMaybeEncryptedFileOptions = {},
|
|
319
|
+
memoryDir?: string,
|
|
320
|
+
): Promise<void> {
|
|
321
|
+
const { mode = 0o600, atomic = true } = options;
|
|
322
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
323
|
+
const writePath = atomic ? `${filePath}.tmp-${process.pid}-${Date.now()}` : filePath;
|
|
324
|
+
let completed = false;
|
|
325
|
+
try {
|
|
326
|
+
const handle = await openFile(writePath, "w", mode);
|
|
327
|
+
try {
|
|
328
|
+
if (key !== null) {
|
|
329
|
+
const salt = generateSalt();
|
|
330
|
+
const iv = randomBytes(IV_LENGTH);
|
|
331
|
+
const header = Buffer.alloc(MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
|
|
332
|
+
MAGIC_BYTES.copy(header, 0);
|
|
333
|
+
header.writeUInt8(FILE_FORMAT_VERSION, MAGIC_BYTES.length);
|
|
334
|
+
header.writeUInt8(FILE_FORMAT_FLAGS, MAGIC_BYTES.length + 1);
|
|
335
|
+
const envelopeOffset = MAGIC_HEADER_SIZE;
|
|
336
|
+
header.writeUInt8(ENVELOPE_VERSION, envelopeOffset + ENVELOPE_LAYOUT.version);
|
|
337
|
+
salt.copy(header, envelopeOffset + ENVELOPE_LAYOUT.salt);
|
|
338
|
+
iv.copy(header, envelopeOffset + ENVELOPE_LAYOUT.iv);
|
|
339
|
+
await handle.write(header);
|
|
340
|
+
|
|
341
|
+
const cipher = createCipheriv("aes-256-gcm", key, iv, { authTagLength: AUTH_TAG_LENGTH });
|
|
342
|
+
const aad = filePathAad(filePath, memoryDir);
|
|
343
|
+
cipher.setAAD(Buffer.concat([buildHeaderAad(salt), aad]));
|
|
344
|
+
for await (const chunk of chunks) {
|
|
345
|
+
if (chunk.length === 0) continue;
|
|
346
|
+
const encrypted = cipher.update(chunk);
|
|
347
|
+
if (encrypted.length > 0) await handle.write(encrypted);
|
|
348
|
+
}
|
|
349
|
+
const final = cipher.final();
|
|
350
|
+
if (final.length > 0) await handle.write(final);
|
|
351
|
+
const authTag = cipher.getAuthTag();
|
|
352
|
+
await handle.write(
|
|
353
|
+
authTag,
|
|
354
|
+
0,
|
|
355
|
+
authTag.length,
|
|
356
|
+
MAGIC_HEADER_SIZE + ENVELOPE_LAYOUT.authTag,
|
|
357
|
+
);
|
|
358
|
+
} else {
|
|
359
|
+
for await (const chunk of chunks) {
|
|
360
|
+
if (chunk.length > 0) await handle.write(chunk);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
} finally {
|
|
364
|
+
await handle.close();
|
|
365
|
+
}
|
|
366
|
+
if (atomic) {
|
|
367
|
+
await rename(writePath, filePath);
|
|
368
|
+
}
|
|
369
|
+
completed = true;
|
|
370
|
+
} finally {
|
|
371
|
+
if (!completed && atomic) {
|
|
372
|
+
await unlink(writePath).catch(() => {});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
296
377
|
// ---------------------------------------------------------------------------
|
|
297
378
|
// Migration
|
|
298
379
|
// ---------------------------------------------------------------------------
|