@remnic/core 1.1.29 → 1.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +13 -13
- package/dist/access-http.d.ts +1 -1
- package/dist/access-http.js +8 -8
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-mcp.js +7 -7
- package/dist/access-schema.d.ts +55 -5
- package/dist/access-schema.js +4 -2
- package/dist/{access-service-CEyV8XJ5.d.ts → access-service-B5hgZPCN.d.ts} +4 -1
- package/dist/access-service.d.ts +1 -1
- package/dist/access-service.js +5 -5
- package/dist/briefing.js +2 -2
- package/dist/causal-consolidation.js +3 -3
- package/dist/{chunk-25YQM6XW.js → chunk-2IWUMAES.js} +3 -3
- package/dist/{chunk-JUYT2J3K.js → chunk-3OWUCDKH.js} +39 -7
- package/dist/chunk-3OWUCDKH.js.map +1 -0
- package/dist/{chunk-QYHQ2JHL.js → chunk-43PJZYGL.js} +2 -2
- package/dist/{chunk-YITUHONZ.js → chunk-4KGVTPGD.js} +2 -2
- package/dist/{chunk-TR4DK5OH.js → chunk-76FLAAUC.js} +2 -2
- package/dist/{chunk-6BFAEWQS.js → chunk-77H5NU3M.js} +2 -2
- package/dist/{chunk-IANK6Y5W.js → chunk-A6KTB5R6.js} +2 -2
- package/dist/{chunk-7D6O46PF.js → chunk-BVF3AGJP.js} +2 -2
- package/dist/{chunk-4H6DURG6.js → chunk-JA3AK3PT.js} +2 -2
- package/dist/{chunk-WDSIV3AK.js → chunk-KRBK4BQH.js} +12 -12
- package/dist/{chunk-AMVN77EU.js → chunk-MG7NA5H3.js} +365 -90
- package/dist/chunk-MG7NA5H3.js.map +1 -0
- package/dist/{chunk-RCZRL5BE.js → chunk-MRILGULB.js} +2 -2
- package/dist/{chunk-NW7JW5GA.js → chunk-OC7KHOOX.js} +41 -6
- package/dist/chunk-OC7KHOOX.js.map +1 -0
- package/dist/{chunk-LCTP7YRU.js → chunk-QKZGQIPJ.js} +16 -7
- package/dist/chunk-QKZGQIPJ.js.map +1 -0
- package/dist/{chunk-CWWDIQZB.js → chunk-QLLBRHAT.js} +8 -8
- package/dist/{chunk-2WIPXV3Y.js → chunk-RR2PKP3I.js} +2 -2
- package/dist/{chunk-3F24QTRI.js → chunk-SAZS2QZB.js} +2 -2
- package/dist/{chunk-VYU7PXUS.js → chunk-SIC6U3GZ.js} +2 -2
- package/dist/{chunk-6CB4E7ZV.js → chunk-UL2NNBUL.js} +4 -4
- package/dist/{chunk-F33CJ5CH.js → chunk-VBJ7V5SK.js} +40 -8
- package/dist/chunk-VBJ7V5SK.js.map +1 -0
- package/dist/{chunk-6WV2HYTZ.js → chunk-W6AQJ2PY.js} +4 -4
- package/dist/{chunk-PUXCIHRL.js → chunk-XSZEP4SF.js} +2 -2
- package/dist/{cli-BguVmIwO.d.ts → cli-CJKI2JIe.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +17 -17
- package/dist/compounding/engine.js +2 -2
- package/dist/connectors/codex-materialize-runner.js +2 -2
- package/dist/connectors/index.js +2 -2
- package/dist/entity-retrieval.js +2 -2
- package/dist/index.d.ts +4 -4
- package/dist/index.js +32 -22
- package/dist/index.js.map +1 -1
- package/dist/maintenance/memory-governance.js +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +2 -2
- package/dist/maintenance/rebuild-memory-projection.js +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/namespaces/migrate.js +3 -3
- package/dist/namespaces/storage.js +2 -2
- package/dist/offline-sync.d.ts +49 -1
- package/dist/offline-sync.js +13 -1
- package/dist/operator-toolkit.js +5 -5
- package/dist/orchestrator.js +9 -9
- package/dist/semantic-consolidation.js +3 -3
- package/dist/semantic-rule-promotion.js +2 -2
- package/dist/semantic-rule-verifier.js +2 -2
- package/dist/storage.d.ts +5 -0
- package/dist/storage.js +1 -1
- package/dist/verified-recall.js +2 -2
- package/package.json +1 -1
- package/src/access-http.test.ts +184 -0
- package/src/access-http.ts +37 -0
- package/src/access-schema.ts +58 -3
- package/src/access-service-namespace.test.ts +56 -1
- package/src/access-service-offline-file-content.test.ts +17 -0
- package/src/access-service.ts +16 -1
- package/src/index.ts +6 -0
- package/src/offline-sync.test.ts +1055 -1
- package/src/offline-sync.ts +453 -96
- package/src/storage.ts +36 -2
- package/dist/chunk-AMVN77EU.js.map +0 -1
- package/dist/chunk-F33CJ5CH.js.map +0 -1
- package/dist/chunk-JUYT2J3K.js.map +0 -1
- package/dist/chunk-LCTP7YRU.js.map +0 -1
- package/dist/chunk-NW7JW5GA.js.map +0 -1
- /package/dist/{chunk-25YQM6XW.js.map → chunk-2IWUMAES.js.map} +0 -0
- /package/dist/{chunk-QYHQ2JHL.js.map → chunk-43PJZYGL.js.map} +0 -0
- /package/dist/{chunk-YITUHONZ.js.map → chunk-4KGVTPGD.js.map} +0 -0
- /package/dist/{chunk-TR4DK5OH.js.map → chunk-76FLAAUC.js.map} +0 -0
- /package/dist/{chunk-6BFAEWQS.js.map → chunk-77H5NU3M.js.map} +0 -0
- /package/dist/{chunk-IANK6Y5W.js.map → chunk-A6KTB5R6.js.map} +0 -0
- /package/dist/{chunk-7D6O46PF.js.map → chunk-BVF3AGJP.js.map} +0 -0
- /package/dist/{chunk-4H6DURG6.js.map → chunk-JA3AK3PT.js.map} +0 -0
- /package/dist/{chunk-WDSIV3AK.js.map → chunk-KRBK4BQH.js.map} +0 -0
- /package/dist/{chunk-RCZRL5BE.js.map → chunk-MRILGULB.js.map} +0 -0
- /package/dist/{chunk-CWWDIQZB.js.map → chunk-QLLBRHAT.js.map} +0 -0
- /package/dist/{chunk-2WIPXV3Y.js.map → chunk-RR2PKP3I.js.map} +0 -0
- /package/dist/{chunk-3F24QTRI.js.map → chunk-SAZS2QZB.js.map} +0 -0
- /package/dist/{chunk-VYU7PXUS.js.map → chunk-SIC6U3GZ.js.map} +0 -0
- /package/dist/{chunk-6CB4E7ZV.js.map → chunk-UL2NNBUL.js.map} +0 -0
- /package/dist/{chunk-6WV2HYTZ.js.map → chunk-W6AQJ2PY.js.map} +0 -0
- /package/dist/{chunk-PUXCIHRL.js.map → chunk-XSZEP4SF.js.map} +0 -0
package/src/offline-sync.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
2
3
|
import {
|
|
3
4
|
lstat,
|
|
4
5
|
mkdir,
|
|
@@ -9,6 +10,7 @@ import {
|
|
|
9
10
|
rm,
|
|
10
11
|
stat,
|
|
11
12
|
unlink,
|
|
13
|
+
utimes,
|
|
12
14
|
writeFile,
|
|
13
15
|
} from "node:fs/promises";
|
|
14
16
|
import path from "node:path";
|
|
@@ -31,6 +33,8 @@ export const OFFLINE_SYNC_STATE_VERSION = 1;
|
|
|
31
33
|
export const OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES = 64 * 1024 * 1024;
|
|
32
34
|
export const OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
33
35
|
export const OFFLINE_SYNC_APPLY_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
|
36
|
+
export const OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES = 64 * 1024 * 1024;
|
|
37
|
+
export const OFFLINE_SYNC_MAX_MTIME_MS = 8_640_000_000_000_000;
|
|
34
38
|
|
|
35
39
|
export interface OfflineSyncFileState {
|
|
36
40
|
path: string;
|
|
@@ -44,6 +48,11 @@ export interface OfflineSyncFileRecord extends OfflineSyncFileState {
|
|
|
44
48
|
contentBase64?: string;
|
|
45
49
|
}
|
|
46
50
|
|
|
51
|
+
export interface OfflineSyncFileDigest {
|
|
52
|
+
sha256: string;
|
|
53
|
+
bytes: number;
|
|
54
|
+
}
|
|
55
|
+
|
|
47
56
|
export interface OfflineSyncSnapshot {
|
|
48
57
|
format: typeof OFFLINE_SYNC_SNAPSHOT_FORMAT;
|
|
49
58
|
schemaVersion: 1;
|
|
@@ -115,6 +124,7 @@ export interface OfflineSyncApplyChangesetResult {
|
|
|
115
124
|
skipped: number;
|
|
116
125
|
conflicts: OfflineSyncConflict[];
|
|
117
126
|
currentFiles: OfflineSyncFileState[];
|
|
127
|
+
currentFilesComplete?: boolean;
|
|
118
128
|
}
|
|
119
129
|
|
|
120
130
|
export interface OfflineSyncChangesetSummary {
|
|
@@ -172,10 +182,12 @@ interface OfflineSyncFileRecordOptions {
|
|
|
172
182
|
filePath: string;
|
|
173
183
|
includeContent: boolean;
|
|
174
184
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
185
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
175
186
|
}
|
|
176
187
|
|
|
177
188
|
const SYNC_INTERNAL_DIR = ".offline-sync";
|
|
178
189
|
const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
190
|
+
const OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS = 1_000;
|
|
179
191
|
const EXCLUDED_FILE_NAMES = new Set([
|
|
180
192
|
".sync-state.json",
|
|
181
193
|
]);
|
|
@@ -223,6 +235,14 @@ function assertNonNegativeFinite(value: unknown, field: string): number {
|
|
|
223
235
|
return value;
|
|
224
236
|
}
|
|
225
237
|
|
|
238
|
+
function assertOfflineSyncMtimeMs(value: unknown, field: string): number {
|
|
239
|
+
const mtimeMs = assertNonNegativeFinite(value, field);
|
|
240
|
+
if (mtimeMs > OFFLINE_SYNC_MAX_MTIME_MS) {
|
|
241
|
+
throw new Error(`${field} must be within JavaScript Date range`);
|
|
242
|
+
}
|
|
243
|
+
return mtimeMs;
|
|
244
|
+
}
|
|
245
|
+
|
|
226
246
|
function assertBoolean(value: unknown, field: string): boolean {
|
|
227
247
|
if (typeof value !== "boolean") {
|
|
228
248
|
throw new Error(`${field} must be a boolean`);
|
|
@@ -247,7 +267,7 @@ function normalizeFileState(input: unknown, fieldPrefix: string): OfflineSyncFil
|
|
|
247
267
|
path: relPath,
|
|
248
268
|
sha256: assertSha256(obj.sha256, `${fieldPrefix}.sha256`),
|
|
249
269
|
bytes: assertNonNegativeInteger(obj.bytes, `${fieldPrefix}.bytes`),
|
|
250
|
-
mtimeMs:
|
|
270
|
+
mtimeMs: assertOfflineSyncMtimeMs(obj.mtimeMs, `${fieldPrefix}.mtimeMs`),
|
|
251
271
|
};
|
|
252
272
|
}
|
|
253
273
|
|
|
@@ -454,6 +474,23 @@ function isCanonicalRuntimeStatePath(parts: string[]): boolean {
|
|
|
454
474
|
return parts[0] === "namespaces" && parts.length >= 4 && parts[2] === "state";
|
|
455
475
|
}
|
|
456
476
|
|
|
477
|
+
const REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES = new Set([
|
|
478
|
+
".artifact-write-version.log",
|
|
479
|
+
".memory-status-version.log",
|
|
480
|
+
"buffer.json",
|
|
481
|
+
"embeddings.json",
|
|
482
|
+
"index_time.json",
|
|
483
|
+
"last_recall.json",
|
|
484
|
+
"memory-lifecycle-ledger.jsonl",
|
|
485
|
+
"recall_impressions.jsonl",
|
|
486
|
+
]);
|
|
487
|
+
|
|
488
|
+
export function shouldPreferIncomingOfflineRuntimeFile(relPosix: string): boolean {
|
|
489
|
+
const parts = relPosix.split("/");
|
|
490
|
+
const basename = parts[parts.length - 1] ?? "";
|
|
491
|
+
return isCanonicalRuntimeStatePath(parts) && REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES.has(basename);
|
|
492
|
+
}
|
|
493
|
+
|
|
457
494
|
function filterBaseFilesForMode(
|
|
458
495
|
files: readonly OfflineSyncFileState[],
|
|
459
496
|
includeTranscripts: boolean,
|
|
@@ -461,21 +498,70 @@ function filterBaseFilesForMode(
|
|
|
461
498
|
return files.filter((file) => !shouldExcludeRelPath(file.path, includeTranscripts));
|
|
462
499
|
}
|
|
463
500
|
|
|
501
|
+
function canReuseFastBaseFileState(
|
|
502
|
+
baseEntry: OfflineSyncFileState,
|
|
503
|
+
st: { size: number; mtimeMs: number; ctimeMs: number },
|
|
504
|
+
baseCapturedAtMs: number | null,
|
|
505
|
+
): boolean {
|
|
506
|
+
if (baseEntry.bytes !== st.size) return false;
|
|
507
|
+
if (Math.abs(baseEntry.mtimeMs - st.mtimeMs) > OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS) {
|
|
508
|
+
return false;
|
|
509
|
+
}
|
|
510
|
+
if (baseCapturedAtMs === null) return false;
|
|
511
|
+
return st.ctimeMs - baseCapturedAtMs <= OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
async function canReuseFastBaseFileStateFromDisk(
|
|
515
|
+
baseEntry: OfflineSyncFileState,
|
|
516
|
+
filePath: string,
|
|
517
|
+
st: { size: number; mtimeMs: number; ctimeMs: number },
|
|
518
|
+
baseCapturedAtMs: number | null,
|
|
519
|
+
): Promise<boolean> {
|
|
520
|
+
if (!canReuseFastBaseFileState(baseEntry, st, baseCapturedAtMs)) return false;
|
|
521
|
+
return !(await fileIsSecureStoreEncrypted(filePath).catch(() => true));
|
|
522
|
+
}
|
|
523
|
+
|
|
464
524
|
async function readOfflineSyncFileRecord(
|
|
465
525
|
options: OfflineSyncFileRecordOptions,
|
|
466
526
|
): Promise<OfflineSyncFileRecord> {
|
|
467
527
|
const relPath = validateArchiveRelativePath(options.relPath, "offlineSyncFile.path");
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
528
|
+
let content: Buffer | null = null;
|
|
529
|
+
let digest: OfflineSyncFileDigest;
|
|
530
|
+
if (options.includeContent) {
|
|
531
|
+
content = options.readFile
|
|
532
|
+
? await options.readFile({ root: options.root.abs, path: relPath, filePath: options.filePath })
|
|
533
|
+
: await readFile(options.filePath);
|
|
534
|
+
digest = sha256Buffer(content);
|
|
535
|
+
} else if (options.readFileDigest) {
|
|
536
|
+
digest = await options.readFileDigest({ root: options.root.abs, path: relPath, filePath: options.filePath });
|
|
537
|
+
} else if (options.readFile) {
|
|
538
|
+
content = await options.readFile({ root: options.root.abs, path: relPath, filePath: options.filePath });
|
|
539
|
+
digest = sha256Buffer(content);
|
|
540
|
+
content = null;
|
|
541
|
+
} else {
|
|
542
|
+
digest = await sha256File(options.filePath);
|
|
543
|
+
}
|
|
472
544
|
const st = await stat(options.filePath);
|
|
473
545
|
return {
|
|
474
546
|
path: relPath,
|
|
475
547
|
sha256: digest.sha256,
|
|
476
548
|
bytes: digest.bytes,
|
|
477
549
|
mtimeMs: st.mtimeMs,
|
|
478
|
-
...(
|
|
550
|
+
...(content ? { contentBase64: content.toString("base64") } : {}),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
async function sha256File(filePath: string): Promise<OfflineSyncFileDigest> {
|
|
555
|
+
const hash = createHash("sha256");
|
|
556
|
+
let bytes = 0;
|
|
557
|
+
for await (const chunk of createReadStream(filePath)) {
|
|
558
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
559
|
+
hash.update(buffer);
|
|
560
|
+
bytes += buffer.length;
|
|
561
|
+
}
|
|
562
|
+
return {
|
|
563
|
+
sha256: hash.digest("hex"),
|
|
564
|
+
bytes,
|
|
479
565
|
};
|
|
480
566
|
}
|
|
481
567
|
|
|
@@ -515,6 +601,7 @@ export async function buildOfflineSyncSnapshot(options: {
|
|
|
515
601
|
includeTranscripts?: boolean;
|
|
516
602
|
now?: Date;
|
|
517
603
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
604
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
518
605
|
}): Promise<OfflineSyncSnapshot> {
|
|
519
606
|
const rootAbs = path.resolve(options.root);
|
|
520
607
|
const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshot", "root");
|
|
@@ -540,6 +627,78 @@ export async function buildOfflineSyncSnapshot(options: {
|
|
|
540
627
|
filePath: abs,
|
|
541
628
|
includeContent: options.includeContent === true,
|
|
542
629
|
readFile: options.readFile,
|
|
630
|
+
readFileDigest: options.readFileDigest,
|
|
631
|
+
}));
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
await walk(root.abs);
|
|
636
|
+
|
|
637
|
+
return {
|
|
638
|
+
format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
639
|
+
schemaVersion: 1,
|
|
640
|
+
createdAt: (options.now ?? new Date()).toISOString(),
|
|
641
|
+
sourceId: normalizeSourceId(options.sourceId, "sourceId"),
|
|
642
|
+
includeTranscripts,
|
|
643
|
+
files: files.sort(compareByPath),
|
|
644
|
+
};
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
export async function buildOfflineSyncSnapshotFromBase(options: {
|
|
648
|
+
root: string;
|
|
649
|
+
sourceId: string;
|
|
650
|
+
baseFiles?: readonly OfflineSyncFileState[];
|
|
651
|
+
baseCapturedAt?: Date;
|
|
652
|
+
includeContent?: boolean;
|
|
653
|
+
includeTranscripts?: boolean;
|
|
654
|
+
now?: Date;
|
|
655
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
656
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
657
|
+
}): Promise<OfflineSyncSnapshot> {
|
|
658
|
+
const rootAbs = path.resolve(options.root);
|
|
659
|
+
const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshotFromBase", "root");
|
|
660
|
+
const includeTranscripts = options.includeTranscripts !== false;
|
|
661
|
+
const base = byPath(filterBaseFilesForMode(
|
|
662
|
+
normalizeFileStates(options.baseFiles),
|
|
663
|
+
includeTranscripts,
|
|
664
|
+
));
|
|
665
|
+
const rawBaseCapturedAtMs = options.baseCapturedAt?.getTime();
|
|
666
|
+
const baseCapturedAtMs = rawBaseCapturedAtMs !== undefined && Number.isFinite(rawBaseCapturedAtMs)
|
|
667
|
+
? rawBaseCapturedAtMs
|
|
668
|
+
: null;
|
|
669
|
+
const files: OfflineSyncFileRecord[] = [];
|
|
670
|
+
|
|
671
|
+
async function walk(dirAbs: string): Promise<void> {
|
|
672
|
+
let entries = await readdir(dirAbs, { withFileTypes: true });
|
|
673
|
+
entries = entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
674
|
+
for (const entry of entries) {
|
|
675
|
+
const abs = path.join(dirAbs, entry.name);
|
|
676
|
+
const relPosix = path.relative(root.abs, abs).split(path.sep).join("/");
|
|
677
|
+
if (shouldExcludeRelPath(relPosix, includeTranscripts)) continue;
|
|
678
|
+
if (entry.isSymbolicLink()) continue;
|
|
679
|
+
if (entry.isDirectory()) {
|
|
680
|
+
await walk(abs);
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
if (!entry.isFile()) continue;
|
|
684
|
+
const st = await stat(abs);
|
|
685
|
+
const baseEntry = base.get(relPosix);
|
|
686
|
+
if (
|
|
687
|
+
options.includeContent !== true &&
|
|
688
|
+
baseEntry &&
|
|
689
|
+
baseCapturedAtMs !== null &&
|
|
690
|
+
await canReuseFastBaseFileStateFromDisk(baseEntry, abs, st, baseCapturedAtMs)
|
|
691
|
+
) {
|
|
692
|
+
files.push(baseEntry);
|
|
693
|
+
continue;
|
|
694
|
+
}
|
|
695
|
+
files.push(await readOfflineSyncFileRecord({
|
|
696
|
+
root,
|
|
697
|
+
relPath: relPosix,
|
|
698
|
+
filePath: abs,
|
|
699
|
+
includeContent: options.includeContent === true,
|
|
700
|
+
readFile: options.readFile,
|
|
701
|
+
readFileDigest: options.readFileDigest,
|
|
543
702
|
}));
|
|
544
703
|
}
|
|
545
704
|
}
|
|
@@ -564,6 +723,7 @@ export async function buildOfflineSyncSnapshotForPaths(options: {
|
|
|
564
723
|
includeTranscripts?: boolean;
|
|
565
724
|
now?: Date;
|
|
566
725
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
726
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
567
727
|
}): Promise<OfflineSyncSnapshot> {
|
|
568
728
|
const rootAbs = path.resolve(options.root);
|
|
569
729
|
const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshotForPaths", "root");
|
|
@@ -590,6 +750,7 @@ export async function buildOfflineSyncSnapshotForPaths(options: {
|
|
|
590
750
|
filePath,
|
|
591
751
|
includeContent: options.includeContent === true,
|
|
592
752
|
readFile: options.readFile,
|
|
753
|
+
readFileDigest: options.readFileDigest,
|
|
593
754
|
}));
|
|
594
755
|
}
|
|
595
756
|
|
|
@@ -682,6 +843,42 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
682
843
|
root: string;
|
|
683
844
|
sourceId: string;
|
|
684
845
|
baseFiles?: readonly OfflineSyncFileState[];
|
|
846
|
+
baseCapturedAt?: Date;
|
|
847
|
+
excludePaths?: readonly string[];
|
|
848
|
+
includeTranscripts?: boolean;
|
|
849
|
+
now?: Date;
|
|
850
|
+
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
851
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
852
|
+
}): Promise<OfflineSyncChangeset> {
|
|
853
|
+
const includeTranscripts = options.includeTranscripts !== false;
|
|
854
|
+
const current = await buildOfflineSyncSnapshotFromBase({
|
|
855
|
+
root: options.root,
|
|
856
|
+
sourceId: options.sourceId,
|
|
857
|
+
baseFiles: options.baseFiles,
|
|
858
|
+
baseCapturedAt: options.baseCapturedAt,
|
|
859
|
+
includeContent: false,
|
|
860
|
+
includeTranscripts,
|
|
861
|
+
now: options.now,
|
|
862
|
+
readFile: options.readFile,
|
|
863
|
+
readFileDigest: options.readFileDigest,
|
|
864
|
+
});
|
|
865
|
+
return buildOfflineSyncChangesetFromSnapshot({
|
|
866
|
+
root: options.root,
|
|
867
|
+
sourceId: options.sourceId,
|
|
868
|
+
baseFiles: options.baseFiles,
|
|
869
|
+
currentFiles: current.files,
|
|
870
|
+
excludePaths: options.excludePaths,
|
|
871
|
+
includeTranscripts,
|
|
872
|
+
now: options.now,
|
|
873
|
+
readFile: options.readFile,
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
export async function buildOfflineSyncChangesetFromSnapshot(options: {
|
|
878
|
+
root: string;
|
|
879
|
+
sourceId: string;
|
|
880
|
+
currentFiles: readonly OfflineSyncFileState[];
|
|
881
|
+
baseFiles?: readonly OfflineSyncFileState[];
|
|
685
882
|
excludePaths?: readonly string[];
|
|
686
883
|
includeTranscripts?: boolean;
|
|
687
884
|
now?: Date;
|
|
@@ -695,19 +892,18 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
695
892
|
normalizeFileStates(options.baseFiles),
|
|
696
893
|
includeTranscripts,
|
|
697
894
|
));
|
|
698
|
-
const
|
|
699
|
-
|
|
700
|
-
sourceId: options.sourceId,
|
|
701
|
-
includeContent: false,
|
|
895
|
+
const currentMap = byPath(filterBaseFilesForMode(
|
|
896
|
+
normalizeFileStates(options.currentFiles),
|
|
702
897
|
includeTranscripts,
|
|
703
|
-
|
|
704
|
-
readFile: options.readFile,
|
|
705
|
-
});
|
|
706
|
-
const currentMap = byPath(current.files);
|
|
898
|
+
));
|
|
707
899
|
const changes: OfflineSyncChange[] = [];
|
|
708
900
|
|
|
709
901
|
for (const relPath of unionPaths(base, currentMap)) {
|
|
710
902
|
if (excludedPaths.has(relPath)) continue;
|
|
903
|
+
// Runtime state is remote-authoritative in offline sync: local edits and
|
|
904
|
+
// deletes are not pushed; the pull phase restores or removes these files
|
|
905
|
+
// from the remote snapshot.
|
|
906
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath)) continue;
|
|
711
907
|
const baseEntry = base.get(relPath);
|
|
712
908
|
const currentEntry = currentMap.get(relPath);
|
|
713
909
|
if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
|
|
@@ -746,7 +942,7 @@ export async function buildOfflineSyncChangeset(options: {
|
|
|
746
942
|
schemaVersion: 1,
|
|
747
943
|
createdAt: (options.now ?? new Date()).toISOString(),
|
|
748
944
|
sourceId: normalizeSourceId(options.sourceId, "sourceId"),
|
|
749
|
-
includeTranscripts
|
|
945
|
+
includeTranscripts,
|
|
750
946
|
changes: changes.sort(compareByPath),
|
|
751
947
|
};
|
|
752
948
|
}
|
|
@@ -767,28 +963,50 @@ export async function summarizeOfflineSyncPendingChanges(options: {
|
|
|
767
963
|
root: string;
|
|
768
964
|
sourceId: string;
|
|
769
965
|
baseFiles?: readonly OfflineSyncFileState[];
|
|
966
|
+
baseCapturedAt?: Date;
|
|
770
967
|
includeTranscripts?: boolean;
|
|
771
968
|
now?: Date;
|
|
772
969
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
970
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
773
971
|
}): Promise<OfflineSyncChangesetSummary> {
|
|
774
972
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
775
|
-
const
|
|
776
|
-
normalizeFileStates(options.baseFiles),
|
|
777
|
-
includeTranscripts,
|
|
778
|
-
));
|
|
779
|
-
const current = await buildOfflineSyncSnapshot({
|
|
973
|
+
const current = await buildOfflineSyncSnapshotFromBase({
|
|
780
974
|
root: options.root,
|
|
781
975
|
sourceId: options.sourceId,
|
|
976
|
+
baseFiles: options.baseFiles,
|
|
977
|
+
baseCapturedAt: options.baseCapturedAt,
|
|
782
978
|
includeContent: false,
|
|
783
979
|
includeTranscripts,
|
|
784
980
|
now: options.now,
|
|
785
981
|
readFile: options.readFile,
|
|
982
|
+
readFileDigest: options.readFileDigest,
|
|
983
|
+
});
|
|
984
|
+
return summarizeOfflineSyncPendingFiles({
|
|
985
|
+
baseFiles: options.baseFiles,
|
|
986
|
+
currentFiles: current.files,
|
|
987
|
+
includeTranscripts,
|
|
786
988
|
});
|
|
787
|
-
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
export function summarizeOfflineSyncPendingFiles(options: {
|
|
992
|
+
baseFiles?: readonly OfflineSyncFileState[];
|
|
993
|
+
currentFiles: readonly OfflineSyncFileState[];
|
|
994
|
+
includeTranscripts?: boolean;
|
|
995
|
+
}): OfflineSyncChangesetSummary {
|
|
996
|
+
const includeTranscripts = options.includeTranscripts !== false;
|
|
997
|
+
const base = byPath(filterBaseFilesForMode(
|
|
998
|
+
normalizeFileStates(options.baseFiles),
|
|
999
|
+
includeTranscripts,
|
|
1000
|
+
));
|
|
1001
|
+
const currentMap = byPath(filterBaseFilesForMode(
|
|
1002
|
+
normalizeFileStates(options.currentFiles),
|
|
1003
|
+
includeTranscripts,
|
|
1004
|
+
));
|
|
788
1005
|
let upserts = 0;
|
|
789
1006
|
let deletes = 0;
|
|
790
1007
|
|
|
791
1008
|
for (const relPath of unionPaths(base, currentMap)) {
|
|
1009
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath)) continue;
|
|
792
1010
|
const baseEntry = base.get(relPath);
|
|
793
1011
|
const currentEntry = currentMap.get(relPath);
|
|
794
1012
|
if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
|
|
@@ -811,8 +1029,12 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
811
1029
|
root: string;
|
|
812
1030
|
snapshot: unknown;
|
|
813
1031
|
baseFiles?: readonly OfflineSyncFileState[];
|
|
1032
|
+
currentFiles?: readonly OfflineSyncFileState[];
|
|
1033
|
+
deferredPaths?: readonly string[];
|
|
1034
|
+
allowMissingConflictContent?: boolean;
|
|
814
1035
|
writeConflictCopies?: boolean;
|
|
815
1036
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1037
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
816
1038
|
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
817
1039
|
deleteFile?: (target: OfflineSyncFileTarget) => Promise<void>;
|
|
818
1040
|
}): Promise<OfflineSyncApplySnapshotResult> {
|
|
@@ -826,32 +1048,66 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
826
1048
|
requireContent: false,
|
|
827
1049
|
});
|
|
828
1050
|
const root = await ensureSyncRoot(options.root, "applyOfflineSyncSnapshot");
|
|
829
|
-
const
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
1051
|
+
const currentFiles = options.currentFiles
|
|
1052
|
+
? filterBaseFilesForMode(normalizeFileStates(options.currentFiles), snapshot.includeTranscripts).sort(compareByPath)
|
|
1053
|
+
: (await buildOfflineSyncSnapshot({
|
|
1054
|
+
root: root.abs,
|
|
1055
|
+
sourceId: "local",
|
|
1056
|
+
includeContent: false,
|
|
1057
|
+
includeTranscripts: snapshot.includeTranscripts,
|
|
1058
|
+
readFile: options.readFile,
|
|
1059
|
+
readFileDigest: options.readFileDigest,
|
|
1060
|
+
})).files;
|
|
1061
|
+
const currentMap = byPath(currentFiles);
|
|
1062
|
+
const deferredPaths = new Set(options.deferredPaths ?? []);
|
|
837
1063
|
const nextBase = new Map(baseMap);
|
|
838
1064
|
const conflicts: OfflineSyncConflict[] = [];
|
|
839
1065
|
let upserted = 0;
|
|
840
1066
|
let deleted = 0;
|
|
841
1067
|
let skipped = 0;
|
|
842
1068
|
let pendingLocal = 0;
|
|
1069
|
+
const conflictIncomingBuffer = (relPath: string): Buffer | undefined => {
|
|
1070
|
+
if (options.writeConflictCopies === false) return undefined;
|
|
1071
|
+
const buffer = incomingBuffers.get(relPath);
|
|
1072
|
+
if (buffer || options.allowMissingConflictContent === true) return buffer;
|
|
1073
|
+
return requiredBuffer(incomingBuffers, relPath);
|
|
1074
|
+
};
|
|
843
1075
|
|
|
844
1076
|
for (const relPath of unionPaths(baseMap, incomingMap, currentMap)) {
|
|
845
1077
|
const base = baseMap.get(relPath);
|
|
846
1078
|
const incoming = incomingMap.get(relPath);
|
|
847
1079
|
const currentEntry = currentMap.get(relPath);
|
|
848
1080
|
|
|
1081
|
+
if (deferredPaths.has(relPath)) {
|
|
1082
|
+
if (base) nextBase.set(relPath, base);
|
|
1083
|
+
else nextBase.delete(relPath);
|
|
1084
|
+
skipped += 1;
|
|
1085
|
+
continue;
|
|
1086
|
+
}
|
|
1087
|
+
|
|
849
1088
|
if (incoming) {
|
|
850
1089
|
if (currentEntry?.sha256 === incoming.sha256) {
|
|
851
|
-
|
|
1090
|
+
if (await setSafeFileMtime(root, relPath, incoming.mtimeMs)) {
|
|
1091
|
+
nextBase.set(relPath, toFileState(incoming));
|
|
1092
|
+
} else {
|
|
1093
|
+
if (base) nextBase.set(relPath, base);
|
|
1094
|
+
else nextBase.delete(relPath);
|
|
1095
|
+
pendingLocal += 1;
|
|
1096
|
+
}
|
|
1097
|
+
skipped += 1;
|
|
1098
|
+
continue;
|
|
1099
|
+
}
|
|
1100
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath) && currentEntry && base && incoming.sha256 === base.sha256) {
|
|
1101
|
+
nextBase.set(relPath, base);
|
|
852
1102
|
skipped += 1;
|
|
853
1103
|
continue;
|
|
854
1104
|
}
|
|
1105
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath)) {
|
|
1106
|
+
await writeSafeFile(root, relPath, requiredBuffer(incomingBuffers, relPath), options.writeFile, incoming.mtimeMs);
|
|
1107
|
+
nextBase.set(relPath, toFileState(incoming));
|
|
1108
|
+
upserted += 1;
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
855
1111
|
if (!currentEntry && base && incoming.sha256 === base.sha256) {
|
|
856
1112
|
nextBase.set(relPath, base);
|
|
857
1113
|
pendingLocal += 1;
|
|
@@ -865,9 +1121,7 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
865
1121
|
reason: "local_deleted_remote_modified",
|
|
866
1122
|
baseSha256: base.sha256,
|
|
867
1123
|
incomingSha256: incoming.sha256,
|
|
868
|
-
incomingBuffer:
|
|
869
|
-
? incomingBuffers.get(relPath)
|
|
870
|
-
: requiredBuffer(incomingBuffers, relPath),
|
|
1124
|
+
incomingBuffer: conflictIncomingBuffer(relPath),
|
|
871
1125
|
writeConflictCopies: options.writeConflictCopies !== false,
|
|
872
1126
|
sourceId: snapshot.sourceId,
|
|
873
1127
|
writeFile: options.writeFile,
|
|
@@ -876,13 +1130,13 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
876
1130
|
continue;
|
|
877
1131
|
}
|
|
878
1132
|
if (!currentEntry && !base) {
|
|
879
|
-
await writeSafeFile(root, relPath, requiredBuffer(incomingBuffers, relPath), options.writeFile);
|
|
1133
|
+
await writeSafeFile(root, relPath, requiredBuffer(incomingBuffers, relPath), options.writeFile, incoming.mtimeMs);
|
|
880
1134
|
nextBase.set(relPath, toFileState(incoming));
|
|
881
1135
|
upserted += 1;
|
|
882
1136
|
continue;
|
|
883
1137
|
}
|
|
884
1138
|
if (base && currentEntry && currentEntry.sha256 === base.sha256) {
|
|
885
|
-
await writeSafeFile(root, relPath, requiredBuffer(incomingBuffers, relPath), options.writeFile);
|
|
1139
|
+
await writeSafeFile(root, relPath, requiredBuffer(incomingBuffers, relPath), options.writeFile, incoming.mtimeMs);
|
|
886
1140
|
nextBase.set(relPath, toFileState(incoming));
|
|
887
1141
|
upserted += 1;
|
|
888
1142
|
continue;
|
|
@@ -900,9 +1154,7 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
900
1154
|
baseSha256: base?.sha256,
|
|
901
1155
|
localSha256: currentEntry?.sha256,
|
|
902
1156
|
incomingSha256: incoming.sha256,
|
|
903
|
-
incomingBuffer:
|
|
904
|
-
? incomingBuffers.get(relPath)
|
|
905
|
-
: requiredBuffer(incomingBuffers, relPath),
|
|
1157
|
+
incomingBuffer: conflictIncomingBuffer(relPath),
|
|
906
1158
|
writeConflictCopies: options.writeConflictCopies !== false,
|
|
907
1159
|
sourceId: snapshot.sourceId,
|
|
908
1160
|
writeFile: options.writeFile,
|
|
@@ -916,6 +1168,17 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
916
1168
|
skipped += 1;
|
|
917
1169
|
continue;
|
|
918
1170
|
}
|
|
1171
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath) && base) {
|
|
1172
|
+
await deleteSafeFile(root, relPath, options.deleteFile);
|
|
1173
|
+
nextBase.delete(relPath);
|
|
1174
|
+
deleted += 1;
|
|
1175
|
+
continue;
|
|
1176
|
+
}
|
|
1177
|
+
if (shouldPreferIncomingOfflineRuntimeFile(relPath)) {
|
|
1178
|
+
pendingLocal += 1;
|
|
1179
|
+
skipped += 1;
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
919
1182
|
if (base && currentEntry.sha256 === base.sha256) {
|
|
920
1183
|
await deleteSafeFile(root, relPath, options.deleteFile);
|
|
921
1184
|
nextBase.delete(relPath);
|
|
@@ -949,8 +1212,11 @@ export async function applyOfflineSyncSnapshot(options: {
|
|
|
949
1212
|
export async function applyOfflineSyncChangeset(options: {
|
|
950
1213
|
root: string;
|
|
951
1214
|
changeset: unknown;
|
|
1215
|
+
currentFiles?: readonly OfflineSyncFileState[];
|
|
1216
|
+
returnCurrentFiles?: boolean;
|
|
952
1217
|
writeConflictCopies?: boolean;
|
|
953
1218
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1219
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
954
1220
|
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
955
1221
|
deleteFile?: (target: OfflineSyncFileTarget) => Promise<void>;
|
|
956
1222
|
}): Promise<OfflineSyncApplyChangesetResult> {
|
|
@@ -970,14 +1236,18 @@ export async function applyOfflineSyncChangeset(options: {
|
|
|
970
1236
|
.filter((change): change is Extract<OfflineSyncChange, { type: "upsert" }> => change.type === "upsert")
|
|
971
1237
|
.map((change) => change.file);
|
|
972
1238
|
const incomingBuffers = verifyRecordContents(records, "offline sync changeset");
|
|
973
|
-
const
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
1239
|
+
const currentFiles = options.currentFiles
|
|
1240
|
+
? filterBaseFilesForMode(normalizeFileStates(options.currentFiles), changeset.includeTranscripts).sort(compareByPath)
|
|
1241
|
+
: (await buildOfflineSyncSnapshotForPaths({
|
|
1242
|
+
root: root.abs,
|
|
1243
|
+
sourceId: "local",
|
|
1244
|
+
paths: changeset.changes.map((change) => change.path),
|
|
1245
|
+
includeContent: false,
|
|
1246
|
+
includeTranscripts: changeset.includeTranscripts,
|
|
1247
|
+
readFile: options.readFile,
|
|
1248
|
+
readFileDigest: options.readFileDigest,
|
|
1249
|
+
})).files;
|
|
1250
|
+
const currentMap = byPath(currentFiles);
|
|
981
1251
|
const conflicts: OfflineSyncConflict[] = [];
|
|
982
1252
|
let appliedUpserts = 0;
|
|
983
1253
|
let appliedDeletes = 0;
|
|
@@ -987,12 +1257,18 @@ export async function applyOfflineSyncChangeset(options: {
|
|
|
987
1257
|
const currentEntry = currentMap.get(change.path);
|
|
988
1258
|
if (change.type === "upsert") {
|
|
989
1259
|
if (currentEntry?.sha256 === change.file.sha256) {
|
|
990
|
-
|
|
1260
|
+
if (await setSafeFileMtime(root, change.path, change.file.mtimeMs)) {
|
|
1261
|
+
skipped += 1;
|
|
1262
|
+
} else {
|
|
1263
|
+
await writeSafeFile(root, change.path, requiredBuffer(incomingBuffers, change.path), options.writeFile, change.file.mtimeMs);
|
|
1264
|
+
currentMap.set(change.path, toFileState(change.file));
|
|
1265
|
+
appliedUpserts += 1;
|
|
1266
|
+
}
|
|
991
1267
|
continue;
|
|
992
1268
|
}
|
|
993
1269
|
if (!change.baseSha256) {
|
|
994
1270
|
if (!currentEntry) {
|
|
995
|
-
await writeSafeFile(root, change.path, requiredBuffer(incomingBuffers, change.path), options.writeFile);
|
|
1271
|
+
await writeSafeFile(root, change.path, requiredBuffer(incomingBuffers, change.path), options.writeFile, change.file.mtimeMs);
|
|
996
1272
|
currentMap.set(change.path, toFileState(change.file));
|
|
997
1273
|
appliedUpserts += 1;
|
|
998
1274
|
continue;
|
|
@@ -1011,7 +1287,7 @@ export async function applyOfflineSyncChangeset(options: {
|
|
|
1011
1287
|
continue;
|
|
1012
1288
|
}
|
|
1013
1289
|
if (currentEntry?.sha256 === change.baseSha256) {
|
|
1014
|
-
await writeSafeFile(root, change.path, requiredBuffer(incomingBuffers, change.path), options.writeFile);
|
|
1290
|
+
await writeSafeFile(root, change.path, requiredBuffer(incomingBuffers, change.path), options.writeFile, change.file.mtimeMs);
|
|
1015
1291
|
currentMap.set(change.path, toFileState(change.file));
|
|
1016
1292
|
appliedUpserts += 1;
|
|
1017
1293
|
continue;
|
|
@@ -1054,7 +1330,17 @@ export async function applyOfflineSyncChangeset(options: {
|
|
|
1054
1330
|
appliedDeletes,
|
|
1055
1331
|
skipped,
|
|
1056
1332
|
conflicts,
|
|
1057
|
-
currentFiles:
|
|
1333
|
+
currentFiles: options.returnCurrentFiles === false
|
|
1334
|
+
? [...currentMap.values()].sort(compareByPath)
|
|
1335
|
+
: (await buildOfflineSyncSnapshot({
|
|
1336
|
+
root: root.abs,
|
|
1337
|
+
sourceId: "local",
|
|
1338
|
+
includeContent: false,
|
|
1339
|
+
includeTranscripts: changeset.includeTranscripts,
|
|
1340
|
+
readFile: options.readFile,
|
|
1341
|
+
readFileDigest: options.readFileDigest,
|
|
1342
|
+
})).files,
|
|
1343
|
+
...(options.returnCurrentFiles === false ? { currentFilesComplete: false } : {}),
|
|
1058
1344
|
};
|
|
1059
1345
|
}
|
|
1060
1346
|
|
|
@@ -1125,10 +1411,12 @@ async function writeSafeFile(
|
|
|
1125
1411
|
relPath: string,
|
|
1126
1412
|
content: Buffer,
|
|
1127
1413
|
writeFileHook?: (target: OfflineSyncFileWriteTarget) => Promise<void>,
|
|
1414
|
+
mtimeMs?: number,
|
|
1128
1415
|
): Promise<void> {
|
|
1129
1416
|
const target = await resolveSafeArchiveTarget(root, relPath);
|
|
1130
1417
|
if (writeFileHook) {
|
|
1131
1418
|
await writeFileHook({ root: root.abs, path: relPath, filePath: target, content });
|
|
1419
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1132
1420
|
return;
|
|
1133
1421
|
}
|
|
1134
1422
|
await mkdir(path.dirname(target), { recursive: true });
|
|
@@ -1146,12 +1434,33 @@ async function writeSafeFile(
|
|
|
1146
1434
|
throw new Error(`offline sync target is a symlink: ${relPath}`);
|
|
1147
1435
|
}
|
|
1148
1436
|
await rename(tmp, target);
|
|
1437
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1149
1438
|
} catch (error) {
|
|
1150
1439
|
await unlink(tmp).catch(() => {});
|
|
1151
1440
|
throw error;
|
|
1152
1441
|
}
|
|
1153
1442
|
}
|
|
1154
1443
|
|
|
1444
|
+
async function setSafeFileMtime(
|
|
1445
|
+
root: SafeArchiveRoot,
|
|
1446
|
+
relPath: string,
|
|
1447
|
+
mtimeMs: number | undefined,
|
|
1448
|
+
): Promise<boolean> {
|
|
1449
|
+
if (mtimeMs === undefined) return true;
|
|
1450
|
+
const target = await resolveSafeArchiveTarget(root, relPath);
|
|
1451
|
+
const targetStat = await lstat(target).catch((error: unknown) => {
|
|
1452
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
1453
|
+
throw error;
|
|
1454
|
+
});
|
|
1455
|
+
if (!targetStat) return false;
|
|
1456
|
+
if (targetStat.isSymbolicLink()) {
|
|
1457
|
+
throw new Error(`offline sync target is a symlink: ${relPath}`);
|
|
1458
|
+
}
|
|
1459
|
+
const mtime = new Date(assertOfflineSyncMtimeMs(mtimeMs, "mtimeMs"));
|
|
1460
|
+
await utimes(target, mtime, mtime);
|
|
1461
|
+
return true;
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1155
1464
|
export async function applyOfflineSyncFileContentChunk(options: {
|
|
1156
1465
|
root: string;
|
|
1157
1466
|
sourceId: string;
|
|
@@ -1164,6 +1473,7 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1164
1473
|
baseSha256?: string;
|
|
1165
1474
|
includeTranscripts?: boolean;
|
|
1166
1475
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
1476
|
+
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
1167
1477
|
writeFile?: (target: OfflineSyncFileWriteTarget) => Promise<void>;
|
|
1168
1478
|
writeStagingFile?: (target: OfflineSyncFileStagingWriteTarget) => Promise<void>;
|
|
1169
1479
|
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>;
|
|
@@ -1177,13 +1487,14 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1177
1487
|
}
|
|
1178
1488
|
const sha256 = assertSha256(options.sha256, "sha256");
|
|
1179
1489
|
const bytes = assertNonNegativeInteger(options.bytes, "bytes");
|
|
1180
|
-
const mtimeMs =
|
|
1490
|
+
const mtimeMs = assertOfflineSyncMtimeMs(options.mtimeMs, "mtimeMs");
|
|
1181
1491
|
const offset = options.offset === undefined
|
|
1182
1492
|
? 0
|
|
1183
1493
|
: assertNonNegativeInteger(options.offset, "offset");
|
|
1184
1494
|
const baseSha256 = options.baseSha256 === undefined
|
|
1185
1495
|
? undefined
|
|
1186
1496
|
: assertSha256(options.baseSha256, "baseSha256");
|
|
1497
|
+
const preferIncomingRuntimeFile = shouldPreferIncomingOfflineRuntimeFile(relPath);
|
|
1187
1498
|
if (!Buffer.isBuffer(options.content)) {
|
|
1188
1499
|
throw new Error("content must be a Buffer");
|
|
1189
1500
|
}
|
|
@@ -1204,8 +1515,87 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1204
1515
|
if (options.writeFile && !options.writeStagingFile) {
|
|
1205
1516
|
throw new Error("offline sync upload storage hooks require writeStagingFile");
|
|
1206
1517
|
}
|
|
1518
|
+
const baseResult = {
|
|
1519
|
+
path: relPath,
|
|
1520
|
+
sha256,
|
|
1521
|
+
bytes,
|
|
1522
|
+
mtimeMs,
|
|
1523
|
+
offset,
|
|
1524
|
+
chunkBytes: options.content.length,
|
|
1525
|
+
done: offset + options.content.length === bytes,
|
|
1526
|
+
};
|
|
1527
|
+
const currentFileConflict = async (
|
|
1528
|
+
currentFile: OfflineSyncFileState | undefined,
|
|
1529
|
+
): Promise<{ conflict: OfflineSyncConflict; currentFile?: OfflineSyncFileState } | null> => {
|
|
1530
|
+
if (!baseSha256 && currentFile && !preferIncomingRuntimeFile) {
|
|
1531
|
+
const conflict = await recordConflict({
|
|
1532
|
+
root,
|
|
1533
|
+
relPath,
|
|
1534
|
+
reason: "remote_exists_for_local_create",
|
|
1535
|
+
localSha256: currentFile.sha256,
|
|
1536
|
+
incomingSha256: sha256,
|
|
1537
|
+
writeConflictCopies: false,
|
|
1538
|
+
sourceId,
|
|
1539
|
+
writeFile: options.writeFile,
|
|
1540
|
+
});
|
|
1541
|
+
return {
|
|
1542
|
+
conflict,
|
|
1543
|
+
currentFile,
|
|
1544
|
+
};
|
|
1545
|
+
}
|
|
1546
|
+
if (baseSha256 && currentFile?.sha256 !== baseSha256 && !preferIncomingRuntimeFile) {
|
|
1547
|
+
const conflict = await recordConflict({
|
|
1548
|
+
root,
|
|
1549
|
+
relPath,
|
|
1550
|
+
reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
|
|
1551
|
+
baseSha256,
|
|
1552
|
+
localSha256: currentFile?.sha256,
|
|
1553
|
+
incomingSha256: sha256,
|
|
1554
|
+
writeConflictCopies: false,
|
|
1555
|
+
sourceId,
|
|
1556
|
+
writeFile: options.writeFile,
|
|
1557
|
+
});
|
|
1558
|
+
return {
|
|
1559
|
+
conflict,
|
|
1560
|
+
...(currentFile ? { currentFile } : {}),
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
return null;
|
|
1564
|
+
};
|
|
1207
1565
|
if (offset === 0) {
|
|
1208
1566
|
await pruneOfflineUploadStaging(root);
|
|
1567
|
+
const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
|
|
1568
|
+
root: root.abs,
|
|
1569
|
+
sourceId: "local",
|
|
1570
|
+
paths: [relPath],
|
|
1571
|
+
includeContent: false,
|
|
1572
|
+
includeTranscripts,
|
|
1573
|
+
readFile: options.readFile,
|
|
1574
|
+
readFileDigest: options.readFileDigest,
|
|
1575
|
+
});
|
|
1576
|
+
const currentFile = currentSnapshot.files[0];
|
|
1577
|
+
if (currentFile?.sha256 === sha256) {
|
|
1578
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1579
|
+
return {
|
|
1580
|
+
...baseResult,
|
|
1581
|
+
done: true,
|
|
1582
|
+
chunkBytes: 0,
|
|
1583
|
+
applied: false,
|
|
1584
|
+
skipped: true,
|
|
1585
|
+
currentFile: toFileState(currentFile),
|
|
1586
|
+
};
|
|
1587
|
+
}
|
|
1588
|
+
const conflictResult = await currentFileConflict(currentFile ? toFileState(currentFile) : undefined);
|
|
1589
|
+
if (conflictResult) {
|
|
1590
|
+
return {
|
|
1591
|
+
...baseResult,
|
|
1592
|
+
done: true,
|
|
1593
|
+
chunkBytes: 0,
|
|
1594
|
+
applied: false,
|
|
1595
|
+
skipped: false,
|
|
1596
|
+
...conflictResult,
|
|
1597
|
+
};
|
|
1598
|
+
}
|
|
1209
1599
|
}
|
|
1210
1600
|
|
|
1211
1601
|
const upload = await writeOfflineUploadChunk({
|
|
@@ -1220,16 +1610,7 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1220
1610
|
writeFile: options.writeFile,
|
|
1221
1611
|
writeStagingFile: options.writeStagingFile,
|
|
1222
1612
|
});
|
|
1223
|
-
const done =
|
|
1224
|
-
const baseResult = {
|
|
1225
|
-
path: relPath,
|
|
1226
|
-
sha256,
|
|
1227
|
-
bytes,
|
|
1228
|
-
mtimeMs,
|
|
1229
|
-
offset,
|
|
1230
|
-
chunkBytes: options.content.length,
|
|
1231
|
-
done,
|
|
1232
|
-
};
|
|
1613
|
+
const done = baseResult.done;
|
|
1233
1614
|
if (!done) {
|
|
1234
1615
|
return {
|
|
1235
1616
|
...baseResult,
|
|
@@ -1255,6 +1636,7 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1255
1636
|
includeContent: false,
|
|
1256
1637
|
includeTranscripts,
|
|
1257
1638
|
readFile: options.readFile,
|
|
1639
|
+
readFileDigest: options.readFileDigest,
|
|
1258
1640
|
});
|
|
1259
1641
|
const currentFile = currentSnapshot.files[0];
|
|
1260
1642
|
const uploadedState: OfflineSyncFileState = {
|
|
@@ -1266,54 +1648,26 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
1266
1648
|
|
|
1267
1649
|
try {
|
|
1268
1650
|
if (currentFile?.sha256 === sha256) {
|
|
1651
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1269
1652
|
return {
|
|
1270
1653
|
...baseResult,
|
|
1271
1654
|
applied: false,
|
|
1272
1655
|
skipped: true,
|
|
1273
|
-
currentFile:
|
|
1274
|
-
};
|
|
1275
|
-
}
|
|
1276
|
-
if (!baseSha256 && currentFile) {
|
|
1277
|
-
const conflict = await recordConflict({
|
|
1278
|
-
root,
|
|
1279
|
-
relPath,
|
|
1280
|
-
reason: "remote_exists_for_local_create",
|
|
1281
|
-
localSha256: currentFile.sha256,
|
|
1282
|
-
incomingSha256: sha256,
|
|
1283
|
-
writeConflictCopies: false,
|
|
1284
|
-
sourceId,
|
|
1285
|
-
writeFile: options.writeFile,
|
|
1286
|
-
});
|
|
1287
|
-
return {
|
|
1288
|
-
...baseResult,
|
|
1289
|
-
applied: false,
|
|
1290
|
-
skipped: false,
|
|
1291
|
-
conflict,
|
|
1292
|
-
currentFile: toFileState(currentFile),
|
|
1656
|
+
currentFile: uploadedState,
|
|
1293
1657
|
};
|
|
1294
1658
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
relPath,
|
|
1299
|
-
reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
|
|
1300
|
-
baseSha256,
|
|
1301
|
-
localSha256: currentFile?.sha256,
|
|
1302
|
-
incomingSha256: sha256,
|
|
1303
|
-
writeConflictCopies: false,
|
|
1304
|
-
sourceId,
|
|
1305
|
-
writeFile: options.writeFile,
|
|
1306
|
-
});
|
|
1659
|
+
|
|
1660
|
+
const conflictResult = await currentFileConflict(currentFile ? toFileState(currentFile) : undefined);
|
|
1661
|
+
if (conflictResult) {
|
|
1307
1662
|
return {
|
|
1308
1663
|
...baseResult,
|
|
1309
1664
|
applied: false,
|
|
1310
1665
|
skipped: false,
|
|
1311
|
-
|
|
1312
|
-
...(currentFile ? { currentFile: toFileState(currentFile) } : {}),
|
|
1666
|
+
...conflictResult,
|
|
1313
1667
|
};
|
|
1314
1668
|
}
|
|
1315
1669
|
|
|
1316
|
-
await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
|
|
1670
|
+
await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks, mtimeMs);
|
|
1317
1671
|
return {
|
|
1318
1672
|
...baseResult,
|
|
1319
1673
|
applied: true,
|
|
@@ -1513,11 +1867,13 @@ async function writeSafeFileFromUpload(
|
|
|
1513
1867
|
upload: OfflineUploadStaging,
|
|
1514
1868
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>,
|
|
1515
1869
|
writeFileChunks?: (target: OfflineSyncFileWriteChunksTarget) => Promise<void>,
|
|
1870
|
+
mtimeMs?: number,
|
|
1516
1871
|
): Promise<void> {
|
|
1517
1872
|
const target = await resolveSafeArchiveTarget(root, relPath);
|
|
1518
1873
|
const chunks = readOfflineUploadStagingChunks({ root, upload, readFile });
|
|
1519
1874
|
if (writeFileChunks) {
|
|
1520
1875
|
await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
|
|
1876
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1521
1877
|
return;
|
|
1522
1878
|
}
|
|
1523
1879
|
|
|
@@ -1540,6 +1896,7 @@ async function writeSafeFileFromUpload(
|
|
|
1540
1896
|
throw new Error(`offline sync target is a symlink: ${relPath}`);
|
|
1541
1897
|
}
|
|
1542
1898
|
await rename(tmp, target);
|
|
1899
|
+
await setSafeFileMtime(root, relPath, mtimeMs);
|
|
1543
1900
|
} catch (error) {
|
|
1544
1901
|
await handle.close().catch(() => {});
|
|
1545
1902
|
await unlink(tmp).catch(() => {});
|