@indigoai-us/hq-cloud 6.16.67 → 6.16.68-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +20 -5
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +27 -4
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +192 -122
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +162 -8
- package/dist/cli/sync.test.js.map +1 -1
- package/package.json +1 -1
package/dist/cli/sync.d.ts
CHANGED
|
@@ -585,10 +585,10 @@ export interface SyncOptions {
|
|
|
585
585
|
*/
|
|
586
586
|
onHeartbeat?: () => void;
|
|
587
587
|
/**
|
|
588
|
-
* Watch-runner hand-off seam for an unscoped full pull.
|
|
589
|
-
*
|
|
590
|
-
* Returning true means realtime ran; this pull
|
|
591
|
-
*
|
|
588
|
+
* Watch-runner hand-off seam for an unscoped full pull. Invoked at LIST
|
|
589
|
+
* page boundaries, every 500 planned rows, and after each drained download
|
|
590
|
+
* chunk. Returning true means realtime ran; this pull overlays the journal
|
|
591
|
+
* delta and continues the same LIST/plan/queue instead of re-LISTing S3.
|
|
592
592
|
*/
|
|
593
593
|
cooperativeCheckpoint?: () => Promise<void | boolean>;
|
|
594
594
|
/**
|
|
@@ -848,6 +848,28 @@ export declare function targetedJournalSessionCensusSizes(): Record<string, numb
|
|
|
848
848
|
* resulting journal delta is durable before realtime can acquire the writer.
|
|
849
849
|
*/
|
|
850
850
|
export declare const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
851
|
+
/**
|
|
852
|
+
* ListObjectsV2 page size (AWS MaxKeys default). Full-pull LIST yields the
|
|
853
|
+
* root lock at each page boundary so queued realtime work is not stuck behind
|
|
854
|
+
* a 30-minute personal-vault spool.
|
|
855
|
+
*/
|
|
856
|
+
export declare const FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE = 1000;
|
|
857
|
+
declare function enableCooperativeRowVisits(): void;
|
|
858
|
+
declare function disableCooperativeRowVisits(): void;
|
|
859
|
+
declare function snapshotCooperativeRowVisits(): {
|
|
860
|
+
list: Map<string, number>;
|
|
861
|
+
plan: Map<string, number>;
|
|
862
|
+
};
|
|
863
|
+
/**
|
|
864
|
+
* Test-only export. Kept under `_testing` so the public surface stays `sync()`.
|
|
865
|
+
* Do NOT import from `_testing` outside of tests in this package.
|
|
866
|
+
*/
|
|
867
|
+
export declare const _testing: {
|
|
868
|
+
enableCooperativeRowVisits: typeof enableCooperativeRowVisits;
|
|
869
|
+
disableCooperativeRowVisits: typeof disableCooperativeRowVisits;
|
|
870
|
+
cooperativeRowVisits: typeof snapshotCooperativeRowVisits;
|
|
871
|
+
cooperativePhase: () => "list" | "plan" | "download" | null;
|
|
872
|
+
};
|
|
851
873
|
/**
|
|
852
874
|
* Best-effort report of the files that were new to this drive during the sync,
|
|
853
875
|
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
@@ -882,4 +904,5 @@ export declare function sync(options: SyncOptions): Promise<SyncResult>;
|
|
|
882
904
|
* false off win32.
|
|
883
905
|
*/
|
|
884
906
|
export declare function win32SymlinkFlavorBroken(localPath: string, win32?: boolean): boolean;
|
|
907
|
+
export {};
|
|
885
908
|
//# sourceMappingURL=sync.d.ts.map
|
package/dist/cli/sync.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAUhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAgEhC,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAML,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAW5F,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,cAAc,EACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,kBAAkB,EAClB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAA4B,KAAK,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEtG,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,+CAAgD,CAAC;AAEzF;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,kBAAkB,CAAC;AAElE,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAI3E;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,KAAK,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,KAAK,CAAC,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;CACnJ;AA+CD;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,CAEjH;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,kBAAkB,EAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC,EACjE,YAAY,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,mBAAmB,CAAC,GACjE,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;IAC/B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,GACD;IACE;;;;;;OAMG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,sBAAsB,GAAG,0BAA0B,CAAC;CAC/D,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,mCAAmC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EACF,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GACD;IACE;;;;;;;OAOG;IACH,IAAI,EAAE,sCAAsC,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GACD;IACE;;;;;;;;;OASG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EACF,kBAAkB,GAClB,iBAAiB,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,2BAA2B,GAC3B,4BAA4B,GAC5B,6BAA6B,GAC7B,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;;;;;;;;;;OAiBG;IACH,IAAI,EAAE,qBAAqB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;IACtD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACrC;;;;;;;;;;;;;;;;;;OAkBG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;OAIG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;CAC3C;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAUhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAgEhC,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAML,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAW5F,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,cAAc,EACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,kBAAkB,EAClB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAA4B,KAAK,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEtG,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,+CAAgD,CAAC;AAEzF;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,kBAAkB,CAAC;AAElE,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAI3E;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,KAAK,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,KAAK,CAAC,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;CACnJ;AA+CD;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,CAEjH;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,kBAAkB,EAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC,EACjE,YAAY,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,mBAAmB,CAAC,GACjE,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;IAC/B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,GACD;IACE;;;;;;OAMG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,sBAAsB,GAAG,0BAA0B,CAAC;CAC/D,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,mCAAmC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EACF,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GACD;IACE;;;;;;;OAOG;IACH,IAAI,EAAE,sCAAsC,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GACD;IACE;;;;;;;;;OASG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EACF,kBAAkB,GAClB,iBAAiB,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,2BAA2B,GAC3B,4BAA4B,GAC5B,6BAA6B,GAC7B,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;;;;;;;;;;OAiBG;IACH,IAAI,EAAE,qBAAqB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;IACtD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACrC;;;;;;;;;;;;;;;;;;OAkBG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;OAIG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;CAC3C;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAsJD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAM5C;AAwDD,2EAA2E;AAC3E,wBAAgB,iCAAiC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAK1E;AAOD;;;GAGG;AACH,eAAO,MAAM,2CAA2C,MAAM,CAAC;AAC/D;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,OAAO,CAAC;AAmBxD,iBAAS,0BAA0B,IAAI,IAAI,CAK1C;AAED,iBAAS,2BAA2B,IAAI,IAAI,CAK3C;AAED,iBAAS,4BAA4B,IAAI;IACvC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1B,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3B,CAKA;AAED;;;GAGG;AACH,eAAO,MAAM,QAAQ;;;;4BAIG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI;CAC1D,CAAC;AAYF;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,kBAAkB,EAC/B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,EACrE,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,GACvC,OAAO,CAAC,IAAI,CAAC,CA6Ef;AAoCD;;GAEG;AACH,wBAAsB,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAOpE;AA4sED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,OAAsC,GAC5C,OAAO,CAoBT"}
|
package/dist/cli/sync.js
CHANGED
|
@@ -123,6 +123,56 @@ export function createSyncMutationClient(options) {
|
|
|
123
123
|
export async function applyRealtimeDeltaPage(engine, page, fetchContent) {
|
|
124
124
|
await engine.applyPage(page, fetchContent);
|
|
125
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Let queued realtime take the root lock at a bounded LIST/plan/download
|
|
128
|
+
* unit, then resume the same spool and remaining queue. A true return means
|
|
129
|
+
* the scheduler handed off; the caller overlays the journal delta rather than
|
|
130
|
+
* throwing away the LIST snapshot.
|
|
131
|
+
*/
|
|
132
|
+
async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force, resumedAt) {
|
|
133
|
+
work.rowsSinceYield += 1;
|
|
134
|
+
if (!force && work.rowsSinceYield < FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
work.rowsSinceYield = 0;
|
|
138
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
139
|
+
if (!cooperativeCheckpoint)
|
|
140
|
+
return false;
|
|
141
|
+
const handedOff = (await cooperativeCheckpoint()) === true;
|
|
142
|
+
if (handedOff) {
|
|
143
|
+
process.stderr.write(`${JSON.stringify({
|
|
144
|
+
type: "cooperative-handoff",
|
|
145
|
+
direction: "pull",
|
|
146
|
+
resumedAt,
|
|
147
|
+
})}\n`);
|
|
148
|
+
}
|
|
149
|
+
return handedOff;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Freshness after a cooperative hand-off is the journal dirty-log overlay, not
|
|
153
|
+
* a second S3 LIST. The spool exists so classification can replay one snapshot
|
|
154
|
+
* after the caller refreshes its journal (remote-file-spool.ts:29-33).
|
|
155
|
+
*
|
|
156
|
+
* Trade-off: remote creates/deletes that landed only in S3 during the burst
|
|
157
|
+
* and were never journaled stay invisible until the next full pass.
|
|
158
|
+
* Absence-is-deletion still uses this pass's exact LIST denominators, so a
|
|
159
|
+
* realtime-added journal row cannot be swept as a delete.
|
|
160
|
+
*/
|
|
161
|
+
function overlayPullJournalDelta(run) {
|
|
162
|
+
run.keyedJournal?.refresh();
|
|
163
|
+
const fresh = run.keyedJournal
|
|
164
|
+
? journalForTargetedPull(run.keyedJournal, run.listedPrefixSet)
|
|
165
|
+
: migrateToV2(readJournalForPass(run.journalSlug));
|
|
166
|
+
const live = journalRows(run.journal);
|
|
167
|
+
const incoming = journalRows(fresh);
|
|
168
|
+
for (const key of incoming.keys()) {
|
|
169
|
+
if (live.has(key))
|
|
170
|
+
continue;
|
|
171
|
+
const entry = incoming.get(key);
|
|
172
|
+
if (entry)
|
|
173
|
+
live.set(key, structuredClone(entry));
|
|
174
|
+
}
|
|
175
|
+
}
|
|
126
176
|
/**
|
|
127
177
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
128
178
|
* scope shrink that would delete more than this many CLEAN local files in one
|
|
@@ -194,6 +244,52 @@ const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
|
|
|
194
244
|
* resulting journal delta is durable before realtime can acquire the writer.
|
|
195
245
|
*/
|
|
196
246
|
export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
|
|
247
|
+
/**
|
|
248
|
+
* ListObjectsV2 page size (AWS MaxKeys default). Full-pull LIST yields the
|
|
249
|
+
* root lock at each page boundary so queued realtime work is not stuck behind
|
|
250
|
+
* a 30-minute personal-vault spool.
|
|
251
|
+
*/
|
|
252
|
+
export const FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE = 1000;
|
|
253
|
+
const cooperativeRowVisits = {
|
|
254
|
+
enabled: false,
|
|
255
|
+
list: new Map(),
|
|
256
|
+
plan: new Map(),
|
|
257
|
+
};
|
|
258
|
+
let cooperativePhaseForTest = null;
|
|
259
|
+
function noteCooperativeRowVisit(phase, remoteKey) {
|
|
260
|
+
if (!cooperativeRowVisits.enabled)
|
|
261
|
+
return;
|
|
262
|
+
const bucket = cooperativeRowVisits[phase];
|
|
263
|
+
bucket.set(remoteKey, (bucket.get(remoteKey) ?? 0) + 1);
|
|
264
|
+
}
|
|
265
|
+
function enableCooperativeRowVisits() {
|
|
266
|
+
cooperativeRowVisits.enabled = true;
|
|
267
|
+
cooperativeRowVisits.list = new Map();
|
|
268
|
+
cooperativeRowVisits.plan = new Map();
|
|
269
|
+
cooperativePhaseForTest = null;
|
|
270
|
+
}
|
|
271
|
+
function disableCooperativeRowVisits() {
|
|
272
|
+
cooperativeRowVisits.enabled = false;
|
|
273
|
+
cooperativeRowVisits.list = new Map();
|
|
274
|
+
cooperativeRowVisits.plan = new Map();
|
|
275
|
+
cooperativePhaseForTest = null;
|
|
276
|
+
}
|
|
277
|
+
function snapshotCooperativeRowVisits() {
|
|
278
|
+
return {
|
|
279
|
+
list: new Map(cooperativeRowVisits.list),
|
|
280
|
+
plan: new Map(cooperativeRowVisits.plan),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Test-only export. Kept under `_testing` so the public surface stays `sync()`.
|
|
285
|
+
* Do NOT import from `_testing` outside of tests in this package.
|
|
286
|
+
*/
|
|
287
|
+
export const _testing = {
|
|
288
|
+
enableCooperativeRowVisits,
|
|
289
|
+
disableCooperativeRowVisits,
|
|
290
|
+
cooperativeRowVisits: snapshotCooperativeRowVisits,
|
|
291
|
+
cooperativePhase: () => cooperativePhaseForTest,
|
|
292
|
+
};
|
|
197
293
|
/**
|
|
198
294
|
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
199
295
|
* an oversized batch wholesale, so the client MUST split a large report into
|
|
@@ -333,53 +429,40 @@ async function syncWithOperationLockHeld(options) {
|
|
|
333
429
|
catch (err) {
|
|
334
430
|
console.warn("hq-root-recovery: local HQ recovery failed:", err);
|
|
335
431
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
}
|
|
371
|
-
catch (error) {
|
|
372
|
-
if (error instanceof CooperativePullHandoff) {
|
|
373
|
-
carried = error.completed;
|
|
374
|
-
continue;
|
|
375
|
-
}
|
|
376
|
-
throw error;
|
|
377
|
-
}
|
|
378
|
-
finally {
|
|
379
|
-
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
380
|
-
// The one bounded hot lease remains available only for its same vault.
|
|
381
|
-
run.releaseKeyedJournal?.();
|
|
382
|
-
}
|
|
432
|
+
const run = await buildPullContext(options);
|
|
433
|
+
try {
|
|
434
|
+
const plan = await planPull(run);
|
|
435
|
+
emitPullPlan(run.emit, plan);
|
|
436
|
+
reportInvalidRemoteKeys(run, plan);
|
|
437
|
+
const scopeRun = run.keyedJournal
|
|
438
|
+
? emptyTargetedScopeRun(run)
|
|
439
|
+
: executeScopeShrink(run, planScopeShrink(run));
|
|
440
|
+
const counters = createPullCounters();
|
|
441
|
+
// A streaming all-mode planner retains only executor-relevant items. Its
|
|
442
|
+
// ordinary skips have already been counted while the page was live.
|
|
443
|
+
counters.filesSkipped = plan.preplannedFilesSkipped;
|
|
444
|
+
counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
|
|
445
|
+
const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
|
|
446
|
+
const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
|
|
447
|
+
if (conflictRun.abortResult) {
|
|
448
|
+
return conflictRun.abortResult;
|
|
449
|
+
}
|
|
450
|
+
await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
|
|
451
|
+
// The following added-by and tombstone phases may still make authenticated
|
|
452
|
+
// vault calls. Make a completed sub-threshold download tail durable before
|
|
453
|
+
// they begin, so their failure cannot turn a successful bulk transfer into
|
|
454
|
+
// a full replay on the next pass.
|
|
455
|
+
flushDownloadedJournalTail(run);
|
|
456
|
+
await emitAndReportNewFiles(run, plan);
|
|
457
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
458
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
459
|
+
emitScopeMaterializationGap(run, plan);
|
|
460
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
461
|
+
}
|
|
462
|
+
finally {
|
|
463
|
+
// Release a pass-local lease even on conflict, transfer, or auth failure.
|
|
464
|
+
// The one bounded hot lease remains available only for its same vault.
|
|
465
|
+
run.releaseKeyedJournal?.();
|
|
383
466
|
}
|
|
384
467
|
}
|
|
385
468
|
async function buildPullContext(options) {
|
|
@@ -469,8 +552,11 @@ async function buildPullContext(options) {
|
|
|
469
552
|
: undefined;
|
|
470
553
|
const keyedJournal = keyedJournalLease?.store;
|
|
471
554
|
let remoteFiles;
|
|
555
|
+
const deleteDenominatorKeys = options.cooperativeCheckpoint
|
|
556
|
+
? new Set(journalRows(migrateToV2(readJournalForPass(journalSlug))).keys())
|
|
557
|
+
: undefined;
|
|
472
558
|
try {
|
|
473
|
-
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
|
|
559
|
+
remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0, options.cooperativeCheckpoint);
|
|
474
560
|
const fileTombstones = options.fileTombstones ??
|
|
475
561
|
await fetchFileTombstones(vaultConfig, options.personalMode === true
|
|
476
562
|
? { personal: true }
|
|
@@ -513,6 +599,7 @@ async function buildPullContext(options) {
|
|
|
513
599
|
currentExcludeSet,
|
|
514
600
|
fileTombstones,
|
|
515
601
|
downloadsSinceJournalCheckpoint: 0,
|
|
602
|
+
...(deleteDenominatorKeys ? { deleteDenominatorKeys } : {}),
|
|
516
603
|
};
|
|
517
604
|
}
|
|
518
605
|
catch (error) {
|
|
@@ -527,18 +614,36 @@ async function buildPullContext(options) {
|
|
|
527
614
|
function isStreamingRemoteFileSource(source) {
|
|
528
615
|
return !Array.isArray(source);
|
|
529
616
|
}
|
|
530
|
-
function
|
|
531
|
-
|
|
617
|
+
async function* streamRemoteFilesWithListCheckpoints(ctx, cooperativeCheckpoint) {
|
|
618
|
+
cooperativePhaseForTest = "list";
|
|
619
|
+
const work = { rowsSinceYield: 0 };
|
|
620
|
+
let listed = 0;
|
|
621
|
+
for await (const file of streamRemoteFiles(ctx)) {
|
|
622
|
+
listed++;
|
|
623
|
+
noteCooperativeRowVisit("list", file.key);
|
|
624
|
+
yield file;
|
|
625
|
+
if (listed % FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE === 0) {
|
|
626
|
+
await yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, true, listed);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
function allModeRemoteFileStream(ctx, cooperativeCheckpoint) {
|
|
631
|
+
if (!cooperativeCheckpoint)
|
|
632
|
+
return spoolRemoteFiles(streamRemoteFiles(ctx));
|
|
633
|
+
return spoolRemoteFiles(streamRemoteFilesWithListCheckpoints(ctx, cooperativeCheckpoint));
|
|
532
634
|
}
|
|
533
|
-
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
|
|
635
|
+
async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull, cooperativeCheckpoint) {
|
|
534
636
|
const entries = toScopePrefixEntries(listedPrefixSet);
|
|
535
|
-
if (entries.some((e) => e.prefix === ""))
|
|
536
|
-
return allModeRemoteFileStream(ctx);
|
|
637
|
+
if (entries.some((e) => e.prefix === "")) {
|
|
638
|
+
return allModeRemoteFileStream(ctx, cooperativeCheckpoint);
|
|
639
|
+
}
|
|
537
640
|
if (entries.length === 0) {
|
|
538
641
|
// `all` mode always carries `[""]`, so an empty set only occurs for a
|
|
539
642
|
// scoped mode (empty grants, or a targeted intersection that excluded
|
|
540
643
|
// everything). Defensive `all` branch: never silently narrow a full pull.
|
|
541
|
-
return syncMode === "all"
|
|
644
|
+
return syncMode === "all"
|
|
645
|
+
? allModeRemoteFileStream(ctx, cooperativeCheckpoint)
|
|
646
|
+
: [];
|
|
542
647
|
}
|
|
543
648
|
return listRemoteForScope({
|
|
544
649
|
ctx,
|
|
@@ -560,8 +665,9 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
|
|
|
560
665
|
});
|
|
561
666
|
}
|
|
562
667
|
async function planPull(run) {
|
|
668
|
+
cooperativePhaseForTest = "plan";
|
|
563
669
|
try {
|
|
564
|
-
return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
|
|
670
|
+
return await computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount, run.options.cooperativeCheckpoint, () => overlayPullJournalDelta(run), run.deleteDenominatorKeys);
|
|
565
671
|
}
|
|
566
672
|
finally {
|
|
567
673
|
if (isStreamingRemoteFileSource(run.remoteFiles))
|
|
@@ -683,50 +789,8 @@ function createPullCounters() {
|
|
|
683
789
|
hasRetryableOutcome: false,
|
|
684
790
|
};
|
|
685
791
|
}
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
if (!carried)
|
|
689
|
-
return;
|
|
690
|
-
counters.filesDownloaded += carried.filesDownloaded;
|
|
691
|
-
counters.bytesDownloaded += carried.bytesDownloaded;
|
|
692
|
-
counters.conflicts += carried.conflicts;
|
|
693
|
-
counters.filesTombstoned += carried.filesTombstoned;
|
|
694
|
-
counters.conflictPaths.push(...carried.conflictPaths);
|
|
695
|
-
counters.changedPaths.push(...carried.changedPaths);
|
|
696
|
-
counters.appliedUpserts.push(...carried.appliedUpserts);
|
|
697
|
-
if (carried.hasRetryableOutcome)
|
|
698
|
-
counters.hasRetryableOutcome = true;
|
|
699
|
-
}
|
|
700
|
-
/**
|
|
701
|
-
* Snapshot what survives the hand-off. The numeric and path axes already
|
|
702
|
-
* include everything carried into this attempt; `newFiles` is accumulated
|
|
703
|
-
* separately so it stays a record of pre-restart downloads only, and cannot
|
|
704
|
-
* double up with the current plan's own new files.
|
|
705
|
-
*/
|
|
706
|
-
function carryPullProgress(previous, counters) {
|
|
707
|
-
return {
|
|
708
|
-
filesDownloaded: counters.filesDownloaded,
|
|
709
|
-
bytesDownloaded: counters.bytesDownloaded,
|
|
710
|
-
conflicts: counters.conflicts,
|
|
711
|
-
conflictPaths: [...counters.conflictPaths],
|
|
712
|
-
filesTombstoned: counters.filesTombstoned,
|
|
713
|
-
changedPaths: [...counters.changedPaths],
|
|
714
|
-
appliedUpserts: [...counters.appliedUpserts],
|
|
715
|
-
newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
|
|
716
|
-
hasRetryableOutcome: counters.hasRetryableOutcome,
|
|
717
|
-
};
|
|
718
|
-
}
|
|
719
|
-
/**
|
|
720
|
-
* Every new file this pull is responsible for reporting: the ones a previous
|
|
721
|
-
* attempt downloaded before handing off, then this plan's own. A carried path
|
|
722
|
-
* that the fresh plan also lists is reported once.
|
|
723
|
-
*/
|
|
724
|
-
function reportableNewFiles(run, plan) {
|
|
725
|
-
const carried = run.carriedProgress?.newFiles;
|
|
726
|
-
if (carried === undefined || carried.length === 0)
|
|
727
|
-
return plan.newFiles;
|
|
728
|
-
const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
|
|
729
|
-
return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
|
|
792
|
+
function reportableNewFiles(_run, plan) {
|
|
793
|
+
return plan.newFiles;
|
|
730
794
|
}
|
|
731
795
|
function planScopeShrink(run) {
|
|
732
796
|
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
@@ -1323,20 +1387,6 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
|
|
|
1323
1387
|
discardProbe();
|
|
1324
1388
|
}
|
|
1325
1389
|
}
|
|
1326
|
-
/**
|
|
1327
|
-
* The old plan may have tombstone evidence that predates a realtime hand-off.
|
|
1328
|
-
* Rebuilding from disk after a hand-off makes the continuation observe the
|
|
1329
|
-
* realtime journal and a fresh LIST before it can perform another mutation.
|
|
1330
|
-
*/
|
|
1331
|
-
class CooperativePullHandoff extends Error {
|
|
1332
|
-
/** Completed work the restarted pull must keep reporting. */
|
|
1333
|
-
completed;
|
|
1334
|
-
constructor(completed) {
|
|
1335
|
-
super("full pull handed off to realtime work");
|
|
1336
|
-
this.name = "CooperativePullHandoff";
|
|
1337
|
-
this.completed = completed;
|
|
1338
|
-
}
|
|
1339
|
-
}
|
|
1340
1390
|
async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
|
|
1341
1391
|
const queue = [...downloadItems];
|
|
1342
1392
|
const inFlight = new Set();
|
|
@@ -1399,6 +1449,8 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1399
1449
|
throwDownloadWorkerErrors(run, workerErrors);
|
|
1400
1450
|
return;
|
|
1401
1451
|
}
|
|
1452
|
+
cooperativePhaseForTest = "download";
|
|
1453
|
+
const work = { rowsSinceYield: 0 };
|
|
1402
1454
|
for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
|
|
1403
1455
|
const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
|
|
1404
1456
|
const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
|
|
@@ -1408,8 +1460,8 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
|
|
|
1408
1460
|
flushDownloadedJournalTail(run);
|
|
1409
1461
|
if (workerErrors.length > 0)
|
|
1410
1462
|
throwDownloadWorkerErrors(run, workerErrors);
|
|
1411
|
-
if (await run.options.cooperativeCheckpoint
|
|
1412
|
-
|
|
1463
|
+
if (await yieldToCooperativeCheckpoint(work, run.options.cooperativeCheckpoint, true, start + chunk.length)) {
|
|
1464
|
+
overlayPullJournalDelta(run);
|
|
1413
1465
|
}
|
|
1414
1466
|
}
|
|
1415
1467
|
}
|
|
@@ -1627,9 +1679,6 @@ function persistPullJournal(run) {
|
|
|
1627
1679
|
}
|
|
1628
1680
|
async function emitAndReportNewFiles(run, plan) {
|
|
1629
1681
|
const enrichedNewFiles = [];
|
|
1630
|
-
// New files a cooperative hand-off already downloaded are reported here too:
|
|
1631
|
-
// the restarted plan cannot see them as new, so this is their only
|
|
1632
|
-
// notification.
|
|
1633
1682
|
const newFiles = reportableNewFiles(run, plan);
|
|
1634
1683
|
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
1635
1684
|
// created-by HEADs below reuse the cache instead of each minting its own
|
|
@@ -2104,7 +2153,7 @@ export function win32SymlinkFlavorBroken(localPath, win32 = process.platform ===
|
|
|
2104
2153
|
return true; // target exists but the link cannot traverse: wrong flavor
|
|
2105
2154
|
}
|
|
2106
2155
|
}
|
|
2107
|
-
function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
|
|
2156
|
+
async function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
|
|
2108
2157
|
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
2109
2158
|
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
2110
2159
|
// the scope filter below becomes a no-op and legacy behavior is preserved.
|
|
@@ -2125,7 +2174,7 @@ fileTombstones = new Map(),
|
|
|
2125
2174
|
excludePrefixes = [],
|
|
2126
2175
|
// The runner resolves this once and shares it with the push leg. Direct
|
|
2127
2176
|
// callers preserve the pre-policy, watcher-intent-only behavior.
|
|
2128
|
-
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
|
|
2177
|
+
propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size, cooperativeCheckpoint, onCooperativeHandoff, deleteDenominatorKeys) {
|
|
2129
2178
|
const items = [];
|
|
2130
2179
|
const streaming = isStreamingRemoteFileSource(remoteFiles);
|
|
2131
2180
|
const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
|
|
@@ -2184,14 +2233,18 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2184
2233
|
// `journaledKeyByCanonical` above deliberately retains only the preferred
|
|
2185
2234
|
// representative for classification.
|
|
2186
2235
|
const journaledSkillKeysByCanonical = new Map();
|
|
2236
|
+
const presenceKeys = deleteDenominatorKeys ?? journalRows(journal).keys();
|
|
2237
|
+
for (const journaledKey of presenceKeys) {
|
|
2238
|
+
if (isCoveredByAny(toPosixKey(journaledKey), prefixSet)) {
|
|
2239
|
+
unseenJournalKeys.add(journaledKey);
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2187
2242
|
for (const journaledKey of journalRows(journal).keys()) {
|
|
2188
2243
|
onHeartbeat?.();
|
|
2189
2244
|
const posixJournaledKey = toPosixKey(journaledKey);
|
|
2190
2245
|
// A scoped LIST cannot prove anything about keys outside its prefixes.
|
|
2191
2246
|
// Do not copy the entire vault's journal into its presence bookkeeping.
|
|
2192
2247
|
const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
|
|
2193
|
-
if (inListedScope)
|
|
2194
|
-
unseenJournalKeys.add(journaledKey);
|
|
2195
2248
|
if (inListedScope && posixJournaledKey !== journaledKey) {
|
|
2196
2249
|
const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
|
|
2197
2250
|
if (legacyKeys === undefined) {
|
|
@@ -2235,8 +2288,18 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2235
2288
|
: collectRemoteAncestors(remoteFiles);
|
|
2236
2289
|
// Both sources replay synchronously. for-await on a scoped array introduces
|
|
2237
2290
|
// promise/microtask work per object even though no I/O is being awaited.
|
|
2291
|
+
const cooperativeWork = { rowsSinceYield: 0 };
|
|
2292
|
+
let plannedRows = 0;
|
|
2238
2293
|
for (const remoteFile of listedFiles) {
|
|
2239
2294
|
onHeartbeat?.();
|
|
2295
|
+
noteCooperativeRowVisit("plan", remoteFile.key);
|
|
2296
|
+
plannedRows++;
|
|
2297
|
+
if (cooperativeCheckpoint &&
|
|
2298
|
+
plannedRows % FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE === 0) {
|
|
2299
|
+
if (await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, plannedRows)) {
|
|
2300
|
+
onCooperativeHandoff?.();
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2240
2303
|
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
2241
2304
|
// S3 object keys are POSIX. Match the old delete-sweep membership check:
|
|
2242
2305
|
// a legacy backslash journal entry counts as present only when its POSIX
|
|
@@ -2847,6 +2910,13 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2847
2910
|
localSnapshot: plannedLocalSnapshot,
|
|
2848
2911
|
});
|
|
2849
2912
|
}
|
|
2913
|
+
if (cooperativeCheckpoint &&
|
|
2914
|
+
plannedRows > FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE &&
|
|
2915
|
+
plannedRows % FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE !== 0) {
|
|
2916
|
+
if (await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, plannedRows)) {
|
|
2917
|
+
onCooperativeHandoff?.();
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2850
2920
|
// ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
|
|
2851
2921
|
// Below the breaker → stamp a persisting `local-delete` tombstone (the
|
|
2852
2922
|
// suppression above keeps it from being re-pulled; the push leg propagates the
|