@indigoai-us/hq-cloud 6.16.67 → 6.16.68-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -585,10 +585,10 @@ export interface SyncOptions {
585
585
  */
586
586
  onHeartbeat?: () => void;
587
587
  /**
588
- * Watch-runner hand-off seam for an unscoped full pull. It is invoked only
589
- * after one drained download chunk has made its journal changes durable.
590
- * Returning true means realtime ran; this pull discards its stale plan and
591
- * resumes from a fresh durable journal snapshot.
588
+ * Watch-runner hand-off seam for an unscoped full pull. Invoked at LIST
589
+ * page boundaries, every 500 planned rows, and after each drained download
590
+ * chunk. Returning true means realtime ran; this pull overlays the journal
591
+ * delta and continues the same LIST/plan/queue instead of re-LISTing S3.
592
592
  */
593
593
  cooperativeCheckpoint?: () => Promise<void | boolean>;
594
594
  /**
@@ -848,6 +848,28 @@ export declare function targetedJournalSessionCensusSizes(): Record<string, numb
848
848
  * resulting journal delta is durable before realtime can acquire the writer.
849
849
  */
850
850
  export declare const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
851
+ /**
852
+ * ListObjectsV2 page size (AWS MaxKeys default). Full-pull LIST yields the
853
+ * root lock at each page boundary so queued realtime work is not stuck behind
854
+ * a 30-minute personal-vault spool.
855
+ */
856
+ export declare const FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE = 1000;
857
+ declare function enableCooperativeRowVisits(): void;
858
+ declare function disableCooperativeRowVisits(): void;
859
+ declare function snapshotCooperativeRowVisits(): {
860
+ list: Map<string, number>;
861
+ plan: Map<string, number>;
862
+ };
863
+ /**
864
+ * Test-only export. Kept under `_testing` so the public surface stays `sync()`.
865
+ * Do NOT import from `_testing` outside of tests in this package.
866
+ */
867
+ export declare const _testing: {
868
+ enableCooperativeRowVisits: typeof enableCooperativeRowVisits;
869
+ disableCooperativeRowVisits: typeof disableCooperativeRowVisits;
870
+ cooperativeRowVisits: typeof snapshotCooperativeRowVisits;
871
+ cooperativePhase: () => "list" | "plan" | "download" | null;
872
+ };
851
873
  /**
852
874
  * Best-effort report of the files that were new to this drive during the sync,
853
875
  * so the HQ Sync app can show a persistent cross-session "new files" history.
@@ -882,4 +904,5 @@ export declare function sync(options: SyncOptions): Promise<SyncResult>;
882
904
  * false off win32.
883
905
  */
884
906
  export declare function win32SymlinkFlavorBroken(localPath: string, win32?: boolean): boolean;
907
+ export {};
885
908
  //# sourceMappingURL=sync.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAUhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAgEhC,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAML,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAW5F,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,cAAc,EACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,kBAAkB,EAClB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAA4B,KAAK,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEtG,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,+CAAgD,CAAC;AAEzF;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,kBAAkB,CAAC;AAElE,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAI3E;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,KAAK,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,KAAK,CAAC,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;CACnJ;AA+CD;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,CAEjH;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,kBAAkB,EAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC,EACjE,YAAY,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,mBAAmB,CAAC,GACjE,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;IAC/B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,GACD;IACE;;;;;;OAMG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,sBAAsB,GAAG,0BAA0B,CAAC;CAC/D,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,mCAAmC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EACF,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GACD;IACE;;;;;;;OAOG;IACH,IAAI,EAAE,sCAAsC,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GACD;IACE;;;;;;;;;OASG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EACF,kBAAkB,GAClB,iBAAiB,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,2BAA2B,GAC3B,4BAA4B,GAC5B,6BAA6B,GAC7B,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;;;;;;;;;;OAiBG;IACH,IAAI,EAAE,qBAAqB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;IACtD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACrC;;;;;;;;;;;;;;;;;;OAkBG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;OAIG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;CAC3C;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAmHD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAM5C;AAwDD,2EAA2E;AAC3E,wBAAgB,iCAAiC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAK1E;AAOD;;;GAGG;AACH,eAAO,MAAM,2CAA2C,MAAM,CAAC;AAY/D;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,kBAAkB,EAC/B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,EACrE,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,GACvC,OAAO,CAAC,IAAI,CAAC,CA6Ef;AAoCD;;GAEG;AACH,wBAAsB,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAOpE;AAsuED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,OAAsC,GAC5C,OAAO,CAoBT"}
1
+ {"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAEV,kBAAkB,EAEnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAA+B,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAUhF,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,wBAAwB,CAAC;AAgEhC,OAAO,EAML,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAI/B,OAAO,EAML,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,KAAK,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAW5F,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,KAAK,cAAc,EACpB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,kBAAkB,EAClB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAA4B,KAAK,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAEtG,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,+CAAgD,CAAC;AAEzF;;;;GAIG;AACH,eAAO,MAAM,kCAAkC,kBAAkB,CAAC;AAElE,yFAAyF;AACzF,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAI3E;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,CAAC,KAAK,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,KAAK,CAAC,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,cAAc,CAAA;KAAE,CAAC,CAAC;CACnJ;AA+CD;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAWjB;AAED,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc,CAEjH;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,kBAAkB,EAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,GAAG,eAAe,GAAG,QAAQ,CAAC,EACjE,YAAY,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,OAAO,CAAC,mBAAmB,CAAC,GACjE,OAAO,CAAC,IAAI,CAAC,CAEf;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,iBAAiB,GACzB;IACE,IAAI,EAAE,MAAM,CAAC;IACb,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB,GACD;IACE,IAAI,EAAE,OAAO,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB,GACD;IACE,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,UAAU,EAAE,kBAAkB,CAAC;IAC/B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CAC7B,GACD;IACE;;;;;;OAMG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,sBAAsB,GAAG,0BAA0B,CAAC;CAC/D,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,mCAAmC,CAAC;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACvE,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EACF,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;OAeG;IACH,IAAI,EAAE,8BAA8B,CAAC;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9B,GACD;IACE;;;;;;;;;;;;;;;;;;;OAmBG;IACH,IAAI,EAAE,+BAA+B,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GACD;IACE;;;;;;;OAOG;IACH,IAAI,EAAE,sCAAsC,CAAC;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,gBAAgB,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;CACpB,GACD;IACE;;;;;;;;;OASG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE;;;;;;;;;;;OAWG;IACH,IAAI,EAAE,6BAA6B,CAAC;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE;;;;;;;;;;;;OAYG;IACH,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,yBAAyB,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GACD;IACE;;;;;;;;;;;;;;OAcG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,kEAAkE;IAClE,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EACF,kBAAkB,GAClB,iBAAiB,GACjB,gBAAgB,GAChB,qBAAqB,GACrB,2BAA2B,GAC3B,4BAA4B,GAC5B,6BAA6B,GAC7B,2BAA2B,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GACD;IACE;;;;;;;;;;;;;;;;OAgBG;IACH,IAAI,EAAE,wBAAwB,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;;;;;;;;;;;;;OAiBG;IACH,IAAI,EAAE,qBAAqB,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,2BAA2B,CAAC;IAClC,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEN,MAAM,WAAW,WAAW;IAC1B,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,2BAA2B;IAC3B,WAAW,EAAE,kBAAkB,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACtD;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;IACtD;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACvD;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,eAAe,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACrC;;;;;;;;;;;;;;;;;;OAkBG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;OAiBG;IACH,iBAAiB,CAAC,EAAE,OAAO,GAAG,cAAc,CAAC;IAC7C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC;;;;OAIG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;CAC3C;AAED,MAAM,WAAW,UAAU;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;OAIG;IACH,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;IACtB;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;OASG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB;;;OAGG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAsJD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,MAAM,CAM5C;AAwDD,2EAA2E;AAC3E,wBAAgB,iCAAiC,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAK1E;AAOD;;;GAGG;AACH,eAAO,MAAM,2CAA2C,MAAM,CAAC;AAC/D;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,OAAO,CAAC;AAmBxD,iBAAS,0BAA0B,IAAI,IAAI,CAK1C;AAED,iBAAS,2BAA2B,IAAI,IAAI,CAK3C;AAED,iBAAS,4BAA4B,IAAI;IACvC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1B,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3B,CAKA;AAED;;;GAGG;AACH,eAAO,MAAM,QAAQ;;;;4BAIG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI;CAC1D,CAAC;AAYF;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,sBAAsB,CAC1C,WAAW,EAAE,kBAAkB,EAC/B,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC,EACrE,eAAe,CAAC,EAAE,eAAe,GAAG,IAAI,GACvC,OAAO,CAAC,IAAI,CAAC,CA6Ef;AAoCD;;GAEG;AACH,wBAAsB,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAOpE;AA4sED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,GAAE,OAAsC,GAC5C,OAAO,CAoBT"}
package/dist/cli/sync.js CHANGED
@@ -123,6 +123,56 @@ export function createSyncMutationClient(options) {
123
123
  export async function applyRealtimeDeltaPage(engine, page, fetchContent) {
124
124
  await engine.applyPage(page, fetchContent);
125
125
  }
126
+ /**
127
+ * Let queued realtime take the root lock at a bounded LIST/plan/download
128
+ * unit, then resume the same spool and remaining queue. A true return means
129
+ * the scheduler handed off; the caller overlays the journal delta rather than
130
+ * throwing away the LIST snapshot.
131
+ */
132
+ async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force, resumedAt) {
133
+ work.rowsSinceYield += 1;
134
+ if (!force && work.rowsSinceYield < FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
135
+ return false;
136
+ }
137
+ work.rowsSinceYield = 0;
138
+ await new Promise((resolve) => setImmediate(resolve));
139
+ if (!cooperativeCheckpoint)
140
+ return false;
141
+ const handedOff = (await cooperativeCheckpoint()) === true;
142
+ if (handedOff) {
143
+ process.stderr.write(`${JSON.stringify({
144
+ type: "cooperative-handoff",
145
+ direction: "pull",
146
+ resumedAt,
147
+ })}\n`);
148
+ }
149
+ return handedOff;
150
+ }
151
+ /**
152
+ * Freshness after a cooperative hand-off is the journal dirty-log overlay, not
153
+ * a second S3 LIST. The spool exists so classification can replay one snapshot
154
+ * after the caller refreshes its journal (remote-file-spool.ts:29-33).
155
+ *
156
+ * Trade-off: remote creates/deletes that landed only in S3 during the burst
157
+ * and were never journaled stay invisible until the next full pass.
158
+ * Absence-is-deletion still uses this pass's exact LIST denominators, so a
159
+ * realtime-added journal row cannot be swept as a delete.
160
+ */
161
+ function overlayPullJournalDelta(run) {
162
+ run.keyedJournal?.refresh();
163
+ const fresh = run.keyedJournal
164
+ ? journalForTargetedPull(run.keyedJournal, run.listedPrefixSet)
165
+ : migrateToV2(readJournalForPass(run.journalSlug));
166
+ const live = journalRows(run.journal);
167
+ const incoming = journalRows(fresh);
168
+ for (const key of incoming.keys()) {
169
+ if (live.has(key))
170
+ continue;
171
+ const entry = incoming.get(key);
172
+ if (entry)
173
+ live.set(key, structuredClone(entry));
174
+ }
175
+ }
126
176
  /**
127
177
  * Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
128
178
  * scope shrink that would delete more than this many CLEAN local files in one
@@ -194,6 +244,52 @@ const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
194
244
  * resulting journal delta is durable before realtime can acquire the writer.
195
245
  */
196
246
  export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
247
+ /**
248
+ * ListObjectsV2 page size (AWS MaxKeys default). Full-pull LIST yields the
249
+ * root lock at each page boundary so queued realtime work is not stuck behind
250
+ * a 30-minute personal-vault spool.
251
+ */
252
+ export const FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE = 1000;
253
+ const cooperativeRowVisits = {
254
+ enabled: false,
255
+ list: new Map(),
256
+ plan: new Map(),
257
+ };
258
+ let cooperativePhaseForTest = null;
259
+ function noteCooperativeRowVisit(phase, remoteKey) {
260
+ if (!cooperativeRowVisits.enabled)
261
+ return;
262
+ const bucket = cooperativeRowVisits[phase];
263
+ bucket.set(remoteKey, (bucket.get(remoteKey) ?? 0) + 1);
264
+ }
265
+ function enableCooperativeRowVisits() {
266
+ cooperativeRowVisits.enabled = true;
267
+ cooperativeRowVisits.list = new Map();
268
+ cooperativeRowVisits.plan = new Map();
269
+ cooperativePhaseForTest = null;
270
+ }
271
+ function disableCooperativeRowVisits() {
272
+ cooperativeRowVisits.enabled = false;
273
+ cooperativeRowVisits.list = new Map();
274
+ cooperativeRowVisits.plan = new Map();
275
+ cooperativePhaseForTest = null;
276
+ }
277
+ function snapshotCooperativeRowVisits() {
278
+ return {
279
+ list: new Map(cooperativeRowVisits.list),
280
+ plan: new Map(cooperativeRowVisits.plan),
281
+ };
282
+ }
283
+ /**
284
+ * Test-only export. Kept under `_testing` so the public surface stays `sync()`.
285
+ * Do NOT import from `_testing` outside of tests in this package.
286
+ */
287
+ export const _testing = {
288
+ enableCooperativeRowVisits,
289
+ disableCooperativeRowVisits,
290
+ cooperativeRowVisits: snapshotCooperativeRowVisits,
291
+ cooperativePhase: () => cooperativePhaseForTest,
292
+ };
197
293
  /**
198
294
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
199
295
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -333,53 +429,40 @@ async function syncWithOperationLockHeld(options) {
333
429
  catch (err) {
334
430
  console.warn("hq-root-recovery: local HQ recovery failed:", err);
335
431
  }
336
- let carried;
337
- for (;;) {
338
- const run = await buildPullContext(options);
339
- if (carried)
340
- run.carriedProgress = carried;
341
- try {
342
- const plan = await planPull(run);
343
- emitPullPlan(run.emit, plan);
344
- reportInvalidRemoteKeys(run, plan);
345
- const scopeRun = run.keyedJournal
346
- ? emptyTargetedScopeRun(run)
347
- : executeScopeShrink(run, planScopeShrink(run));
348
- const counters = createPullCounters();
349
- // A streaming all-mode planner retains only executor-relevant items. Its
350
- // ordinary skips have already been counted while the page was live.
351
- counters.filesSkipped = plan.preplannedFilesSkipped;
352
- counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
353
- seedCarriedPullProgress(counters, carried);
354
- const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
355
- const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
356
- if (conflictRun.abortResult) {
357
- return conflictRun.abortResult;
358
- }
359
- await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
360
- // The following added-by and tombstone phases may still make authenticated
361
- // vault calls. Make a completed sub-threshold download tail durable before
362
- // they begin, so their failure cannot turn a successful bulk transfer into
363
- // a full replay on the next pass.
364
- flushDownloadedJournalTail(run);
365
- await emitAndReportNewFiles(run, plan);
366
- await verifyPlannedJournalTombstones(run, plan);
367
- executeJournalTombstoneDeletes(run, plan, counters);
368
- emitScopeMaterializationGap(run, plan);
369
- return finalizePullRun(run, plan, scopeRun, counters);
370
- }
371
- catch (error) {
372
- if (error instanceof CooperativePullHandoff) {
373
- carried = error.completed;
374
- continue;
375
- }
376
- throw error;
377
- }
378
- finally {
379
- // Release a pass-local lease even on conflict, transfer, or auth failure.
380
- // The one bounded hot lease remains available only for its same vault.
381
- run.releaseKeyedJournal?.();
382
- }
432
+ const run = await buildPullContext(options);
433
+ try {
434
+ const plan = await planPull(run);
435
+ emitPullPlan(run.emit, plan);
436
+ reportInvalidRemoteKeys(run, plan);
437
+ const scopeRun = run.keyedJournal
438
+ ? emptyTargetedScopeRun(run)
439
+ : executeScopeShrink(run, planScopeShrink(run));
440
+ const counters = createPullCounters();
441
+ // A streaming all-mode planner retains only executor-relevant items. Its
442
+ // ordinary skips have already been counted while the page was live.
443
+ counters.filesSkipped = plan.preplannedFilesSkipped;
444
+ counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
445
+ const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
446
+ const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
447
+ if (conflictRun.abortResult) {
448
+ return conflictRun.abortResult;
449
+ }
450
+ await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
451
+ // The following added-by and tombstone phases may still make authenticated
452
+ // vault calls. Make a completed sub-threshold download tail durable before
453
+ // they begin, so their failure cannot turn a successful bulk transfer into
454
+ // a full replay on the next pass.
455
+ flushDownloadedJournalTail(run);
456
+ await emitAndReportNewFiles(run, plan);
457
+ await verifyPlannedJournalTombstones(run, plan);
458
+ executeJournalTombstoneDeletes(run, plan, counters);
459
+ emitScopeMaterializationGap(run, plan);
460
+ return finalizePullRun(run, plan, scopeRun, counters);
461
+ }
462
+ finally {
463
+ // Release a pass-local lease even on conflict, transfer, or auth failure.
464
+ // The one bounded hot lease remains available only for its same vault.
465
+ run.releaseKeyedJournal?.();
383
466
  }
384
467
  }
385
468
  async function buildPullContext(options) {
@@ -469,8 +552,11 @@ async function buildPullContext(options) {
469
552
  : undefined;
470
553
  const keyedJournal = keyedJournalLease?.store;
471
554
  let remoteFiles;
555
+ const deleteDenominatorKeys = options.cooperativeCheckpoint
556
+ ? new Set(journalRows(migrateToV2(readJournalForPass(journalSlug))).keys())
557
+ : undefined;
472
558
  try {
473
- remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
559
+ remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0, options.cooperativeCheckpoint);
474
560
  const fileTombstones = options.fileTombstones ??
475
561
  await fetchFileTombstones(vaultConfig, options.personalMode === true
476
562
  ? { personal: true }
@@ -513,6 +599,7 @@ async function buildPullContext(options) {
513
599
  currentExcludeSet,
514
600
  fileTombstones,
515
601
  downloadsSinceJournalCheckpoint: 0,
602
+ ...(deleteDenominatorKeys ? { deleteDenominatorKeys } : {}),
516
603
  };
517
604
  }
518
605
  catch (error) {
@@ -527,18 +614,36 @@ async function buildPullContext(options) {
527
614
  function isStreamingRemoteFileSource(source) {
528
615
  return !Array.isArray(source);
529
616
  }
530
- function allModeRemoteFileStream(ctx) {
531
- return spoolRemoteFiles(streamRemoteFiles(ctx));
617
+ async function* streamRemoteFilesWithListCheckpoints(ctx, cooperativeCheckpoint) {
618
+ cooperativePhaseForTest = "list";
619
+ const work = { rowsSinceYield: 0 };
620
+ let listed = 0;
621
+ for await (const file of streamRemoteFiles(ctx)) {
622
+ listed++;
623
+ noteCooperativeRowVisit("list", file.key);
624
+ yield file;
625
+ if (listed % FULL_PULL_LIST_PAGE_CHECKPOINT_SIZE === 0) {
626
+ await yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, true, listed);
627
+ }
628
+ }
629
+ }
630
+ function allModeRemoteFileStream(ctx, cooperativeCheckpoint) {
631
+ if (!cooperativeCheckpoint)
632
+ return spoolRemoteFiles(streamRemoteFiles(ctx));
633
+ return spoolRemoteFiles(streamRemoteFilesWithListCheckpoints(ctx, cooperativeCheckpoint));
532
634
  }
533
- async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
635
+ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull, cooperativeCheckpoint) {
534
636
  const entries = toScopePrefixEntries(listedPrefixSet);
535
- if (entries.some((e) => e.prefix === ""))
536
- return allModeRemoteFileStream(ctx);
637
+ if (entries.some((e) => e.prefix === "")) {
638
+ return allModeRemoteFileStream(ctx, cooperativeCheckpoint);
639
+ }
537
640
  if (entries.length === 0) {
538
641
  // `all` mode always carries `[""]`, so an empty set only occurs for a
539
642
  // scoped mode (empty grants, or a targeted intersection that excluded
540
643
  // everything). Defensive `all` branch: never silently narrow a full pull.
541
- return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
644
+ return syncMode === "all"
645
+ ? allModeRemoteFileStream(ctx, cooperativeCheckpoint)
646
+ : [];
542
647
  }
543
648
  return listRemoteForScope({
544
649
  ctx,
@@ -560,8 +665,9 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
560
665
  });
561
666
  }
562
667
  async function planPull(run) {
668
+ cooperativePhaseForTest = "plan";
563
669
  try {
564
- return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
670
+ return await computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount, run.options.cooperativeCheckpoint, () => overlayPullJournalDelta(run), run.deleteDenominatorKeys);
565
671
  }
566
672
  finally {
567
673
  if (isStreamingRemoteFileSource(run.remoteFiles))
@@ -683,50 +789,8 @@ function createPullCounters() {
683
789
  hasRetryableOutcome: false,
684
790
  };
685
791
  }
686
- /** Fold a previous attempt's completed work into a restarted run's counters. */
687
- function seedCarriedPullProgress(counters, carried) {
688
- if (!carried)
689
- return;
690
- counters.filesDownloaded += carried.filesDownloaded;
691
- counters.bytesDownloaded += carried.bytesDownloaded;
692
- counters.conflicts += carried.conflicts;
693
- counters.filesTombstoned += carried.filesTombstoned;
694
- counters.conflictPaths.push(...carried.conflictPaths);
695
- counters.changedPaths.push(...carried.changedPaths);
696
- counters.appliedUpserts.push(...carried.appliedUpserts);
697
- if (carried.hasRetryableOutcome)
698
- counters.hasRetryableOutcome = true;
699
- }
700
- /**
701
- * Snapshot what survives the hand-off. The numeric and path axes already
702
- * include everything carried into this attempt; `newFiles` is accumulated
703
- * separately so it stays a record of pre-restart downloads only, and cannot
704
- * double up with the current plan's own new files.
705
- */
706
- function carryPullProgress(previous, counters) {
707
- return {
708
- filesDownloaded: counters.filesDownloaded,
709
- bytesDownloaded: counters.bytesDownloaded,
710
- conflicts: counters.conflicts,
711
- conflictPaths: [...counters.conflictPaths],
712
- filesTombstoned: counters.filesTombstoned,
713
- changedPaths: [...counters.changedPaths],
714
- appliedUpserts: [...counters.appliedUpserts],
715
- newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
716
- hasRetryableOutcome: counters.hasRetryableOutcome,
717
- };
718
- }
719
- /**
720
- * Every new file this pull is responsible for reporting: the ones a previous
721
- * attempt downloaded before handing off, then this plan's own. A carried path
722
- * that the fresh plan also lists is reported once.
723
- */
724
- function reportableNewFiles(run, plan) {
725
- const carried = run.carriedProgress?.newFiles;
726
- if (carried === undefined || carried.length === 0)
727
- return plan.newFiles;
728
- const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
729
- return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
792
+ function reportableNewFiles(_run, plan) {
793
+ return plan.newFiles;
730
794
  }
731
795
  function planScopeShrink(run) {
732
796
  const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
@@ -1323,20 +1387,6 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
1323
1387
  discardProbe();
1324
1388
  }
1325
1389
  }
1326
- /**
1327
- * The old plan may have tombstone evidence that predates a realtime hand-off.
1328
- * Rebuilding from disk after a hand-off makes the continuation observe the
1329
- * realtime journal and a fresh LIST before it can perform another mutation.
1330
- */
1331
- class CooperativePullHandoff extends Error {
1332
- /** Completed work the restarted pull must keep reporting. */
1333
- completed;
1334
- constructor(completed) {
1335
- super("full pull handed off to realtime work");
1336
- this.name = "CooperativePullHandoff";
1337
- this.completed = completed;
1338
- }
1339
- }
1340
1390
  async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
1341
1391
  const queue = [...downloadItems];
1342
1392
  const inFlight = new Set();
@@ -1399,6 +1449,8 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1399
1449
  throwDownloadWorkerErrors(run, workerErrors);
1400
1450
  return;
1401
1451
  }
1452
+ cooperativePhaseForTest = "download";
1453
+ const work = { rowsSinceYield: 0 };
1402
1454
  for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
1403
1455
  const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
1404
1456
  const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
@@ -1408,8 +1460,8 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
1408
1460
  flushDownloadedJournalTail(run);
1409
1461
  if (workerErrors.length > 0)
1410
1462
  throwDownloadWorkerErrors(run, workerErrors);
1411
- if (await run.options.cooperativeCheckpoint() === true) {
1412
- throw new CooperativePullHandoff(carryPullProgress(run.carriedProgress, counters));
1463
+ if (await yieldToCooperativeCheckpoint(work, run.options.cooperativeCheckpoint, true, start + chunk.length)) {
1464
+ overlayPullJournalDelta(run);
1413
1465
  }
1414
1466
  }
1415
1467
  }
@@ -1627,9 +1679,6 @@ function persistPullJournal(run) {
1627
1679
  }
1628
1680
  async function emitAndReportNewFiles(run, plan) {
1629
1681
  const enrichedNewFiles = [];
1630
- // New files a cooperative hand-off already downloaded are reported here too:
1631
- // the restarted plan cannot see them as new, so this is their only
1632
- // notification.
1633
1682
  const newFiles = reportableNewFiles(run, plan);
1634
1683
  // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
1635
1684
  // created-by HEADs below reuse the cache instead of each minting its own
@@ -2104,7 +2153,7 @@ export function win32SymlinkFlavorBroken(localPath, win32 = process.platform ===
2104
2153
  return true; // target exists but the link cannot traverse: wrong flavor
2105
2154
  }
2106
2155
  }
2107
- function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
2156
+ async function computePullPlan(remoteFiles, journal, companyRoot, shouldSync, personalMode, teamSyncedSlugs,
2108
2157
  // Coalesced, company-relative prefixes the pull is scoped to (US-005).
2109
2158
  // `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
2110
2159
  // the scope filter below becomes a no-op and legacy behavior is preserved.
@@ -2125,7 +2174,7 @@ fileTombstones = new Map(),
2125
2174
  excludePrefixes = [],
2126
2175
  // The runner resolves this once and shares it with the push leg. Direct
2127
2176
  // callers preserve the pre-policy, watcher-intent-only behavior.
2128
- propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
2177
+ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size, cooperativeCheckpoint, onCooperativeHandoff, deleteDenominatorKeys) {
2129
2178
  const items = [];
2130
2179
  const streaming = isStreamingRemoteFileSource(remoteFiles);
2131
2180
  const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
@@ -2184,14 +2233,18 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2184
2233
  // `journaledKeyByCanonical` above deliberately retains only the preferred
2185
2234
  // representative for classification.
2186
2235
  const journaledSkillKeysByCanonical = new Map();
2236
+ const presenceKeys = deleteDenominatorKeys ?? journalRows(journal).keys();
2237
+ for (const journaledKey of presenceKeys) {
2238
+ if (isCoveredByAny(toPosixKey(journaledKey), prefixSet)) {
2239
+ unseenJournalKeys.add(journaledKey);
2240
+ }
2241
+ }
2187
2242
  for (const journaledKey of journalRows(journal).keys()) {
2188
2243
  onHeartbeat?.();
2189
2244
  const posixJournaledKey = toPosixKey(journaledKey);
2190
2245
  // A scoped LIST cannot prove anything about keys outside its prefixes.
2191
2246
  // Do not copy the entire vault's journal into its presence bookkeeping.
2192
2247
  const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
2193
- if (inListedScope)
2194
- unseenJournalKeys.add(journaledKey);
2195
2248
  if (inListedScope && posixJournaledKey !== journaledKey) {
2196
2249
  const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
2197
2250
  if (legacyKeys === undefined) {
@@ -2235,8 +2288,18 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2235
2288
  : collectRemoteAncestors(remoteFiles);
2236
2289
  // Both sources replay synchronously. for-await on a scoped array introduces
2237
2290
  // promise/microtask work per object even though no I/O is being awaited.
2291
+ const cooperativeWork = { rowsSinceYield: 0 };
2292
+ let plannedRows = 0;
2238
2293
  for (const remoteFile of listedFiles) {
2239
2294
  onHeartbeat?.();
2295
+ noteCooperativeRowVisit("plan", remoteFile.key);
2296
+ plannedRows++;
2297
+ if (cooperativeCheckpoint &&
2298
+ plannedRows % FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE === 0) {
2299
+ if (await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, plannedRows)) {
2300
+ onCooperativeHandoff?.();
2301
+ }
2302
+ }
2240
2303
  const posixRemoteKey = toPosixKey(remoteFile.key);
2241
2304
  // S3 object keys are POSIX. Match the old delete-sweep membership check:
2242
2305
  // a legacy backslash journal entry counts as present only when its POSIX
@@ -2847,6 +2910,13 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
2847
2910
  localSnapshot: plannedLocalSnapshot,
2848
2911
  });
2849
2912
  }
2913
+ if (cooperativeCheckpoint &&
2914
+ plannedRows > FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE &&
2915
+ plannedRows % FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE !== 0) {
2916
+ if (await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, plannedRows)) {
2917
+ onCooperativeHandoff?.();
2918
+ }
2919
+ }
2850
2920
  // ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
2851
2921
  // Below the breaker → stamp a persisting `local-delete` tombstone (the
2852
2922
  // suppression above keeps it from being re-pulled; the push leg propagates the