@indigoai-us/hq-cloud 6.16.68 → 6.16.70-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +24 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.js +64 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +19 -0
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts +25 -2
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +171 -21
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +9 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +193 -1
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share.d.ts +18 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +143 -127
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts +181 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +98 -49
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +105 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/journal-decode-window.d.ts +45 -0
- package/dist/journal-decode-window.d.ts.map +1 -0
- package/dist/journal-decode-window.js +70 -0
- package/dist/journal-decode-window.js.map +1 -0
- package/dist/journal-decode-window.test.d.ts +2 -0
- package/dist/journal-decode-window.test.d.ts.map +1 -0
- package/dist/journal-decode-window.test.js +99 -0
- package/dist/journal-decode-window.test.js.map +1 -0
- package/dist/journal-row-store.d.ts +4 -0
- package/dist/journal-row-store.d.ts.map +1 -1
- package/dist/journal-row-store.js +67 -2
- package/dist/journal-row-store.js.map +1 -1
- package/dist/journal-streaming.heap.test.d.ts +2 -0
- package/dist/journal-streaming.heap.test.d.ts.map +1 -0
- package/dist/journal-streaming.heap.test.js +190 -0
- package/dist/journal-streaming.heap.test.js.map +1 -0
- package/dist/journal.d.ts +1 -11
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +4 -53
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +13 -7
- package/dist/journal.test.js.map +1 -1
- package/dist/lib/net-errors.d.ts +12 -0
- package/dist/lib/net-errors.d.ts.map +1 -1
- package/dist/lib/net-errors.js +22 -0
- package/dist/lib/net-errors.js.map +1 -1
- package/dist/lib/net-errors.test.js +35 -1
- package/dist/lib/net-errors.test.js.map +1 -1
- package/dist/lib/s3-content-checksum.d.ts +16 -0
- package/dist/lib/s3-content-checksum.d.ts.map +1 -0
- package/dist/lib/s3-content-checksum.js +91 -0
- package/dist/lib/s3-content-checksum.js.map +1 -0
- package/dist/lib/s3-content-checksum.test.d.ts +2 -0
- package/dist/lib/s3-content-checksum.test.d.ts.map +1 -0
- package/dist/lib/s3-content-checksum.test.js +26 -0
- package/dist/lib/s3-content-checksum.test.js.map +1 -0
- package/dist/object-io.d.ts +2 -0
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +37 -19
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +89 -2
- package/dist/object-io.test.js.map +1 -1
- package/dist/sync/candidate-uploader.d.ts.map +1 -1
- package/dist/sync/candidate-uploader.js +6 -1
- package/dist/sync/candidate-uploader.js.map +1 -1
- package/dist/sync/full-pass-caps.d.ts +113 -0
- package/dist/sync/full-pass-caps.d.ts.map +1 -0
- package/dist/sync/full-pass-caps.js +383 -0
- package/dist/sync/full-pass-caps.js.map +1 -0
- package/dist/sync/full-pass-caps.test.d.ts +2 -0
- package/dist/sync/full-pass-caps.test.d.ts.map +1 -0
- package/dist/sync/full-pass-caps.test.js +126 -0
- package/dist/sync/full-pass-caps.test.js.map +1 -0
- package/dist/sync/multipart-uploader.d.ts.map +1 -1
- package/dist/sync/multipart-uploader.js +2 -1
- package/dist/sync/multipart-uploader.js.map +1 -1
- package/dist/sync/realtime-telemetry.d.ts +5 -1
- package/dist/sync/realtime-telemetry.d.ts.map +1 -1
- package/dist/sync/realtime-telemetry.js +14 -2
- package/dist/sync/realtime-telemetry.js.map +1 -1
- package/dist/sync/realtime-telemetry.test.js +10 -0
- package/dist/sync/realtime-telemetry.test.js.map +1 -1
- package/package.json +2 -1
- package/dist/journal-row-cache.d.ts +0 -102
- package/dist/journal-row-cache.d.ts.map +0 -1
- package/dist/journal-row-cache.heap.test.d.ts +0 -2
- package/dist/journal-row-cache.heap.test.d.ts.map +0 -1
- package/dist/journal-row-cache.heap.test.js +0 -102
- package/dist/journal-row-cache.heap.test.js.map +0 -1
- package/dist/journal-row-cache.js +0 -549
- package/dist/journal-row-cache.js.map +0 -1
- package/dist/journal-row-cache.test.d.ts +0 -2
- package/dist/journal-row-cache.test.d.ts.map +0 -1
- package/dist/journal-row-cache.test.js +0 -155
- package/dist/journal-row-cache.test.js.map +0 -1
package/dist/cli/share.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Broadcasts local file(s) to the company's S3 vault bucket.
|
|
5
5
|
* Refuses to overwrite a newer remote version without prompting.
|
|
6
6
|
*/
|
|
7
|
-
import type { EntityContext, VaultServiceConfig } from "../types.js";
|
|
7
|
+
import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
|
|
8
8
|
import type { UploadAuthor } from "../s3.js";
|
|
9
9
|
import { journalFingerprintMatchesStat } from "../journal.js";
|
|
10
10
|
import type { ConflictStrategy } from "./conflict.js";
|
|
@@ -161,6 +161,8 @@ export declare const _testing: {
|
|
|
161
161
|
enableCooperativeRowVisits: typeof enableCooperativeRowVisits;
|
|
162
162
|
disableCooperativeRowVisits: typeof disableCooperativeRowVisits;
|
|
163
163
|
cooperativeRowVisits: typeof snapshotCooperativeRowVisits;
|
|
164
|
+
computePushPlan: typeof computePushPlan;
|
|
165
|
+
computePushPlanCooperatively: typeof computePushPlanCooperatively;
|
|
164
166
|
};
|
|
165
167
|
/**
|
|
166
168
|
* Stage-1 classification for a single local file in a push run. Pre-HEAD —
|
|
@@ -206,6 +208,14 @@ type PushPlanItem = {
|
|
|
206
208
|
size: number;
|
|
207
209
|
};
|
|
208
210
|
};
|
|
211
|
+
interface PushPlan {
|
|
212
|
+
items: PushPlanItem[];
|
|
213
|
+
filesToUpload: number;
|
|
214
|
+
bytesToUpload: number;
|
|
215
|
+
filesToSkip: number;
|
|
216
|
+
/** Skip-unchanged rows omitted from `items` so a full pass does not retain them. */
|
|
217
|
+
preplannedFilesSkipped: number;
|
|
218
|
+
}
|
|
209
219
|
/**
|
|
210
220
|
* Pure Stage-1 pass for push: walk the candidate file list, hash each one,
|
|
211
221
|
* apply the size-limit and skip-unchanged gates, and return a classified
|
|
@@ -223,6 +233,13 @@ interface StatSnapshot {
|
|
|
223
233
|
ctimeMs: number;
|
|
224
234
|
}
|
|
225
235
|
declare function statMatchesJournal(absolutePath: string, entry: Parameters<typeof journalFingerprintMatchesStat>[0]): boolean;
|
|
236
|
+
declare function computePushPlan(filesToShare: CollectedEntry[], journal: SyncJournal, skipUnchanged: boolean, onHeartbeat?: () => void): PushPlan;
|
|
237
|
+
/**
|
|
238
|
+
* Build a full-pass plan in bounded row batches. A scoped replan deliberately
|
|
239
|
+
* uses the synchronous single-row planner above, so it does not add an await
|
|
240
|
+
* boundary between upload admissions and collapse the transfer pool.
|
|
241
|
+
*/
|
|
242
|
+
declare function computePushPlanCooperatively(filesToShare: CollectedEntry[], journal: SyncJournal, skipUnchanged: boolean, onHeartbeat: (() => void) | undefined, cooperativeCheckpoint: (() => Promise<void | boolean>) | undefined): Promise<PushPlan>;
|
|
226
243
|
export interface ShareOptions {
|
|
227
244
|
/** Path(s) to share (files or directories) */
|
|
228
245
|
paths: string[];
|
package/dist/cli/share.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,
|
|
1
|
+
{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/cli/share.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAwBlF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,OAAO,EASL,6BAA6B,EAW9B,MAAM,eAAe,CAAC;AAsBvB,OAAO,KAAK,EAAwC,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC5F,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAOL,KAAK,iBAAiB,EAEvB,MAAM,iBAAiB,CAAC;AASzB,OAAO,EAML,KAAK,0BAA0B,EAChC,MAAM,oBAAoB,CAAC;AAQ5B,OAAO,EAA4B,KAAK,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AAiEtG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,eAAO,MAAM,sBAAsB,QACmD,CAAC;AAEvF;;;;;;;;;GASG;AACH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,kBAAkB,QAAqB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAElD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AAiBD,iBAAS,0BAA0B,IAAI,IAAI,CAI1C;AAED,iBAAS,2BAA2B,IAAI,IAAI,CAI3C;AAED,iBAAS,4BAA4B,IAAI;IACvC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3B,CAKA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;CAcpB,CAAC;AAEF;;;;;;;;;GASG;AACH,KAAK,YAAY,GACb;IACE,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IAOb,UAAU,CAAC,EAAE,YAAY,CAAC;CAC3B,GACD;IACE,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IAIrB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,CAAC,CAAC;CACT,GACD;IACE,MAAM,EAAE,qBAAqB,CAAC;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE,MAAM,EAAE,iBAAiB,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD;IACE,MAAM,EAAE,gBAAgB,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IAMrB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D,CAAC;AAEN,UAAU,QAAQ;IAChB,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;;GASG;AACH,oDAAoD;AACpD,UAAU,YAAY;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAsDD,iBAAS,kBAAkB,CACzB,YAAY,EAAE,MAAM,EACpB,KAAK,EAAE,UAAU,CAAC,OAAO,6BAA6B,CAAC,CAAC,CAAC,CAAC,GACzD,OAAO,CAaT;AA+BD,iBAAS,eAAe,CACtB,YAAY,EAAE,cAAc,EAAE,EAC9B,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,OAAO,EACtB,WAAW,CAAC,EAAE,MAAM,IAAI,GACvB,QAAQ,CA2KV;AAED;;;;GAIG;AACH,iBAAe,4BAA4B,CACzC,YAAY,EAAE,cAAc,EAAE,EAC9B,OAAO,EAAE,WAAW,EACpB,aAAa,EAAE,OAAO,EACtB,WAAW,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,EACrC,qBAAqB,EAAE,CAAC,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,GACjE,OAAO,CAAC,QAAQ,CAAC,CAmCnB;AA4DD,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wCAAwC;IACxC,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC;;;;;;;;;;;;;;;;;;OAkBG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,qBAAqB,CAAC;IAC7C,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC7C,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACtD;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;IACtD;;;;;;;;;OASG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAyCG;IACH,qBAAqB,CAAC,EAAE,gBAAgB,GAAG,YAAY,GAAG,KAAK,CAAC;IAChE;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B;;;OAGG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC/B;;;;;;;;;;;OAWG;IACH,cAAc,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACvD;;;OAGG;IACH,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C;;;;;;;;;;;;;;;;;;;OAmBG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IACzC;;;;;;;OAOG;IACH,qBAAqB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;CAC1C;AAED,MAAM,WAAW,+BAA+B;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,uEAAuE;IACvE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,qBAAqB,GAAG,CAClC,KAAK,EAAE,+BAA+B,KACnC,OAAO,CAAC,0BAA0B,CAAC,CAAC;AAEzC,qBAAa,6BAA8B,SAAQ,KAAK;aAC1B,KAAK,EAAE,MAAM,EAAE;gBAAf,KAAK,EAAE,MAAM,EAAE;CAO5C;AAED,8DAA8D;AAC9D,MAAM,MAAM,qBAAqB,GAAG,SAAS,GAAG,iBAAiB,GAAG,iBAAiB,CAAC;AAEtF;;;;;GAKG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;IAClD,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;gBAE5C,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC,EAAE,QAAQ,EAAE,MAAM;CAuBtF;AAED,0EAA0E;AAC1E,qBAAa,yBAA0B,SAAQ,KAAK;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;gBAEb,KAAK,EAAE,MAAM,EAAE;CAa5B;AA6BD,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;;OAOG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;;OASG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;;OAQG;IACH,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC;;;;;;;OAOG;IACH,0BAA0B,EAAE,MAAM,CAAC;IACnC;;;;;;;OAOG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;;OAKG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;;;;;;;;OAiBG;IACH,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B;;;;;;;OAOG;IACH,wBAAwB,EAAE,MAAM,EAAE,CAAC;IACnC;;;;OAIG;IACH,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,eAAe,EAAE,CAAC;IAChC,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,MAAM,wBAAwB,GAChC,YAAY,GACZ,gBAAgB,GAChB,gBAAgB,GAChB,iBAAiB,GACjB,uBAAuB,GACvB,gBAAgB,GAChB,mBAAmB,GACnB,iBAAiB,GACjB,gBAAgB,GAChB,gBAAgB,CAAC;AAErB,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,UAAU,GAAG,SAAS,CAAC;IAC/B,SAAS,EAAE,QAAQ,GAAG,WAAW,CAAC;IAClC,MAAM,CAAC,EAAE,wBAAwB,CAAC;CACnC;AAmDD;;;;;;GAMG;AACH,wBAAgB,8BAA8B,CAC5C,UAAU,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,EACzD,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAC3C,aAAa,CAAC,EAAE,MAAM,IAAI,GACzB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,CAW/C;AAED;;;;;;;;GAQG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAEvE;AA4GD,KAAK,cAAc,GAAG,OAAO,CAAC,YAAY,EAAE;IAAE,MAAM,EAAE,QAAQ,CAAA;CAAE,CAAC,CAAC;AAGlE;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,cAAc,GAAG,MAAM,CAO/D;AA2TD,8EAA8E;AAC9E,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAQlF;AA28ED;;;;GAIG;AACH,eAAO,MAAM,2CAA2C,MAAM,CAAC;AAoS/D;;;GAGG;AACH,iBAAS,oBAAoB,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI,CA8L5D;AAED;;;;;;;;;GASG;AACH,KAAK,cAAc,GACf;IACE,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,YAAY,CAAA;KAAE,CAAC;CAClD,GACD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,UAAU,YAAY;IACpB,iBAAiB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/E,eAAe,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,CAAC;CACvD;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,iBAAS,gBAAgB,CACvB,CAAC,EAAE,MAAM,EACT,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAsB,GAC1B,MAAM,CAuBR;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,iBAAS,qBAAqB,CAC5B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,gBAAgB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GACvC,OAAO,CAwBT;AA8HD;;;;;;;;;;;GAWG;AACH,iBAAe,YAAY,CACzB,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,OAAO,EAC/C,KAAK,GAAE,YAAiB,GACvB,OAAO,CAAC,cAAc,EAAE,CAAC,CAsG3B"}
|
package/dist/cli/share.js
CHANGED
|
@@ -14,9 +14,10 @@ import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.
|
|
|
14
14
|
import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
|
|
15
15
|
import * as crypto from "crypto";
|
|
16
16
|
import { RateLimitedError } from "../object-io.js";
|
|
17
|
-
import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
17
|
+
import { isUnderAnyRoot, readJournalForPass, readJournalScoped, writeJournal, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, } from "../journal.js";
|
|
18
18
|
import { createIgnoreFilter, hasControlCharacters, isExpectedIgnore, isWithinSizeLimit, } from "../ignore.js";
|
|
19
19
|
import { wrapFilterWithPersonalVaultDefaults, } from "../personal-vault-exclusions.js";
|
|
20
|
+
import { getActiveFullPassDecodeWindow, getActiveFullPassUnitSize, } from "../sync/full-pass-caps.js";
|
|
20
21
|
import { isGeneratedCoreMirrorKey, isPersonalVaultDeletionExcluded, isSessionLogDeletionKey, purgeExcludedPersonalVaultJournalEntries, } from "../personal-vault.js";
|
|
21
22
|
import { pickWinner, resolveConflict, usesVersionAwareWinner } from "./conflict.js";
|
|
22
23
|
import { fetchFileTombstones, } from "./tombstones.js";
|
|
@@ -260,6 +261,8 @@ export const _testing = {
|
|
|
260
261
|
enableCooperativeRowVisits,
|
|
261
262
|
disableCooperativeRowVisits,
|
|
262
263
|
cooperativeRowVisits: snapshotCooperativeRowVisits,
|
|
264
|
+
computePushPlan,
|
|
265
|
+
computePushPlanCooperatively,
|
|
263
266
|
};
|
|
264
267
|
/** Snapshot of `absolutePath`, or null when it is missing or not a plain file. */
|
|
265
268
|
function statSnapshot(absolutePath) {
|
|
@@ -333,7 +336,7 @@ function statMatchesJournal(absolutePath, entry) {
|
|
|
333
336
|
*/
|
|
334
337
|
async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force = false, resumedAt = 0) {
|
|
335
338
|
work.rowsSinceYield += 1;
|
|
336
|
-
if (!force && work.rowsSinceYield <
|
|
339
|
+
if (!force && work.rowsSinceYield < getActiveFullPassUnitSize())
|
|
337
340
|
return;
|
|
338
341
|
const completed = resumedAt > 0 ? resumedAt : work.rowsSinceYield;
|
|
339
342
|
work.rowsSinceYield = 0;
|
|
@@ -351,141 +354,150 @@ async function yieldToCooperativeCheckpoint(work, cooperativeCheckpoint, force =
|
|
|
351
354
|
function computePushPlan(filesToShare, journal, skipUnchanged, onHeartbeat) {
|
|
352
355
|
const items = [];
|
|
353
356
|
let omittedSkips = 0;
|
|
357
|
+
let plannedRows = 0;
|
|
354
358
|
for (const entry of filesToShare) {
|
|
355
359
|
onHeartbeat?.();
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
360
|
+
plannedRows += 1;
|
|
361
|
+
try {
|
|
362
|
+
const { absolutePath, relativePath } = entry;
|
|
363
|
+
// Server-generated paths are read-only to normal client sync. Classify
|
|
364
|
+
// them before hashing or skip-unchanged so local-only files and files with
|
|
365
|
+
// a divergent remote baseline both surface the same honest refusal. The
|
|
366
|
+
// board remains the documented push-side exception.
|
|
367
|
+
if (relativePath !== "board.json" && isCloudAuthoritative(relativePath)) {
|
|
368
|
+
items.push({ action: "refuse-server-owned", absolutePath, relativePath });
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
// Symlinks bypass the size-limit gate (the wire body is small,
|
|
372
|
+
// bounded by target length) and hash the target through the
|
|
373
|
+
// symlink-namespaced hash so a symlink to "real.md" can never
|
|
374
|
+
// collide with a regular file containing the bytes "real.md" in
|
|
375
|
+
// the skip-unchanged gate. The target is what we're actually
|
|
376
|
+
// uploading, so its hash is what should drive change detection —
|
|
377
|
+
// a target rewrite must re-fire an upload even when the link
|
|
378
|
+
// itself "looks" identical.
|
|
379
|
+
if (entry.kind === "symlink") {
|
|
380
|
+
const localHash = hashSymlinkTarget(entry.target);
|
|
381
|
+
if (skipUnchanged) {
|
|
382
|
+
const existing = skipFingerprint(journal, relativePath);
|
|
383
|
+
// `!localDiverges`: see the regular-file gate below — a divergent entry
|
|
384
|
+
// stores the current LOCAL hash, so hash equality here proves only that
|
|
385
|
+
// local has not moved since the pull-side hold, never that it matches remote.
|
|
386
|
+
if (existing && existing.hash === localHash && !existing.localDiverges) {
|
|
387
|
+
omittedSkips += 1;
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
items.push({
|
|
392
|
+
action: "upload",
|
|
393
|
+
kind: "symlink",
|
|
394
|
+
absolutePath,
|
|
395
|
+
relativePath,
|
|
396
|
+
target: entry.target,
|
|
397
|
+
localHash,
|
|
398
|
+
size: 0,
|
|
399
|
+
});
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
if (!isWithinSizeLimit(absolutePath)) {
|
|
403
|
+
items.push({ action: "skip-size-limit", absolutePath, relativePath });
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
// Stat fast path. Reading and hashing every regular file every pass costs
|
|
407
|
+
// one full read of the vault — hundreds of thousands of files on a real
|
|
408
|
+
// install — to answer a question stat can usually answer for free.
|
|
409
|
+
//
|
|
410
|
+
// The gate is size + mtime + ctime, and ctime is what makes it safe. mtime
|
|
411
|
+
// is forgeable: `cp -p`, `touch -r` and `rsync --times` put an old mtime
|
|
412
|
+
// back on new bytes, so a size+mtime check would skip such an edit forever
|
|
413
|
+
// — the failure this code previously hashed everything to avoid. ctime is
|
|
414
|
+
// the inode-change stamp and userspace cannot set it; the very utimes call
|
|
415
|
+
// that restores mtime moves ctime to now. A file that clears all three has
|
|
416
|
+
// not been written since we hashed it.
|
|
417
|
+
//
|
|
418
|
+
// Deliberately NOT skipped here:
|
|
419
|
+
// - entries with no recorded mtime/ctime (written before stat tracking,
|
|
420
|
+
// or by a downloader that did not stamp) — hashed once, then stamped
|
|
421
|
+
// by the restamp below so the next pass is free;
|
|
422
|
+
// - `localDiverges` entries — see the hash gate below for why equality
|
|
423
|
+
// proves nothing about them.
|
|
424
|
+
if (skipUnchanged && statMatchesJournal(absolutePath, skipFingerprint(journal, relativePath))) {
|
|
425
|
+
omittedSkips += 1;
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
// Snapshot stat either side of the read. If the two differ the file was
|
|
429
|
+
// rewritten while we hashed it, so `localHash` may describe bytes that are
|
|
430
|
+
// already gone — and nothing downstream may stamp a timestamp against it.
|
|
431
|
+
const statBeforeHash = statSnapshot(absolutePath);
|
|
432
|
+
let localHash;
|
|
433
|
+
let statAtHash;
|
|
434
|
+
if (entry.knownHash !== undefined &&
|
|
435
|
+
statBeforeHash !== null &&
|
|
436
|
+
sameStatSnapshot(entry.knownHash.stat, statBeforeHash) &&
|
|
437
|
+
!timestampsTooCoarseToTrust(statBeforeHash.mtimeMs, statBeforeHash.ctimeMs)) {
|
|
438
|
+
// The execution seam re-plans every upload candidate up to three times
|
|
439
|
+
// (admission, dispatch, after HEAD). Each re-plan hashed the body again,
|
|
440
|
+
// so a candidate that never changes — on a real install ~3.5k archived
|
|
441
|
+
// transcripts, 6.3 GB, skipped after HEAD — was read four times per pass.
|
|
442
|
+
// size + mtime + ctime unchanged since the earlier hash proves the bytes
|
|
443
|
+
// are the same (ctime cannot be set from userspace), so reuse it.
|
|
444
|
+
localHash = entry.knownHash.hash;
|
|
445
|
+
statAtHash = statBeforeHash;
|
|
446
|
+
}
|
|
447
|
+
else {
|
|
448
|
+
localHash = hashFile(absolutePath);
|
|
449
|
+
const statAfterHash = statSnapshot(absolutePath);
|
|
450
|
+
statAtHash = sameStatSnapshot(statBeforeHash, statAfterHash)
|
|
451
|
+
? statAfterHash ?? undefined
|
|
452
|
+
: undefined;
|
|
453
|
+
}
|
|
375
454
|
if (skipUnchanged) {
|
|
376
|
-
const existing =
|
|
377
|
-
//
|
|
378
|
-
//
|
|
379
|
-
//
|
|
455
|
+
const existing = journalRows(journal).get(relativePath);
|
|
456
|
+
// A `localDiverges` entry records the hash of the KEPT LOCAL body, so
|
|
457
|
+
// `existing.hash === localHash` is true for every such file by
|
|
458
|
+
// construction. Skipping on that equality asserts "local matches remote",
|
|
459
|
+
// which is precisely the claim the flag exists to deny — and it would
|
|
460
|
+
// strand the file: the runner's push leg always passes skipUnchanged,
|
|
461
|
+
// so the conflict path that could publish it never runs, and the pull leg
|
|
462
|
+
// sees an unchanged remote and re-arms the flag. Route these to the HEAD
|
|
463
|
+
// + conflict path instead and let the resolution decide.
|
|
380
464
|
if (existing && existing.hash === localHash && !existing.localDiverges) {
|
|
381
|
-
|
|
465
|
+
// The hash proved the bytes are unchanged — a no-op skip. Capture the
|
|
466
|
+
// current (mtimeMs, size) so the journal metadata stays accurate. Only
|
|
467
|
+
// attach when the stored stat actually differs (avoid a needless
|
|
468
|
+
// journal write on an already-current entry). Stat failure → no
|
|
469
|
+
// restamp; the skip still stands.
|
|
470
|
+
let restamp;
|
|
471
|
+
if (statAtHash &&
|
|
472
|
+
(existing.mtimeMs !== statAtHash.mtimeMs ||
|
|
473
|
+
existing.ctimeMs !== statAtHash.ctimeMs ||
|
|
474
|
+
existing.size !== statAtHash.size)) {
|
|
475
|
+
restamp = statAtHash;
|
|
476
|
+
}
|
|
477
|
+
if (restamp)
|
|
478
|
+
items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
|
|
479
|
+
else
|
|
480
|
+
omittedSkips += 1;
|
|
382
481
|
continue;
|
|
383
482
|
}
|
|
384
483
|
}
|
|
484
|
+
const size = fs.statSync(absolutePath).size;
|
|
385
485
|
items.push({
|
|
386
486
|
action: "upload",
|
|
387
|
-
kind: "
|
|
487
|
+
kind: "file",
|
|
388
488
|
absolutePath,
|
|
389
489
|
relativePath,
|
|
390
|
-
target: entry.target,
|
|
391
490
|
localHash,
|
|
392
|
-
size
|
|
491
|
+
size,
|
|
492
|
+
...(statAtHash ? { statAtHash } : {}),
|
|
393
493
|
});
|
|
394
|
-
continue;
|
|
395
|
-
}
|
|
396
|
-
if (!isWithinSizeLimit(absolutePath)) {
|
|
397
|
-
items.push({ action: "skip-size-limit", absolutePath, relativePath });
|
|
398
|
-
continue;
|
|
399
|
-
}
|
|
400
|
-
// Stat fast path. Reading and hashing every regular file every pass costs
|
|
401
|
-
// one full read of the vault — hundreds of thousands of files on a real
|
|
402
|
-
// install — to answer a question stat can usually answer for free.
|
|
403
|
-
//
|
|
404
|
-
// The gate is size + mtime + ctime, and ctime is what makes it safe. mtime
|
|
405
|
-
// is forgeable: `cp -p`, `touch -r` and `rsync --times` put an old mtime
|
|
406
|
-
// back on new bytes, so a size+mtime check would skip such an edit forever
|
|
407
|
-
// — the failure this code previously hashed everything to avoid. ctime is
|
|
408
|
-
// the inode-change stamp and userspace cannot set it; the very utimes call
|
|
409
|
-
// that restores mtime moves ctime to now. A file that clears all three has
|
|
410
|
-
// not been written since we hashed it.
|
|
411
|
-
//
|
|
412
|
-
// Deliberately NOT skipped here:
|
|
413
|
-
// - entries with no recorded mtime/ctime (written before stat tracking,
|
|
414
|
-
// or by a downloader that did not stamp) — hashed once, then stamped
|
|
415
|
-
// by the restamp below so the next pass is free;
|
|
416
|
-
// - `localDiverges` entries — see the hash gate below for why equality
|
|
417
|
-
// proves nothing about them.
|
|
418
|
-
if (skipUnchanged && statMatchesJournal(absolutePath, skipFingerprint(journal, relativePath))) {
|
|
419
|
-
omittedSkips += 1;
|
|
420
|
-
continue;
|
|
421
494
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
const statBeforeHash = statSnapshot(absolutePath);
|
|
426
|
-
let localHash;
|
|
427
|
-
let statAtHash;
|
|
428
|
-
if (entry.knownHash !== undefined &&
|
|
429
|
-
statBeforeHash !== null &&
|
|
430
|
-
sameStatSnapshot(entry.knownHash.stat, statBeforeHash) &&
|
|
431
|
-
!timestampsTooCoarseToTrust(statBeforeHash.mtimeMs, statBeforeHash.ctimeMs)) {
|
|
432
|
-
// The execution seam re-plans every upload candidate up to three times
|
|
433
|
-
// (admission, dispatch, after HEAD). Each re-plan hashed the body again,
|
|
434
|
-
// so a candidate that never changes — on a real install ~3.5k archived
|
|
435
|
-
// transcripts, 6.3 GB, skipped after HEAD — was read four times per pass.
|
|
436
|
-
// size + mtime + ctime unchanged since the earlier hash proves the bytes
|
|
437
|
-
// are the same (ctime cannot be set from userspace), so reuse it.
|
|
438
|
-
localHash = entry.knownHash.hash;
|
|
439
|
-
statAtHash = statBeforeHash;
|
|
440
|
-
}
|
|
441
|
-
else {
|
|
442
|
-
localHash = hashFile(absolutePath);
|
|
443
|
-
const statAfterHash = statSnapshot(absolutePath);
|
|
444
|
-
statAtHash = sameStatSnapshot(statBeforeHash, statAfterHash)
|
|
445
|
-
? statAfterHash ?? undefined
|
|
446
|
-
: undefined;
|
|
447
|
-
}
|
|
448
|
-
if (skipUnchanged) {
|
|
449
|
-
const existing = journalRows(journal).get(relativePath);
|
|
450
|
-
// A `localDiverges` entry records the hash of the KEPT LOCAL body, so
|
|
451
|
-
// `existing.hash === localHash` is true for every such file by
|
|
452
|
-
// construction. Skipping on that equality asserts "local matches remote",
|
|
453
|
-
// which is precisely the claim the flag exists to deny — and it would
|
|
454
|
-
// strand the file: the runner's push leg always passes skipUnchanged,
|
|
455
|
-
// so the conflict path that could publish it never runs, and the pull leg
|
|
456
|
-
// sees an unchanged remote and re-arms the flag. Route these to the HEAD
|
|
457
|
-
// + conflict path instead and let the resolution decide.
|
|
458
|
-
if (existing && existing.hash === localHash && !existing.localDiverges) {
|
|
459
|
-
// The hash proved the bytes are unchanged — a no-op skip. Capture the
|
|
460
|
-
// current (mtimeMs, size) so the journal metadata stays accurate. Only
|
|
461
|
-
// attach when the stored stat actually differs (avoid a needless
|
|
462
|
-
// journal write on an already-current entry). Stat failure → no
|
|
463
|
-
// restamp; the skip still stands.
|
|
464
|
-
let restamp;
|
|
465
|
-
if (statAtHash &&
|
|
466
|
-
(existing.mtimeMs !== statAtHash.mtimeMs ||
|
|
467
|
-
existing.ctimeMs !== statAtHash.ctimeMs ||
|
|
468
|
-
existing.size !== statAtHash.size)) {
|
|
469
|
-
restamp = statAtHash;
|
|
470
|
-
}
|
|
471
|
-
if (restamp)
|
|
472
|
-
items.push({ action: "skip-unchanged", absolutePath, relativePath, restamp });
|
|
473
|
-
else
|
|
474
|
-
omittedSkips += 1;
|
|
475
|
-
continue;
|
|
476
|
-
}
|
|
495
|
+
finally {
|
|
496
|
+
if (plannedRows % getActiveFullPassDecodeWindow() === 0)
|
|
497
|
+
dropJournalDecodeWindow();
|
|
477
498
|
}
|
|
478
|
-
const size = fs.statSync(absolutePath).size;
|
|
479
|
-
items.push({
|
|
480
|
-
action: "upload",
|
|
481
|
-
kind: "file",
|
|
482
|
-
absolutePath,
|
|
483
|
-
relativePath,
|
|
484
|
-
localHash,
|
|
485
|
-
size,
|
|
486
|
-
...(statAtHash ? { statAtHash } : {}),
|
|
487
|
-
});
|
|
488
499
|
}
|
|
500
|
+
dropJournalDecodeWindow();
|
|
489
501
|
let filesToUpload = 0;
|
|
490
502
|
let bytesToUpload = 0;
|
|
491
503
|
let filesToSkip = 0;
|
|
@@ -513,8 +525,8 @@ async function computePushPlanCooperatively(filesToShare, journal, skipUnchanged
|
|
|
513
525
|
let filesToSkip = 0;
|
|
514
526
|
let preplannedFilesSkipped = 0;
|
|
515
527
|
const cooperativeWork = { rowsSinceYield: 0 };
|
|
516
|
-
for (let start = 0; start < filesToShare.length; start +=
|
|
517
|
-
const slice = filesToShare.slice(start, start +
|
|
528
|
+
for (let start = 0; start < filesToShare.length; start += getActiveFullPassUnitSize()) {
|
|
529
|
+
const slice = filesToShare.slice(start, start + getActiveFullPassUnitSize());
|
|
518
530
|
const plan = computePushPlan(slice, journal, skipUnchanged, onHeartbeat);
|
|
519
531
|
items.push(...plan.items);
|
|
520
532
|
filesToUpload += plan.filesToUpload;
|
|
@@ -525,6 +537,7 @@ async function computePushPlanCooperatively(filesToShare, journal, skipUnchanged
|
|
|
525
537
|
noteCooperativeRowVisit("plan", entry.relativePath);
|
|
526
538
|
}
|
|
527
539
|
await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, start + slice.length);
|
|
540
|
+
dropJournalDecodeWindow();
|
|
528
541
|
}
|
|
529
542
|
return { items, filesToUpload, bytesToUpload, filesToSkip, preplannedFilesSkipped };
|
|
530
543
|
}
|
|
@@ -2489,9 +2502,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2489
2502
|
!sessionAuthAborted &&
|
|
2490
2503
|
(!run.options.cooperativeCheckpoint ||
|
|
2491
2504
|
(admittedSinceCooperativeCheckpoint <
|
|
2492
|
-
|
|
2505
|
+
getActiveFullPassUnitSize() &&
|
|
2493
2506
|
skippedSinceCooperativeCheckpoint <
|
|
2494
|
-
|
|
2507
|
+
getActiveFullPassUnitSize())) &&
|
|
2495
2508
|
inFlight.size < TRANSFER_CONCURRENCY &&
|
|
2496
2509
|
(queue.length > 0 || remainingPreplannedSkips > 0)) {
|
|
2497
2510
|
if (remainingPreplannedSkips > 0) {
|
|
@@ -2560,9 +2573,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
|
|
|
2560
2573
|
!hadUploadFailure &&
|
|
2561
2574
|
(admittedSinceCooperativeCheckpoint > 0 || skippedSinceCooperativeCheckpoint > 0) &&
|
|
2562
2575
|
(admittedSinceCooperativeCheckpoint >=
|
|
2563
|
-
|
|
2576
|
+
getActiveFullPassUnitSize() ||
|
|
2564
2577
|
skippedSinceCooperativeCheckpoint >=
|
|
2565
|
-
|
|
2578
|
+
getActiveFullPassUnitSize() ||
|
|
2566
2579
|
(queue.length === 0 && remainingPreplannedSkips === 0))) {
|
|
2567
2580
|
// Every admitted upload has settled, so `processUploadItem` has
|
|
2568
2581
|
// finished its mutable journal updates. Persist that durable unit before
|
|
@@ -3513,6 +3526,9 @@ function defaultConsoleLogger(event) {
|
|
|
3513
3526
|
else if (event.type === "skip-rename-blocked") {
|
|
3514
3527
|
console.warn(` ! ${event.path} skipped — another program on this PC is holding the file open, so HQ could not replace it yet. ${event.remedy}`);
|
|
3515
3528
|
}
|
|
3529
|
+
else if (event.type === "skip-vanished-remote") {
|
|
3530
|
+
console.warn(` ! ${event.path} skipped — the vault object disappeared before it could be downloaded (${event.message})`);
|
|
3531
|
+
}
|
|
3516
3532
|
else if (event.type === "skip-archived-session-log") {
|
|
3517
3533
|
console.log(` - ${event.path} skipped — the cloud copy of this session log is archived`);
|
|
3518
3534
|
}
|