@indigoai-us/hq-cloud 5.35.0 → 5.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli/share.ts CHANGED
@@ -213,6 +213,42 @@ function computePushPlan(
213
213
  continue;
214
214
  }
215
215
 
216
+ // Fast-path (5.36.0): before SHA256-ing the file, lstat it and compare
217
+ // (size, mtimeMs) against the journal entry. When both match, treat the
218
+ // file as unchanged without reading its bytes. This is the same
219
+ // industry-standard "is it changed?" cheap-check rsync and gitignore
220
+ // use. Trade-off: a same-length edit that doesn't bump mtime will be
221
+ // missed — vanishingly rare in practice. The big win is on no-op syncs
222
+ // (most syncs): a 5000-file tree of mostly-unchanged content used to
223
+ // SHA256 every file on every walk. Now it lstats once and short-
224
+ // circuits in O(file count) instead of O(file bytes).
225
+ //
226
+ // Only applies when `skipUnchanged` is on AND the journal carries an
227
+ // `mtimeMs` for this path AND the path is a regular file (symlinks
228
+ // already hash via the cheap hashSymlinkTarget path above). Entries
229
+ // without `mtimeMs` (pre-5.36 journal) fall through to the hash path;
230
+ // the next upload stamps the field so subsequent syncs use the fast-
231
+ // path. Back-compat is automatic.
232
+ if (skipUnchanged) {
233
+ const existing = journal.files[relativePath];
234
+ if (existing && existing.mtimeMs !== undefined && existing.hash) {
235
+ try {
236
+ const lstat = fs.lstatSync(absolutePath);
237
+ if (
238
+ lstat.isFile() &&
239
+ lstat.size === existing.size &&
240
+ lstat.mtimeMs === existing.mtimeMs
241
+ ) {
242
+ items.push({ action: "skip-unchanged", absolutePath, relativePath });
243
+ continue;
244
+ }
245
+ } catch {
246
+ // Fall through to hashFile, which will surface the I/O error in
247
+ // its own readFileSync.
248
+ }
249
+ }
250
+ }
251
+
216
252
  const localHash = hashFile(absolutePath);
217
253
 
218
254
  if (skipUnchanged) {
@@ -682,6 +718,41 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
682
718
 
683
719
  // Stage 2: execute. Skip items pre-classified as no-ops, then for each
684
720
  // upload candidate run the HEAD + 3-way conflict check + actual PUT.
721
+ //
722
+ // 5.36.0: upload candidates go through a bounded-concurrent pool
723
+ // (`TRANSFER_CONCURRENCY`, default 16, tunable via
724
+ // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
725
+ // syncs. Skip-size-limit and skip-unchanged are handled inline first
726
+ // (pure local-state mutation). Upload candidates are collected into
727
+ // `uploadItems[]` and processed in parallel via the pool below.
728
+ //
729
+ // Abort handling: when any item's conflict resolution is "abort", we
730
+ // set `aborted = true` so the pool stops queueing new items, drain
731
+ // in-flight cleanly, and short-circuit to the abort return. In-flight
732
+ // PUTs that already issued will complete (S3 doesn't have client-side
733
+ // cancellation in this code path); their results are still recorded on
734
+ // the journal so the next sync's planner doesn't re-fire them.
735
+ // Interactive-mode guard (Codex P1, 5.36.x): when onConflict is unset
736
+ // the per-item conflict path falls into resolveConflict()'s readline
737
+ // prompt on process.stdin. Multiple pool workers prompting at once
738
+ // would race for the same terminal input and interleave answers
739
+ // nondeterministically. Force concurrency=1 for interactive sessions —
740
+ // the operator is at the keyboard anyway, the throughput penalty only
741
+ // applies when conflicts actually exist, and Option A (whole-pool
742
+ // serialization) has the smallest blast radius vs. a per-prompt mutex.
743
+ // Non-interactive (onConflict set) keeps the env-tunable default.
744
+ const isInteractiveConflictMode = onConflict === undefined;
745
+ const TRANSFER_CONCURRENCY = (() => {
746
+ if (isInteractiveConflictMode) return 1;
747
+ const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
748
+ if (raw === undefined || raw === "") return 16;
749
+ const parsed = Number.parseInt(raw, 10);
750
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
751
+ })();
752
+
753
+ // Phase A: serial classification pass — handle skips inline, collect
754
+ // upload candidates for the parallel pool.
755
+ const uploadItems: Array<typeof plan.items[number] & { action: "upload" }> = [];
685
756
  for (const item of plan.items) {
686
757
  if (item.action === "skip-size-limit") {
687
758
  emit({
@@ -696,7 +767,20 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
696
767
  filesSkipped++;
697
768
  continue;
698
769
  }
770
+ uploadItems.push(item);
771
+ }
699
772
 
773
+ // Phase B: parallel upload pool. Each task runs the full per-item flow
774
+ // (HEAD + conflict + PUT + journal stamp + emit). Aborts flip the
775
+ // shared `aborted` flag and the pool stops draining the queue; tasks
776
+ // already in flight complete normally.
777
+ let aborted = false;
778
+ let abortFlightConflictPaths: string[] = [];
779
+
780
+ const processUploadItem = async (
781
+ item: typeof uploadItems[number],
782
+ ): Promise<void> => {
783
+ if (aborted) return;
700
784
  const { absolutePath, relativePath, localHash } = item;
701
785
 
702
786
  // Auto-refresh context if credentials expiring. Only available on the
@@ -788,29 +872,13 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
788
872
  });
789
873
 
790
874
  if (resolution === "abort") {
791
- return {
792
- filesUploaded,
793
- bytesUploaded,
794
- filesSkipped,
795
- filesDeleted,
796
- // Abort path: delete stage never runs, so tombstone + refused-
797
- // stale counts are necessarily zero. Explicit fields keep the
798
- // ShareResult shape stable for consumers that destructure.
799
- filesTombstoned,
800
- filesRefusedStale,
801
- // Always present so consumers can destructure without a
802
- // defaulting fallback. Empty on the abort path because the
803
- // delete-plan execution loop is short-circuited.
804
- filesRefusedStalePaths,
805
- // Exclusions are computed during the upload walk which has
806
- // already completed by the time we hit a per-file conflict-
807
- // abort, so the count is meaningful here. No event emit on
808
- // abort (matches the existing convention: abort short-circuits
809
- // before the end-of-run telemetry emits).
810
- filesExcludedByPolicy: excludedSet.size,
811
- conflictPaths,
812
- aborted: true,
813
- };
875
+ // Flip the shared aborted flag — the pool drainer below sees this
876
+ // and stops queueing new items. In-flight tasks complete normally
877
+ // (S3 PUTs have no client-side cancel here). The outer abort
878
+ // return is built after the pool drains.
879
+ aborted = true;
880
+ abortFlightConflictPaths = [...conflictPaths];
881
+ return;
814
882
  }
815
883
  if (resolution === "keep" || resolution === "skip") {
816
884
  // Bug #7 mirror branch: when the resolution is keep/skip on a
@@ -853,19 +921,33 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
853
921
  }
854
922
  }
855
923
  filesSkipped++;
856
- continue;
924
+ return;
857
925
  }
858
926
  // "overwrite" falls through to upload
859
927
  }
860
928
  }
861
929
 
930
+ // Re-check abort flag right before the PUT — if a peer task aborted
931
+ // while we were waiting on HEAD, skip the PUT entirely. This is
932
+ // belt-and-suspenders alongside the queue-drain check; without it,
933
+ // up to TRANSFER_CONCURRENCY in-flight uploads could still issue PUTs
934
+ // after the user signaled abort.
935
+ if (aborted) return;
936
+
862
937
  // Upload — symlinks go through uploadSymlink (zero-byte body + target
863
938
  // metadata), regular files through uploadFile (file contents). The
864
939
  // discriminator is item.kind set by computePushPlan; both branches
865
940
  // converge on the same journal/event update path below.
866
941
  try {
867
942
  const isSymlinkUpload = item.kind === "symlink";
868
- const size = isSymlinkUpload ? 0 : fs.statSync(absolutePath).size;
943
+ // Capture lstat post-upload so size + mtimeMs stamped into the
944
+ // journal reflect the bytes we actually shipped. lstat (not stat)
945
+ // so a symlink stamps the link's own mtime, not the target's —
946
+ // matches the fast-path's lstat comparison so the next sync can
947
+ // skip without dereferencing.
948
+ const lstat = fs.lstatSync(absolutePath);
949
+ const size = isSymlinkUpload ? 0 : lstat.size;
950
+ const mtimeMs = lstat.mtimeMs;
869
951
 
870
952
  const { etag } = isSymlinkUpload
871
953
  ? options.author
@@ -877,8 +959,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
877
959
 
878
960
  // Update journal with optional message; capture the post-upload ETag
879
961
  // so the next sync can distinguish "remote moved since we last wrote"
880
- // from "user edited locally" without conflating the two.
881
- updateEntry(journal, relativePath, localHash, size, "up", etag);
962
+ // from "user edited locally" without conflating the two. mtimeMs
963
+ // feeds the 5.36.0 lstat fast-path on the next push.
964
+ updateEntry(journal, relativePath, localHash, size, "up", etag, mtimeMs);
882
965
  if (message) {
883
966
  journal.files[relativePath] = {
884
967
  ...journal.files[relativePath],
@@ -901,6 +984,80 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
901
984
  message: err instanceof Error ? err.message : String(err),
902
985
  });
903
986
  }
987
+ };
988
+
989
+ // Drain the upload queue with bounded concurrency. Per-file progress
990
+ // events fire from inside processUploadItem at file-settle time, so
991
+ // cross-file event ordering is settle-order, not plan-walk-order — the
992
+ // menubar's stream parser already tolerates per-company interleave so
993
+ // this is shape-compatible. allSettled-style waiting (via Promise.race
994
+ // on the in-flight set) keeps the pool topped up without idling on slow
995
+ // members.
996
+ //
997
+ // Codex P1 (5.36.x): each worker promise is wrapped in a .catch that
998
+ // records the error to `workerErrors` and resolves normally — so
999
+ // `Promise.race(inFlight)` can never reject and unwind the drain loop
1000
+ // mid-flight. Without the wrap, an unhandled rejection inside
1001
+ // processUploadItem (e.g. headRemoteFile or refreshEntityContext, both
1002
+ // of which sit outside the per-item PUT try/catch) bubbled out of the
1003
+ // race, abandoned still-in-flight uploads that kept mutating remote
1004
+ // state, and skipped writeJournal — leaving remote and journal
1005
+ // permanently out of sync for the next run. After the pool fully
1006
+ // drains we throw an aggregated error so the caller still surfaces the
1007
+ // failure (and the journal reflects the writes that actually landed).
1008
+ const workerErrors: Error[] = [];
1009
+ {
1010
+ const queue = [...uploadItems];
1011
+ const inFlight: Set<Promise<unknown>> = new Set();
1012
+ while (queue.length > 0 || inFlight.size > 0) {
1013
+ while (!aborted && inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
1014
+ const item = queue.shift()!;
1015
+ const p: Promise<void> = processUploadItem(item)
1016
+ .catch((err: unknown) => {
1017
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1018
+ })
1019
+ .finally(() => {
1020
+ inFlight.delete(p);
1021
+ });
1022
+ inFlight.add(p);
1023
+ }
1024
+ if (inFlight.size > 0) {
1025
+ await Promise.race(Array.from(inFlight));
1026
+ } else {
1027
+ // Aborted with nothing in flight → exit the drain loop.
1028
+ break;
1029
+ }
1030
+ }
1031
+ }
1032
+
1033
+ if (aborted) {
1034
+ return {
1035
+ filesUploaded,
1036
+ bytesUploaded,
1037
+ filesSkipped,
1038
+ filesDeleted,
1039
+ // Abort path: delete stage never runs, so tombstone + refused-
1040
+ // stale counts are necessarily zero. Explicit fields keep the
1041
+ // ShareResult shape stable for consumers that destructure.
1042
+ filesTombstoned,
1043
+ filesRefusedStale,
1044
+ // Always present so consumers can destructure without a
1045
+ // defaulting fallback. Empty on the abort path because the
1046
+ // delete-plan execution loop is short-circuited.
1047
+ filesRefusedStalePaths,
1048
+ // Exclusions are computed during the upload walk which has
1049
+ // already completed by the time we hit a per-file conflict-
1050
+ // abort, so the count is meaningful here. No event emit on
1051
+ // abort (matches the existing convention: abort short-circuits
1052
+ // before the end-of-run telemetry emits).
1053
+ filesExcludedByPolicy: excludedSet.size,
1054
+ // Use the snapshot of conflictPaths taken at the moment the abort
1055
+ // fired — additional in-flight items may have appended to the
1056
+ // shared array after the abort signal, and those should not show
1057
+ // up in the abort result.
1058
+ conflictPaths: abortFlightConflictPaths,
1059
+ aborted: true,
1060
+ };
904
1061
  }
905
1062
 
906
1063
  // Stage 3: propagate deletes. Three buckets, three actions:
@@ -1009,6 +1166,23 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
1009
1166
  });
1010
1167
  }
1011
1168
 
1169
+ // Codex P1 (5.36.x): if any worker rejected (unhandled error in
1170
+ // headRemoteFile / refreshEntityContext / resolveConflict — paths
1171
+ // outside the per-item PUT try/catch), we deliberately let the pool
1172
+ // drain AND let post-pool stages (deletes, tombstones, journal write,
1173
+ // personal-vault summary) run to completion so journal + remote stay
1174
+ // converged on what actually landed. NOW surface the failure to the
1175
+ // caller — preserving the first error's stack so debugging works.
1176
+ // Aggregate count is reported in the message for visibility when
1177
+ // multiple workers failed.
1178
+ if (workerErrors.length > 0) {
1179
+ const first = workerErrors[0]!;
1180
+ if (workerErrors.length > 1) {
1181
+ first.message = `${first.message} (and ${workerErrors.length - 1} more upload-worker errors)`;
1182
+ }
1183
+ throw first;
1184
+ }
1185
+
1012
1186
  return {
1013
1187
  filesUploaded,
1014
1188
  bytesUploaded,
package/src/cli/sync.ts CHANGED
@@ -335,7 +335,39 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
335
335
  // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
336
336
  // inline loop; the only structural change is that classification has
337
337
  // already happened (so `localHash` is reused instead of re-hashing).
338
+ //
339
+ // 5.36.0: download items go through a bounded-concurrent pool
340
+ // (`TRANSFER_CONCURRENCY`, default 16, tunable via
341
+ // `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
342
+ // syncs. Conflict items stay serial — they may prompt the operator
343
+ // and the abort path must short-circuit before any further work. We
344
+ // partition the plan items into "conflict (serial)" and "download
345
+ // (parallel)" buckets and run the serial pass first; the parallel pass
346
+ // only runs if no conflict aborted.
347
+ //
348
+ // Per-file `progress` events fire at the moment each individual download
349
+ // settles (inside the pool wrapper), NOT in plan-walk order. The cross-
350
+ // file interleave is acceptable: the menubar stream parser already
351
+ // handles per-company interleave, and the same shape applies within a
352
+ // single company's pool. Per-file event-count correctness is preserved
353
+ // (one progress per download, one error per failure).
354
+ const TRANSFER_CONCURRENCY = (() => {
355
+ const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
356
+ if (raw === undefined || raw === "") return 16;
357
+ const parsed = Number.parseInt(raw, 10);
358
+ return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
359
+ })();
360
+
361
+ // First pass: serial walk for non-download outcomes (skips + conflicts).
362
+ // Conflicts may set `aborted = true` and short-circuit the whole pull;
363
+ // we detect that and skip the parallel pass. Download items are
364
+ // collected into `downloadItems[]` for the pool pass below.
365
+ const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
366
+ let aborted = false;
367
+ let abortResult: SyncResult | null = null;
368
+
338
369
  for (const item of plan.items) {
370
+ if (aborted) break;
339
371
  if (
340
372
  item.action === "skip-ignored" ||
341
373
  item.action === "skip-personal-mode" ||
@@ -353,6 +385,11 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
353
385
  continue;
354
386
  }
355
387
 
388
+ if (item.action === "download") {
389
+ downloadItems.push(item);
390
+ continue;
391
+ }
392
+
356
393
  const { remoteFile, localPath } = item;
357
394
 
358
395
  // Auto-refresh context if credentials expiring (kept in execute phase
@@ -428,7 +465,8 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
428
465
  if (resolution === "abort") {
429
466
  emit({ type: "new-files", files: [] });
430
467
  writeJournal(journalSlug, journal);
431
- return {
468
+ aborted = true;
469
+ abortResult = {
432
470
  filesDownloaded,
433
471
  bytesDownloaded,
434
472
  filesSkipped,
@@ -443,6 +481,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
443
481
  // destructure it.
444
482
  filesTombstoned: 0,
445
483
  };
484
+ break;
446
485
  }
447
486
  if (resolution === "keep" || resolution === "skip") {
448
487
  filesSkipped++;
@@ -467,63 +506,157 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
467
506
  item.localSize,
468
507
  "down",
469
508
  remoteFile.etag,
509
+ item.localMtime.getTime(),
470
510
  );
471
511
  continue;
472
512
  }
473
- // "overwrite" falls through to download
513
+ // "overwrite" falls through to download — re-route through the pool
514
+ // so it benefits from parallelism too. Synthesize a download item
515
+ // pointing at the same remoteFile/localPath; isNew=false because
516
+ // there was a conflict-eligible local file present.
517
+ downloadItems.push({
518
+ action: "download",
519
+ remoteFile,
520
+ localPath,
521
+ isNew: false,
522
+ });
523
+ continue;
474
524
  }
525
+ }
475
526
 
476
- // Download (action === "download" or conflict resolved to "overwrite")
477
- try {
478
- const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
479
- const author = metadata?.["created-by"] ?? null;
527
+ // Early-return on conflict abort BEFORE running the parallel download
528
+ // pool — the abort intent is "stop the pull now", not "stop new work but
529
+ // finish what's in flight". Since the pool hasn't started yet, this is
530
+ // a clean drain (zero items in flight) by construction.
531
+ if (aborted && abortResult) {
532
+ return abortResult;
533
+ }
480
534
 
481
- // Symlink records materialize as real symlinks on disk. lstat
482
- // (does not follow) lets us detect that case so the journal stamp
483
- // mirrors what the push side would emit on the next tick:
484
- // hash = sha256(readlink target string)
485
- // size = 0
486
- // Without this check, hashFile would follow the link and stamp the
487
- // target file's contents — a value the next push would never
488
- // produce which makes skipUnchanged perpetually re-upload every
489
- // symlink, defeating the point of the gate.
490
- const localLstat = fs.lstatSync(localPath);
491
- const isLocalSymlink = localLstat.isSymbolicLink();
492
- const hash = isLocalSymlink
493
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
494
- : hashFile(localPath);
495
- const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
496
- // Capture the listing's ETag so subsequent syncs can detect remote
497
- // drift independently of mtime drift.
498
- updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag);
535
+ // ── Parallel download pool (5.36.0) ───────────────────────────────────
536
+ // Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
537
+ // race-based shape as the HEAD-verify pool above but applied to the body
538
+ // of each download. Per-item progress events fire at file-settle time
539
+ // (inside the wrapper), so cross-file interleave is expected and the
540
+ // menubar's stream parser already handles it.
541
+ if (downloadItems.length > 0) {
542
+ const queue = [...downloadItems];
543
+ const inFlight: Set<Promise<unknown>> = new Set();
499
544
 
500
- // Attach message from the prior journal entry if present (set by a
501
- // previous `share` operation that included a --message).
502
- const priorEntry = getEntry(journal, remoteFile.key);
503
- const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
504
- emit({
505
- type: "progress",
506
- path: remoteFile.key,
507
- bytes: size,
508
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
509
- ...(author ? { author } : {}),
510
- });
545
+ const downloadOne = async (
546
+ downloadItem: typeof downloadItems[number],
547
+ ): Promise<void> => {
548
+ const { remoteFile, localPath } = downloadItem;
511
549
 
512
- filesDownloaded++;
513
- bytesDownloaded += size;
514
- } catch (err) {
515
- // STS session policy may deny access to some paths — this is expected
516
- // for guest members with allowedPrefixes
517
- if (isAccessDenied(err)) {
518
- filesSkipped++;
519
- } else {
550
+ // Auto-refresh context if credentials expiring. Each task checks
551
+ // independently — refresh is idempotent on the same context object.
552
+ if (isExpiringSoon(ctx.expiresAt)) {
553
+ ctx = await refreshEntityContext(companyRef, vaultConfig);
554
+ }
555
+
556
+ try {
557
+ const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
558
+ const author = metadata?.["created-by"] ?? null;
559
+
560
+ // Symlink records materialize as real symlinks on disk. lstat
561
+ // (does not follow) lets us detect that case so the journal stamp
562
+ // mirrors what the push side would emit on the next tick:
563
+ // hash = sha256(readlink target string)
564
+ // size = 0
565
+ // Without this check, hashFile would follow the link and stamp the
566
+ // target file's contents — a value the next push would never
567
+ // produce — which makes skipUnchanged perpetually re-upload every
568
+ // symlink, defeating the point of the gate.
569
+ const localLstat = fs.lstatSync(localPath);
570
+ const isLocalSymlink = localLstat.isSymbolicLink();
571
+ const hash = isLocalSymlink
572
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
573
+ : hashFile(localPath);
574
+ const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
575
+ // Capture the listing's ETag so subsequent syncs can detect remote
576
+ // drift independently of mtime drift. Stamp mtimeMs from localLstat
577
+ // (5.36.0) so the next push planner's lstat fast-path can skip the
578
+ // SHA256 for this file without reading its bytes.
579
+ updateEntry(
580
+ journal,
581
+ remoteFile.key,
582
+ hash,
583
+ size,
584
+ "down",
585
+ remoteFile.etag,
586
+ localLstat.mtimeMs,
587
+ );
588
+
589
+ // Attach message from the prior journal entry if present (set by a
590
+ // previous `share` operation that included a --message).
591
+ const priorEntry = getEntry(journal, remoteFile.key);
592
+ const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
520
593
  emit({
521
- type: "error",
594
+ type: "progress",
522
595
  path: remoteFile.key,
523
- message: err instanceof Error ? err.message : String(err),
596
+ bytes: size,
597
+ ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
598
+ ...(author ? { author } : {}),
524
599
  });
600
+
601
+ filesDownloaded++;
602
+ bytesDownloaded += size;
603
+ } catch (err) {
604
+ // STS session policy may deny access to some paths — this is expected
605
+ // for guest members with allowedPrefixes
606
+ if (isAccessDenied(err)) {
607
+ filesSkipped++;
608
+ } else {
609
+ emit({
610
+ type: "error",
611
+ path: remoteFile.key,
612
+ message: err instanceof Error ? err.message : String(err),
613
+ });
614
+ }
615
+ }
616
+ };
617
+
618
+ // Codex P1 (5.36.x): worker promises wrapped in .catch so an
619
+ // unhandled rejection inside downloadOne (e.g. refreshEntityContext
620
+ // before the per-item try/catch, or lstatSync after the download
621
+ // succeeded but before journal stamping) cannot escape
622
+ // `Promise.race(inFlight)` and unwind the drain mid-flight. Without
623
+ // this wrap, sibling downloads kept running after share()/sync()
624
+ // had already failed, their files materialized on disk without
625
+ // matching journal entries, and the next sync re-downloaded
626
+ // everything. Errors are collected and surfaced after the pool
627
+ // fully drains — see workerErrors throw below.
628
+ const workerErrors: Error[] = [];
629
+ while (queue.length > 0 || inFlight.size > 0) {
630
+ while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
631
+ const downloadItem = queue.shift()!;
632
+ const p: Promise<void> = downloadOne(downloadItem)
633
+ .catch((err: unknown) => {
634
+ workerErrors.push(err instanceof Error ? err : new Error(String(err)));
635
+ })
636
+ .finally(() => {
637
+ inFlight.delete(p);
638
+ });
639
+ inFlight.add(p);
640
+ }
641
+ if (inFlight.size > 0) {
642
+ // Wait for at least one in-flight task to settle before topping up
643
+ // the pool. allSettled-style semantics via Promise.race — the
644
+ // .catch wrap above guarantees no worker promise can reject.
645
+ await Promise.race(Array.from(inFlight));
525
646
  }
526
647
  }
648
+
649
+ // Pool drained. If any worker rejected, write the journal first
650
+ // (so the lstat fast-path stamps for successfully-downloaded files
651
+ // persist) then throw the first error, preserving its stack.
652
+ if (workerErrors.length > 0) {
653
+ writeJournal(journalSlug, journal);
654
+ const first = workerErrors[0]!;
655
+ if (workerErrors.length > 1) {
656
+ first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
657
+ }
658
+ throw first;
659
+ }
527
660
  }
528
661
 
529
662
  // ── New-files attribution (US-002) ─────────────────────────────────────
@@ -928,9 +1061,28 @@ function computePullPlan(
928
1061
  items.push({ action: "skip-local-only", remoteFile, localPath });
929
1062
  continue;
930
1063
  }
931
- const localHash = isLocalSymlink
932
- ? hashSymlinkTarget(fs.readlinkSync(localPath))
933
- : hashFile(localPath);
1064
+ // Fast-path (5.36.0): when the journal entry has an mtimeMs and the
1065
+ // local lstat matches both size and mtimeMs, the file hasn't been
1066
+ // touched locally since the last sync, so we can reuse the journal's
1067
+ // recorded hash without re-reading the file's bytes. Same shape as
1068
+ // the share-side fast-path; only kicks in for regular files (symlink
1069
+ // hashing through hashSymlinkTarget is already cheap — it hashes a
1070
+ // short target string, not file bytes).
1071
+ let localHash: string;
1072
+ if (
1073
+ !isLocalSymlink &&
1074
+ journalEntry &&
1075
+ journalEntry.mtimeMs !== undefined &&
1076
+ journalEntry.hash &&
1077
+ localLstat!.size === journalEntry.size &&
1078
+ localLstat!.mtimeMs === journalEntry.mtimeMs
1079
+ ) {
1080
+ localHash = journalEntry.hash;
1081
+ } else {
1082
+ localHash = isLocalSymlink
1083
+ ? hashSymlinkTarget(fs.readlinkSync(localPath))
1084
+ : hashFile(localPath);
1085
+ }
934
1086
  const localChanged = !!journalEntry && journalEntry.hash !== localHash;
935
1087
  const remoteChanged =
936
1088
  !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
package/src/journal.ts CHANGED
@@ -210,6 +210,7 @@ export function updateEntry(
210
210
  size: number,
211
211
  direction: "up" | "down",
212
212
  remoteEtag?: string,
213
+ mtimeMs?: number,
213
214
  ): void {
214
215
  const entry: JournalEntry = {
215
216
  hash,
@@ -220,6 +221,9 @@ export function updateEntry(
220
221
  if (remoteEtag !== undefined && remoteEtag !== "") {
221
222
  entry.remoteEtag = normalizeEtag(remoteEtag);
222
223
  }
224
+ if (mtimeMs !== undefined) {
225
+ entry.mtimeMs = mtimeMs;
226
+ }
223
227
  journal.files[relativePath] = entry;
224
228
  journal.lastSync = new Date().toISOString();
225
229
  }
package/src/types.ts CHANGED
@@ -34,6 +34,22 @@ export interface JournalEntry {
34
34
  * against `syncedAt`.
35
35
  */
36
36
  remoteEtag?: string;
37
+ /**
38
+ * Local mtime (epoch ms) of the file at the moment we last synced it. Used
39
+ * by the push planner as the fast-path "is this file unchanged?" check:
40
+ * when `lstat.size === entry.size && lstat.mtimeMs === entry.mtimeMs`,
41
+ * the SHA256 is skipped and the file is classified `unchanged` without
42
+ * reading its bytes. Same trade-off rsync/gitignore use — a same-length
43
+ * edit that doesn't bump mtime will be missed, but that's vanishingly
44
+ * rare in practice and the speedup on no-op syncs (~5–10×) is the goal.
45
+ *
46
+ * Optional for backwards compatibility: entries written before this field
47
+ * existed (or symlink entries, where lstat.mtimeMs is the link's own
48
+ * mtime and the prefixed-hash check is already cheap) fall through to
49
+ * `hashFile()` on the first post-upgrade sync; the next `updateEntry`
50
+ * stamps the field so subsequent syncs use the fast-path.
51
+ */
52
+ mtimeMs?: number;
37
53
  /**
38
54
  * Tombstone marker (Journal v2, US-005). When set, this entry represents
39
55
  * a file that was pruned by a scope shrink — either implicitly (next pull