@indigoai-us/hq-cloud 5.34.0 → 5.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +196 -27
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +532 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +182 -49
- package/dist/cli/sync.js.map +1 -1
- package/dist/ignore.d.ts.map +1 -1
- package/dist/ignore.js +13 -0
- package/dist/ignore.js.map +1 -1
- package/dist/ignore.test.js +28 -0
- package/dist/ignore.test.js.map +1 -1
- package/dist/journal.d.ts +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +4 -1
- package/dist/journal.js.map +1 -1
- package/dist/types.d.ts +16 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/cli/share.test.ts +594 -0
- package/src/cli/share.ts +201 -27
- package/src/cli/sync.ts +200 -48
- package/src/ignore.test.ts +37 -0
- package/src/ignore.ts +14 -0
- package/src/journal.ts +4 -0
- package/src/types.ts +16 -0
package/src/cli/share.ts
CHANGED
|
@@ -213,6 +213,42 @@ function computePushPlan(
|
|
|
213
213
|
continue;
|
|
214
214
|
}
|
|
215
215
|
|
|
216
|
+
// Fast-path (5.36.0): before SHA256-ing the file, lstat it and compare
|
|
217
|
+
// (size, mtimeMs) against the journal entry. When both match, treat the
|
|
218
|
+
// file as unchanged without reading its bytes. This is the same
|
|
219
|
+
// industry-standard "is it changed?" cheap-check rsync and gitignore
|
|
220
|
+
// use. Trade-off: a same-length edit that doesn't bump mtime will be
|
|
221
|
+
// missed — vanishingly rare in practice. The big win is on no-op syncs
|
|
222
|
+
// (most syncs): a 5000-file tree of mostly-unchanged content used to
|
|
223
|
+
// SHA256 every file on every walk. Now it lstats once and short-
|
|
224
|
+
// circuits in O(file count) instead of O(file bytes).
|
|
225
|
+
//
|
|
226
|
+
// Only applies when `skipUnchanged` is on AND the journal carries an
|
|
227
|
+
// `mtimeMs` for this path AND the path is a regular file (symlinks
|
|
228
|
+
// already hash via the cheap hashSymlinkTarget path above). Entries
|
|
229
|
+
// without `mtimeMs` (pre-5.36 journal) fall through to the hash path;
|
|
230
|
+
// the next upload stamps the field so subsequent syncs use the fast-
|
|
231
|
+
// path. Back-compat is automatic.
|
|
232
|
+
if (skipUnchanged) {
|
|
233
|
+
const existing = journal.files[relativePath];
|
|
234
|
+
if (existing && existing.mtimeMs !== undefined && existing.hash) {
|
|
235
|
+
try {
|
|
236
|
+
const lstat = fs.lstatSync(absolutePath);
|
|
237
|
+
if (
|
|
238
|
+
lstat.isFile() &&
|
|
239
|
+
lstat.size === existing.size &&
|
|
240
|
+
lstat.mtimeMs === existing.mtimeMs
|
|
241
|
+
) {
|
|
242
|
+
items.push({ action: "skip-unchanged", absolutePath, relativePath });
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
} catch {
|
|
246
|
+
// Fall through to hashFile, which will surface the I/O error in
|
|
247
|
+
// its own readFileSync.
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
216
252
|
const localHash = hashFile(absolutePath);
|
|
217
253
|
|
|
218
254
|
if (skipUnchanged) {
|
|
@@ -682,6 +718,41 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
682
718
|
|
|
683
719
|
// Stage 2: execute. Skip items pre-classified as no-ops, then for each
|
|
684
720
|
// upload candidate run the HEAD + 3-way conflict check + actual PUT.
|
|
721
|
+
//
|
|
722
|
+
// 5.36.0: upload candidates go through a bounded-concurrent pool
|
|
723
|
+
// (`TRANSFER_CONCURRENCY`, default 16, tunable via
|
|
724
|
+
// `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
|
|
725
|
+
// syncs. Skip-size-limit and skip-unchanged are handled inline first
|
|
726
|
+
// (pure local-state mutation). Upload candidates are collected into
|
|
727
|
+
// `uploadItems[]` and processed in parallel via the pool below.
|
|
728
|
+
//
|
|
729
|
+
// Abort handling: when any item's conflict resolution is "abort", we
|
|
730
|
+
// set `aborted = true` so the pool stops queueing new items, drain
|
|
731
|
+
// in-flight cleanly, and short-circuit to the abort return. In-flight
|
|
732
|
+
// PUTs that already issued will complete (S3 doesn't have client-side
|
|
733
|
+
// cancellation in this code path); their results are still recorded on
|
|
734
|
+
// the journal so the next sync's planner doesn't re-fire them.
|
|
735
|
+
// Interactive-mode guard (Codex P1, 5.36.x): when onConflict is unset
|
|
736
|
+
// the per-item conflict path falls into resolveConflict()'s readline
|
|
737
|
+
// prompt on process.stdin. Multiple pool workers prompting at once
|
|
738
|
+
// would race for the same terminal input and interleave answers
|
|
739
|
+
// nondeterministically. Force concurrency=1 for interactive sessions —
|
|
740
|
+
// the operator is at the keyboard anyway, the throughput penalty only
|
|
741
|
+
// applies when conflicts actually exist, and Option A (whole-pool
|
|
742
|
+
// serialization) has the smallest blast radius vs. a per-prompt mutex.
|
|
743
|
+
// Non-interactive (onConflict set) keeps the env-tunable default.
|
|
744
|
+
const isInteractiveConflictMode = onConflict === undefined;
|
|
745
|
+
const TRANSFER_CONCURRENCY = (() => {
|
|
746
|
+
if (isInteractiveConflictMode) return 1;
|
|
747
|
+
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
748
|
+
if (raw === undefined || raw === "") return 16;
|
|
749
|
+
const parsed = Number.parseInt(raw, 10);
|
|
750
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
751
|
+
})();
|
|
752
|
+
|
|
753
|
+
// Phase A: serial classification pass — handle skips inline, collect
|
|
754
|
+
// upload candidates for the parallel pool.
|
|
755
|
+
const uploadItems: Array<typeof plan.items[number] & { action: "upload" }> = [];
|
|
685
756
|
for (const item of plan.items) {
|
|
686
757
|
if (item.action === "skip-size-limit") {
|
|
687
758
|
emit({
|
|
@@ -696,7 +767,20 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
696
767
|
filesSkipped++;
|
|
697
768
|
continue;
|
|
698
769
|
}
|
|
770
|
+
uploadItems.push(item);
|
|
771
|
+
}
|
|
699
772
|
|
|
773
|
+
// Phase B: parallel upload pool. Each task runs the full per-item flow
|
|
774
|
+
// (HEAD + conflict + PUT + journal stamp + emit). Aborts flip the
|
|
775
|
+
// shared `aborted` flag and the pool stops draining the queue; tasks
|
|
776
|
+
// already in flight complete normally.
|
|
777
|
+
let aborted = false;
|
|
778
|
+
let abortFlightConflictPaths: string[] = [];
|
|
779
|
+
|
|
780
|
+
const processUploadItem = async (
|
|
781
|
+
item: typeof uploadItems[number],
|
|
782
|
+
): Promise<void> => {
|
|
783
|
+
if (aborted) return;
|
|
700
784
|
const { absolutePath, relativePath, localHash } = item;
|
|
701
785
|
|
|
702
786
|
// Auto-refresh context if credentials expiring. Only available on the
|
|
@@ -788,29 +872,13 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
788
872
|
});
|
|
789
873
|
|
|
790
874
|
if (resolution === "abort") {
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
// ShareResult shape stable for consumers that destructure.
|
|
799
|
-
filesTombstoned,
|
|
800
|
-
filesRefusedStale,
|
|
801
|
-
// Always present so consumers can destructure without a
|
|
802
|
-
// defaulting fallback. Empty on the abort path because the
|
|
803
|
-
// delete-plan execution loop is short-circuited.
|
|
804
|
-
filesRefusedStalePaths,
|
|
805
|
-
// Exclusions are computed during the upload walk which has
|
|
806
|
-
// already completed by the time we hit a per-file conflict-
|
|
807
|
-
// abort, so the count is meaningful here. No event emit on
|
|
808
|
-
// abort (matches the existing convention: abort short-circuits
|
|
809
|
-
// before the end-of-run telemetry emits).
|
|
810
|
-
filesExcludedByPolicy: excludedSet.size,
|
|
811
|
-
conflictPaths,
|
|
812
|
-
aborted: true,
|
|
813
|
-
};
|
|
875
|
+
// Flip the shared aborted flag — the pool drainer below sees this
|
|
876
|
+
// and stops queueing new items. In-flight tasks complete normally
|
|
877
|
+
// (S3 PUTs have no client-side cancel here). The outer abort
|
|
878
|
+
// return is built after the pool drains.
|
|
879
|
+
aborted = true;
|
|
880
|
+
abortFlightConflictPaths = [...conflictPaths];
|
|
881
|
+
return;
|
|
814
882
|
}
|
|
815
883
|
if (resolution === "keep" || resolution === "skip") {
|
|
816
884
|
// Bug #7 mirror branch: when the resolution is keep/skip on a
|
|
@@ -853,19 +921,33 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
853
921
|
}
|
|
854
922
|
}
|
|
855
923
|
filesSkipped++;
|
|
856
|
-
|
|
924
|
+
return;
|
|
857
925
|
}
|
|
858
926
|
// "overwrite" falls through to upload
|
|
859
927
|
}
|
|
860
928
|
}
|
|
861
929
|
|
|
930
|
+
// Re-check abort flag right before the PUT — if a peer task aborted
|
|
931
|
+
// while we were waiting on HEAD, skip the PUT entirely. This is
|
|
932
|
+
// belt-and-suspenders alongside the queue-drain check; without it,
|
|
933
|
+
// up to TRANSFER_CONCURRENCY in-flight uploads could still issue PUTs
|
|
934
|
+
// after the user signaled abort.
|
|
935
|
+
if (aborted) return;
|
|
936
|
+
|
|
862
937
|
// Upload — symlinks go through uploadSymlink (zero-byte body + target
|
|
863
938
|
// metadata), regular files through uploadFile (file contents). The
|
|
864
939
|
// discriminator is item.kind set by computePushPlan; both branches
|
|
865
940
|
// converge on the same journal/event update path below.
|
|
866
941
|
try {
|
|
867
942
|
const isSymlinkUpload = item.kind === "symlink";
|
|
868
|
-
|
|
943
|
+
// Capture lstat post-upload so size + mtimeMs stamped into the
|
|
944
|
+
// journal reflect the bytes we actually shipped. lstat (not stat)
|
|
945
|
+
// so a symlink stamps the link's own mtime, not the target's —
|
|
946
|
+
// matches the fast-path's lstat comparison so the next sync can
|
|
947
|
+
// skip without dereferencing.
|
|
948
|
+
const lstat = fs.lstatSync(absolutePath);
|
|
949
|
+
const size = isSymlinkUpload ? 0 : lstat.size;
|
|
950
|
+
const mtimeMs = lstat.mtimeMs;
|
|
869
951
|
|
|
870
952
|
const { etag } = isSymlinkUpload
|
|
871
953
|
? options.author
|
|
@@ -877,8 +959,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
877
959
|
|
|
878
960
|
// Update journal with optional message; capture the post-upload ETag
|
|
879
961
|
// so the next sync can distinguish "remote moved since we last wrote"
|
|
880
|
-
// from "user edited locally" without conflating the two.
|
|
881
|
-
|
|
962
|
+
// from "user edited locally" without conflating the two. mtimeMs
|
|
963
|
+
// feeds the 5.36.0 lstat fast-path on the next push.
|
|
964
|
+
updateEntry(journal, relativePath, localHash, size, "up", etag, mtimeMs);
|
|
882
965
|
if (message) {
|
|
883
966
|
journal.files[relativePath] = {
|
|
884
967
|
...journal.files[relativePath],
|
|
@@ -901,6 +984,80 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
901
984
|
message: err instanceof Error ? err.message : String(err),
|
|
902
985
|
});
|
|
903
986
|
}
|
|
987
|
+
};
|
|
988
|
+
|
|
989
|
+
// Drain the upload queue with bounded concurrency. Per-file progress
|
|
990
|
+
// events fire from inside processUploadItem at file-settle time, so
|
|
991
|
+
// cross-file event ordering is settle-order, not plan-walk-order — the
|
|
992
|
+
// menubar's stream parser already tolerates per-company interleave so
|
|
993
|
+
// this is shape-compatible. allSettled-style waiting (via Promise.race
|
|
994
|
+
// on the in-flight set) keeps the pool topped up without idling on slow
|
|
995
|
+
// members.
|
|
996
|
+
//
|
|
997
|
+
// Codex P1 (5.36.x): each worker promise is wrapped in a .catch that
|
|
998
|
+
// records the error to `workerErrors` and resolves normally — so
|
|
999
|
+
// `Promise.race(inFlight)` can never reject and unwind the drain loop
|
|
1000
|
+
// mid-flight. Without the wrap, an unhandled rejection inside
|
|
1001
|
+
// processUploadItem (e.g. headRemoteFile or refreshEntityContext, both
|
|
1002
|
+
// of which sit outside the per-item PUT try/catch) bubbled out of the
|
|
1003
|
+
// race, abandoned still-in-flight uploads that kept mutating remote
|
|
1004
|
+
// state, and skipped writeJournal — leaving remote and journal
|
|
1005
|
+
// permanently out of sync for the next run. After the pool fully
|
|
1006
|
+
// drains we throw an aggregated error so the caller still surfaces the
|
|
1007
|
+
// failure (and the journal reflects the writes that actually landed).
|
|
1008
|
+
const workerErrors: Error[] = [];
|
|
1009
|
+
{
|
|
1010
|
+
const queue = [...uploadItems];
|
|
1011
|
+
const inFlight: Set<Promise<unknown>> = new Set();
|
|
1012
|
+
while (queue.length > 0 || inFlight.size > 0) {
|
|
1013
|
+
while (!aborted && inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
|
|
1014
|
+
const item = queue.shift()!;
|
|
1015
|
+
const p: Promise<void> = processUploadItem(item)
|
|
1016
|
+
.catch((err: unknown) => {
|
|
1017
|
+
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1018
|
+
})
|
|
1019
|
+
.finally(() => {
|
|
1020
|
+
inFlight.delete(p);
|
|
1021
|
+
});
|
|
1022
|
+
inFlight.add(p);
|
|
1023
|
+
}
|
|
1024
|
+
if (inFlight.size > 0) {
|
|
1025
|
+
await Promise.race(Array.from(inFlight));
|
|
1026
|
+
} else {
|
|
1027
|
+
// Aborted with nothing in flight → exit the drain loop.
|
|
1028
|
+
break;
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
if (aborted) {
|
|
1034
|
+
return {
|
|
1035
|
+
filesUploaded,
|
|
1036
|
+
bytesUploaded,
|
|
1037
|
+
filesSkipped,
|
|
1038
|
+
filesDeleted,
|
|
1039
|
+
// Abort path: delete stage never runs, so tombstone + refused-
|
|
1040
|
+
// stale counts are necessarily zero. Explicit fields keep the
|
|
1041
|
+
// ShareResult shape stable for consumers that destructure.
|
|
1042
|
+
filesTombstoned,
|
|
1043
|
+
filesRefusedStale,
|
|
1044
|
+
// Always present so consumers can destructure without a
|
|
1045
|
+
// defaulting fallback. Empty on the abort path because the
|
|
1046
|
+
// delete-plan execution loop is short-circuited.
|
|
1047
|
+
filesRefusedStalePaths,
|
|
1048
|
+
// Exclusions are computed during the upload walk which has
|
|
1049
|
+
// already completed by the time we hit a per-file conflict-
|
|
1050
|
+
// abort, so the count is meaningful here. No event emit on
|
|
1051
|
+
// abort (matches the existing convention: abort short-circuits
|
|
1052
|
+
// before the end-of-run telemetry emits).
|
|
1053
|
+
filesExcludedByPolicy: excludedSet.size,
|
|
1054
|
+
// Use the snapshot of conflictPaths taken at the moment the abort
|
|
1055
|
+
// fired — additional in-flight items may have appended to the
|
|
1056
|
+
// shared array after the abort signal, and those should not show
|
|
1057
|
+
// up in the abort result.
|
|
1058
|
+
conflictPaths: abortFlightConflictPaths,
|
|
1059
|
+
aborted: true,
|
|
1060
|
+
};
|
|
904
1061
|
}
|
|
905
1062
|
|
|
906
1063
|
// Stage 3: propagate deletes. Three buckets, three actions:
|
|
@@ -1009,6 +1166,23 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1009
1166
|
});
|
|
1010
1167
|
}
|
|
1011
1168
|
|
|
1169
|
+
// Codex P1 (5.36.x): if any worker rejected (unhandled error in
|
|
1170
|
+
// headRemoteFile / refreshEntityContext / resolveConflict — paths
|
|
1171
|
+
// outside the per-item PUT try/catch), we deliberately let the pool
|
|
1172
|
+
// drain AND let post-pool stages (deletes, tombstones, journal write,
|
|
1173
|
+
// personal-vault summary) run to completion so journal + remote stay
|
|
1174
|
+
// converged on what actually landed. NOW surface the failure to the
|
|
1175
|
+
// caller — preserving the first error's stack so debugging works.
|
|
1176
|
+
// Aggregate count is reported in the message for visibility when
|
|
1177
|
+
// multiple workers failed.
|
|
1178
|
+
if (workerErrors.length > 0) {
|
|
1179
|
+
const first = workerErrors[0]!;
|
|
1180
|
+
if (workerErrors.length > 1) {
|
|
1181
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more upload-worker errors)`;
|
|
1182
|
+
}
|
|
1183
|
+
throw first;
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1012
1186
|
return {
|
|
1013
1187
|
filesUploaded,
|
|
1014
1188
|
bytesUploaded,
|
package/src/cli/sync.ts
CHANGED
|
@@ -335,7 +335,39 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
335
335
|
// Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
|
|
336
336
|
// inline loop; the only structural change is that classification has
|
|
337
337
|
// already happened (so `localHash` is reused instead of re-hashing).
|
|
338
|
+
//
|
|
339
|
+
// 5.36.0: download items go through a bounded-concurrent pool
|
|
340
|
+
// (`TRANSFER_CONCURRENCY`, default 16, tunable via
|
|
341
|
+
// `HQ_SYNC_TRANSFER_CONCURRENCY`) for 4-8x speedup on transfer-heavy
|
|
342
|
+
// syncs. Conflict items stay serial — they may prompt the operator
|
|
343
|
+
// and the abort path must short-circuit before any further work. We
|
|
344
|
+
// partition the plan items into "conflict (serial)" and "download
|
|
345
|
+
// (parallel)" buckets and run the serial pass first; the parallel pass
|
|
346
|
+
// only runs if no conflict aborted.
|
|
347
|
+
//
|
|
348
|
+
// Per-file `progress` events fire at the moment each individual download
|
|
349
|
+
// settles (inside the pool wrapper), NOT in plan-walk order. The cross-
|
|
350
|
+
// file interleave is acceptable: the menubar stream parser already
|
|
351
|
+
// handles per-company interleave, and the same shape applies within a
|
|
352
|
+
// single company's pool. Per-file event-count correctness is preserved
|
|
353
|
+
// (one progress per download, one error per failure).
|
|
354
|
+
const TRANSFER_CONCURRENCY = (() => {
|
|
355
|
+
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
356
|
+
if (raw === undefined || raw === "") return 16;
|
|
357
|
+
const parsed = Number.parseInt(raw, 10);
|
|
358
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
359
|
+
})();
|
|
360
|
+
|
|
361
|
+
// First pass: serial walk for non-download outcomes (skips + conflicts).
|
|
362
|
+
// Conflicts may set `aborted = true` and short-circuit the whole pull;
|
|
363
|
+
// we detect that and skip the parallel pass. Download items are
|
|
364
|
+
// collected into `downloadItems[]` for the pool pass below.
|
|
365
|
+
const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
|
|
366
|
+
let aborted = false;
|
|
367
|
+
let abortResult: SyncResult | null = null;
|
|
368
|
+
|
|
338
369
|
for (const item of plan.items) {
|
|
370
|
+
if (aborted) break;
|
|
339
371
|
if (
|
|
340
372
|
item.action === "skip-ignored" ||
|
|
341
373
|
item.action === "skip-personal-mode" ||
|
|
@@ -353,6 +385,11 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
353
385
|
continue;
|
|
354
386
|
}
|
|
355
387
|
|
|
388
|
+
if (item.action === "download") {
|
|
389
|
+
downloadItems.push(item);
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
|
|
356
393
|
const { remoteFile, localPath } = item;
|
|
357
394
|
|
|
358
395
|
// Auto-refresh context if credentials expiring (kept in execute phase
|
|
@@ -428,7 +465,8 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
428
465
|
if (resolution === "abort") {
|
|
429
466
|
emit({ type: "new-files", files: [] });
|
|
430
467
|
writeJournal(journalSlug, journal);
|
|
431
|
-
|
|
468
|
+
aborted = true;
|
|
469
|
+
abortResult = {
|
|
432
470
|
filesDownloaded,
|
|
433
471
|
bytesDownloaded,
|
|
434
472
|
filesSkipped,
|
|
@@ -443,6 +481,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
443
481
|
// destructure it.
|
|
444
482
|
filesTombstoned: 0,
|
|
445
483
|
};
|
|
484
|
+
break;
|
|
446
485
|
}
|
|
447
486
|
if (resolution === "keep" || resolution === "skip") {
|
|
448
487
|
filesSkipped++;
|
|
@@ -467,63 +506,157 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
467
506
|
item.localSize,
|
|
468
507
|
"down",
|
|
469
508
|
remoteFile.etag,
|
|
509
|
+
item.localMtime.getTime(),
|
|
470
510
|
);
|
|
471
511
|
continue;
|
|
472
512
|
}
|
|
473
|
-
// "overwrite" falls through to download
|
|
513
|
+
// "overwrite" falls through to download — re-route through the pool
|
|
514
|
+
// so it benefits from parallelism too. Synthesize a download item
|
|
515
|
+
// pointing at the same remoteFile/localPath; isNew=false because
|
|
516
|
+
// there was a conflict-eligible local file present.
|
|
517
|
+
downloadItems.push({
|
|
518
|
+
action: "download",
|
|
519
|
+
remoteFile,
|
|
520
|
+
localPath,
|
|
521
|
+
isNew: false,
|
|
522
|
+
});
|
|
523
|
+
continue;
|
|
474
524
|
}
|
|
525
|
+
}
|
|
475
526
|
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
527
|
+
// Early-return on conflict abort BEFORE running the parallel download
|
|
528
|
+
// pool — the abort intent is "stop the pull now", not "stop new work but
|
|
529
|
+
// finish what's in flight". Since the pool hasn't started yet, this is
|
|
530
|
+
// a clean drain (zero items in flight) by construction.
|
|
531
|
+
if (aborted && abortResult) {
|
|
532
|
+
return abortResult;
|
|
533
|
+
}
|
|
480
534
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
const localLstat = fs.lstatSync(localPath);
|
|
491
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
492
|
-
const hash = isLocalSymlink
|
|
493
|
-
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
494
|
-
: hashFile(localPath);
|
|
495
|
-
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
496
|
-
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
497
|
-
// drift independently of mtime drift.
|
|
498
|
-
updateEntry(journal, remoteFile.key, hash, size, "down", remoteFile.etag);
|
|
535
|
+
// ── Parallel download pool (5.36.0) ───────────────────────────────────
|
|
536
|
+
// Bounded concurrency: TRANSFER_CONCURRENCY simultaneous downloads. Same
|
|
537
|
+
// race-based shape as the HEAD-verify pool above but applied to the body
|
|
538
|
+
// of each download. Per-item progress events fire at file-settle time
|
|
539
|
+
// (inside the wrapper), so cross-file interleave is expected and the
|
|
540
|
+
// menubar's stream parser already handles it.
|
|
541
|
+
if (downloadItems.length > 0) {
|
|
542
|
+
const queue = [...downloadItems];
|
|
543
|
+
const inFlight: Set<Promise<unknown>> = new Set();
|
|
499
544
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
const
|
|
504
|
-
emit({
|
|
505
|
-
type: "progress",
|
|
506
|
-
path: remoteFile.key,
|
|
507
|
-
bytes: size,
|
|
508
|
-
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
509
|
-
...(author ? { author } : {}),
|
|
510
|
-
});
|
|
545
|
+
const downloadOne = async (
|
|
546
|
+
downloadItem: typeof downloadItems[number],
|
|
547
|
+
): Promise<void> => {
|
|
548
|
+
const { remoteFile, localPath } = downloadItem;
|
|
511
549
|
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
550
|
+
// Auto-refresh context if credentials expiring. Each task checks
|
|
551
|
+
// independently — refresh is idempotent on the same context object.
|
|
552
|
+
if (isExpiringSoon(ctx.expiresAt)) {
|
|
553
|
+
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
try {
|
|
557
|
+
const { metadata } = await downloadFile(ctx, remoteFile.key, localPath);
|
|
558
|
+
const author = metadata?.["created-by"] ?? null;
|
|
559
|
+
|
|
560
|
+
// Symlink records materialize as real symlinks on disk. lstat
|
|
561
|
+
// (does not follow) lets us detect that case so the journal stamp
|
|
562
|
+
// mirrors what the push side would emit on the next tick:
|
|
563
|
+
// hash = sha256(readlink target string)
|
|
564
|
+
// size = 0
|
|
565
|
+
// Without this check, hashFile would follow the link and stamp the
|
|
566
|
+
// target file's contents — a value the next push would never
|
|
567
|
+
// produce — which makes skipUnchanged perpetually re-upload every
|
|
568
|
+
// symlink, defeating the point of the gate.
|
|
569
|
+
const localLstat = fs.lstatSync(localPath);
|
|
570
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
571
|
+
const hash = isLocalSymlink
|
|
572
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
573
|
+
: hashFile(localPath);
|
|
574
|
+
const size = isLocalSymlink ? 0 : fs.statSync(localPath).size;
|
|
575
|
+
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
576
|
+
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
577
|
+
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
578
|
+
// SHA256 for this file without reading its bytes.
|
|
579
|
+
updateEntry(
|
|
580
|
+
journal,
|
|
581
|
+
remoteFile.key,
|
|
582
|
+
hash,
|
|
583
|
+
size,
|
|
584
|
+
"down",
|
|
585
|
+
remoteFile.etag,
|
|
586
|
+
localLstat.mtimeMs,
|
|
587
|
+
);
|
|
588
|
+
|
|
589
|
+
// Attach message from the prior journal entry if present (set by a
|
|
590
|
+
// previous `share` operation that included a --message).
|
|
591
|
+
const priorEntry = getEntry(journal, remoteFile.key);
|
|
592
|
+
const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
|
|
520
593
|
emit({
|
|
521
|
-
type: "
|
|
594
|
+
type: "progress",
|
|
522
595
|
path: remoteFile.key,
|
|
523
|
-
|
|
596
|
+
bytes: size,
|
|
597
|
+
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
598
|
+
...(author ? { author } : {}),
|
|
524
599
|
});
|
|
600
|
+
|
|
601
|
+
filesDownloaded++;
|
|
602
|
+
bytesDownloaded += size;
|
|
603
|
+
} catch (err) {
|
|
604
|
+
// STS session policy may deny access to some paths — this is expected
|
|
605
|
+
// for guest members with allowedPrefixes
|
|
606
|
+
if (isAccessDenied(err)) {
|
|
607
|
+
filesSkipped++;
|
|
608
|
+
} else {
|
|
609
|
+
emit({
|
|
610
|
+
type: "error",
|
|
611
|
+
path: remoteFile.key,
|
|
612
|
+
message: err instanceof Error ? err.message : String(err),
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
|
|
618
|
+
// Codex P1 (5.36.x): worker promises wrapped in .catch so an
|
|
619
|
+
// unhandled rejection inside downloadOne (e.g. refreshEntityContext
|
|
620
|
+
// before the per-item try/catch, or lstatSync after the download
|
|
621
|
+
// succeeded but before journal stamping) cannot escape
|
|
622
|
+
// `Promise.race(inFlight)` and unwind the drain mid-flight. Without
|
|
623
|
+
// this wrap, sibling downloads kept running after share()/sync()
|
|
624
|
+
// had already failed, their files materialized on disk without
|
|
625
|
+
// matching journal entries, and the next sync re-downloaded
|
|
626
|
+
// everything. Errors are collected and surfaced after the pool
|
|
627
|
+
// fully drains — see workerErrors throw below.
|
|
628
|
+
const workerErrors: Error[] = [];
|
|
629
|
+
while (queue.length > 0 || inFlight.size > 0) {
|
|
630
|
+
while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
|
|
631
|
+
const downloadItem = queue.shift()!;
|
|
632
|
+
const p: Promise<void> = downloadOne(downloadItem)
|
|
633
|
+
.catch((err: unknown) => {
|
|
634
|
+
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
635
|
+
})
|
|
636
|
+
.finally(() => {
|
|
637
|
+
inFlight.delete(p);
|
|
638
|
+
});
|
|
639
|
+
inFlight.add(p);
|
|
640
|
+
}
|
|
641
|
+
if (inFlight.size > 0) {
|
|
642
|
+
// Wait for at least one in-flight task to settle before topping up
|
|
643
|
+
// the pool. allSettled-style semantics via Promise.race — the
|
|
644
|
+
// .catch wrap above guarantees no worker promise can reject.
|
|
645
|
+
await Promise.race(Array.from(inFlight));
|
|
525
646
|
}
|
|
526
647
|
}
|
|
648
|
+
|
|
649
|
+
// Pool drained. If any worker rejected, write the journal first
|
|
650
|
+
// (so the lstat fast-path stamps for successfully-downloaded files
|
|
651
|
+
// persist) then throw the first error, preserving its stack.
|
|
652
|
+
if (workerErrors.length > 0) {
|
|
653
|
+
writeJournal(journalSlug, journal);
|
|
654
|
+
const first = workerErrors[0]!;
|
|
655
|
+
if (workerErrors.length > 1) {
|
|
656
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
657
|
+
}
|
|
658
|
+
throw first;
|
|
659
|
+
}
|
|
527
660
|
}
|
|
528
661
|
|
|
529
662
|
// ── New-files attribution (US-002) ─────────────────────────────────────
|
|
@@ -928,9 +1061,28 @@ function computePullPlan(
|
|
|
928
1061
|
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
929
1062
|
continue;
|
|
930
1063
|
}
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1064
|
+
// Fast-path (5.36.0): when the journal entry has an mtimeMs and the
|
|
1065
|
+
// local lstat matches both size and mtimeMs, the file hasn't been
|
|
1066
|
+
// touched locally since the last sync, so we can reuse the journal's
|
|
1067
|
+
// recorded hash without re-reading the file's bytes. Same shape as
|
|
1068
|
+
// the share-side fast-path; only kicks in for regular files (symlink
|
|
1069
|
+
// hashing through hashSymlinkTarget is already cheap — it hashes a
|
|
1070
|
+
// short target string, not file bytes).
|
|
1071
|
+
let localHash: string;
|
|
1072
|
+
if (
|
|
1073
|
+
!isLocalSymlink &&
|
|
1074
|
+
journalEntry &&
|
|
1075
|
+
journalEntry.mtimeMs !== undefined &&
|
|
1076
|
+
journalEntry.hash &&
|
|
1077
|
+
localLstat!.size === journalEntry.size &&
|
|
1078
|
+
localLstat!.mtimeMs === journalEntry.mtimeMs
|
|
1079
|
+
) {
|
|
1080
|
+
localHash = journalEntry.hash;
|
|
1081
|
+
} else {
|
|
1082
|
+
localHash = isLocalSymlink
|
|
1083
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1084
|
+
: hashFile(localPath);
|
|
1085
|
+
}
|
|
934
1086
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
935
1087
|
const remoteChanged =
|
|
936
1088
|
!!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
package/src/ignore.test.ts
CHANGED
|
@@ -98,6 +98,43 @@ describe("createIgnoreFilter", () => {
|
|
|
98
98
|
expect(shouldSync(path.join(hqRoot, ".hqinclude"))).toBe(true);
|
|
99
99
|
});
|
|
100
100
|
|
|
101
|
+
it("permissive mode: .claude/state/ and .claude/audit/ are per-host, never synced", () => {
|
|
102
|
+
// 5.34.0 live cross-machine sync test (macOS ↔ Lightsail) wrote
|
|
103
|
+
// ~25 of the 30 conflict mirrors directly to these paths. They're
|
|
104
|
+
// session-/host-scoped by design:
|
|
105
|
+
// - .claude/state/active-session-project: the active project
|
|
106
|
+
// pointer for the currently-foregrounded Claude Code session.
|
|
107
|
+
// - .claude/state/auto-session-project-<sessionUuid>: one
|
|
108
|
+
// marker per spawned session; entirely host-local.
|
|
109
|
+
// - .claude/audit/: hook-decision audit log + suppressions
|
|
110
|
+
// config; host-specific by nature.
|
|
111
|
+
// Syncing produces guaranteed conflicts per machine per session.
|
|
112
|
+
const shouldSync = createIgnoreFilter(hqRoot);
|
|
113
|
+
expect(shouldSync(path.join(hqRoot, ".claude/state/active-session-project"))).toBe(false);
|
|
114
|
+
expect(
|
|
115
|
+
shouldSync(
|
|
116
|
+
path.join(hqRoot, ".claude/state/auto-session-project-571177ff-59bb-4728-8670-0bf66a6410a5"),
|
|
117
|
+
),
|
|
118
|
+
).toBe(false);
|
|
119
|
+
expect(shouldSync(path.join(hqRoot, ".claude/audit/instructions.md"))).toBe(false);
|
|
120
|
+
expect(shouldSync(path.join(hqRoot, ".claude/audit/suppressions.yaml"))).toBe(false);
|
|
121
|
+
expect(shouldSync(path.join(hqRoot, ".claude/state"), true)).toBe(false);
|
|
122
|
+
expect(shouldSync(path.join(hqRoot, ".claude/audit"), true)).toBe(false);
|
|
123
|
+
// Nested .claude/ inside a company also stays local — same anchor
|
|
124
|
+
// shape as the .claude/worktrees/ rule directly above.
|
|
125
|
+
expect(
|
|
126
|
+
shouldSync(path.join(hqRoot, "companies/indigo/.claude/state/active-session-project")),
|
|
127
|
+
).toBe(false);
|
|
128
|
+
expect(
|
|
129
|
+
shouldSync(path.join(hqRoot, "companies/indigo/.claude/audit/instructions.md")),
|
|
130
|
+
).toBe(false);
|
|
131
|
+
// The .claude/ dir's other contents (settings, commands, skills,
|
|
132
|
+
// hooks) still sync — the rule MUST NOT broaden to .claude/.
|
|
133
|
+
expect(shouldSync(path.join(hqRoot, ".claude/settings.json"))).toBe(true);
|
|
134
|
+
expect(shouldSync(path.join(hqRoot, ".claude/skills/foo/SKILL.md"))).toBe(true);
|
|
135
|
+
expect(shouldSync(path.join(hqRoot, ".claude/hooks/foo.sh"))).toBe(true);
|
|
136
|
+
});
|
|
137
|
+
|
|
101
138
|
it("permissive mode: .hq/ directory is per-host state, never synced (Bug #1/#6/#8)", () => {
|
|
102
139
|
// Bug catalog: `.hq/install-manifest.json`, `.hq/machine-id`, and
|
|
103
140
|
// `.hq/machine.json` are per-host source-of-truth files. The original
|
package/src/ignore.ts
CHANGED
|
@@ -98,6 +98,20 @@ export const DEFAULT_IGNORES = [
|
|
|
98
98
|
// Claude Code worktrees — local-only working copies, never synced.
|
|
99
99
|
"**/.claude/worktrees/",
|
|
100
100
|
|
|
101
|
+
// Claude Code per-machine session state — every active session writes
|
|
102
|
+
// to `.claude/state/active-session-project` and
|
|
103
|
+
// `.claude/state/auto-session-project-<sessionUuid>` independently on
|
|
104
|
+
// each host. Syncing them produces a guaranteed conflict per machine
|
|
105
|
+
// per session (~25 of the 30 mirrors written during the 5.34.0 live-
|
|
106
|
+
// sync test traced here). The `.claude/audit/` siblings are also
|
|
107
|
+
// per-host (audit log of hook decisions, suppressions, instructions).
|
|
108
|
+
// Both classes are session-/host-scoped by design and must never
|
|
109
|
+
// round-trip. Anchored under `**/.claude/` so the same rule covers
|
|
110
|
+
// hqRoot, company sub-dirs, and embedded knowledge repos uniformly —
|
|
111
|
+
// mirrors the `**/.claude/worktrees/` pattern just above.
|
|
112
|
+
"**/.claude/state/",
|
|
113
|
+
"**/.claude/audit/",
|
|
114
|
+
|
|
101
115
|
// HQ repos directory (managed separately, not synced)
|
|
102
116
|
"repos/",
|
|
103
117
|
|