@indigoai-us/hq-cloud 6.11.12 → 6.11.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +17 -26
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +3 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +73 -1154
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +17 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +197 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +235 -677
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +412 -727
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +20 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/object-io.js +1 -1
- package/dist/object-io.js.map +1 -1
- package/dist/remote-pull.d.ts +2 -2
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +23 -3
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +24 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +6 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +32 -2
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +31 -0
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/sync-progress.d.ts +37 -0
- package/dist/sync-progress.d.ts.map +1 -0
- package/dist/sync-progress.js +104 -0
- package/dist/sync-progress.js.map +1 -0
- package/dist/sync-progress.test.d.ts +2 -0
- package/dist/sync-progress.test.d.ts.map +1 -0
- package/dist/sync-progress.test.js +70 -0
- package/dist/sync-progress.test.js.map +1 -0
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +2 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +3 -113
- package/dist/watcher.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.ts +96 -1253
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +20 -0
- package/src/cli/rescue-core.ts +252 -176
- package/src/cli/share.ts +374 -706
- package/src/cli/sync.test.ts +25 -0
- package/src/cli/sync.ts +625 -803
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +9 -0
- package/src/object-io.ts +1 -1
- package/src/remote-pull.test.ts +30 -1
- package/src/remote-pull.ts +29 -4
- package/src/sync/push-receiver.test.ts +35 -0
- package/src/sync/push-receiver.ts +41 -2
- package/src/sync-core.ts +58 -0
- package/src/sync-progress.test.ts +81 -0
- package/src/sync-progress.ts +133 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.ts +6 -141
package/src/cli/sync.ts
CHANGED
|
@@ -7,9 +7,14 @@
|
|
|
7
7
|
|
|
8
8
|
import * as fs from "fs";
|
|
9
9
|
import * as path from "path";
|
|
10
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
EntityContext,
|
|
12
|
+
VaultServiceConfig,
|
|
13
|
+
SyncJournal,
|
|
14
|
+
} from "../types.js";
|
|
11
15
|
import type { SyncMode } from "../vault-client.js";
|
|
12
16
|
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
17
|
+
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
13
18
|
import {
|
|
14
19
|
downloadFile,
|
|
15
20
|
listRemoteFiles,
|
|
@@ -49,6 +54,12 @@ import {
|
|
|
49
54
|
type ScopePrefixInput,
|
|
50
55
|
} from "../prefix-coalesce.js";
|
|
51
56
|
import { createIgnoreFilter } from "../ignore.js";
|
|
57
|
+
import {
|
|
58
|
+
hasRemoteChanged,
|
|
59
|
+
isAccessDenied,
|
|
60
|
+
resolveActiveCompany,
|
|
61
|
+
resolveTransferConcurrency,
|
|
62
|
+
} from "../sync-core.js";
|
|
52
63
|
import { isEphemeralPath, isMalformedVaultKey } from "./share.js";
|
|
53
64
|
import { resolveConflict } from "./conflict.js";
|
|
54
65
|
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
@@ -489,6 +500,52 @@ export interface SyncResult {
|
|
|
489
500
|
scopeOrphansBlocked: number;
|
|
490
501
|
}
|
|
491
502
|
|
|
503
|
+
type SyncEventEmitter = (event: SyncProgressEvent) => void;
|
|
504
|
+
|
|
505
|
+
type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
|
|
506
|
+
|
|
507
|
+
interface PullRunContext {
|
|
508
|
+
options: SyncOptions;
|
|
509
|
+
companyRef: string;
|
|
510
|
+
vaultConfig: VaultServiceConfig;
|
|
511
|
+
hqRoot: string;
|
|
512
|
+
emit: SyncEventEmitter;
|
|
513
|
+
ctx: EntityContext;
|
|
514
|
+
companyRoot: string;
|
|
515
|
+
shouldSync: (filePath: string, isDir?: boolean) => boolean;
|
|
516
|
+
journalSlug: string;
|
|
517
|
+
startedAt: string;
|
|
518
|
+
journal: SyncJournal;
|
|
519
|
+
remoteFiles: RemoteFile[];
|
|
520
|
+
syncMode: SyncMode;
|
|
521
|
+
currentPrefixSet: string[];
|
|
522
|
+
fileTombstones: ReadonlyMap<string, CompanyTombstone>;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
interface PullCounters {
|
|
526
|
+
filesDownloaded: number;
|
|
527
|
+
bytesDownloaded: number;
|
|
528
|
+
filesSkipped: number;
|
|
529
|
+
conflicts: number;
|
|
530
|
+
filesTombstoned: number;
|
|
531
|
+
filesOutOfScope: number;
|
|
532
|
+
conflictPaths: string[];
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
interface ScopeShrinkPlanState {
|
|
536
|
+
lastRecord: ReturnType<typeof lastPullRecord>;
|
|
537
|
+
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
538
|
+
autoRecover: boolean;
|
|
539
|
+
adviceContext: ScopeShrinkAdviceContext;
|
|
540
|
+
effectiveForce: boolean;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
interface ScopeShrinkRun {
|
|
544
|
+
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
545
|
+
shrinkResult: ReturnType<typeof applyScopeShrink>;
|
|
546
|
+
scopeOrphansRemoved: number;
|
|
547
|
+
}
|
|
548
|
+
|
|
492
549
|
/**
|
|
493
550
|
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
494
551
|
* scope shrink that would delete more than this many CLEAN local files in one
|
|
@@ -616,10 +673,55 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
|
616
673
|
async function syncWithOperationLockHeld(
|
|
617
674
|
options: SyncOptions,
|
|
618
675
|
): Promise<SyncResult> {
|
|
619
|
-
const
|
|
620
|
-
const
|
|
676
|
+
const run = await buildPullContext(options);
|
|
677
|
+
const plan = planPull(run);
|
|
678
|
+
|
|
679
|
+
emitPullPlan(run.emit, plan);
|
|
680
|
+
|
|
681
|
+
const scopePlan = planScopeShrink(run);
|
|
682
|
+
const scopeRun = executeScopeShrink(run, scopePlan);
|
|
683
|
+
const counters = createPullCounters();
|
|
684
|
+
|
|
685
|
+
const transferConcurrency = resolveTransferConcurrency();
|
|
686
|
+
const conflictRun = await executeConflictExecutor(
|
|
687
|
+
run,
|
|
688
|
+
plan,
|
|
689
|
+
scopeRun,
|
|
690
|
+
counters,
|
|
691
|
+
);
|
|
692
|
+
if (conflictRun.abortResult) {
|
|
693
|
+
return conflictRun.abortResult;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
await executeDownloadExecutor(
|
|
697
|
+
run,
|
|
698
|
+
conflictRun.downloadItems,
|
|
699
|
+
transferConcurrency,
|
|
700
|
+
counters,
|
|
701
|
+
);
|
|
702
|
+
|
|
703
|
+
await emitAndReportNewFiles(run, plan);
|
|
704
|
+
await verifyPlannedJournalTombstones(run, plan);
|
|
705
|
+
executeJournalTombstoneDeletes(run, plan, counters);
|
|
706
|
+
|
|
707
|
+
return finalizePullRun(run, plan, scopeRun, counters);
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
|
|
711
|
+
const { company, vaultConfig, hqRoot } = options;
|
|
712
|
+
const baseEmit = options.onEvent ?? defaultConsoleLogger;
|
|
713
|
+
// Mirror every progress event into the shared cross-process snapshot so the
|
|
714
|
+
// menubar shows live progress for THIS sync regardless of who launched it
|
|
715
|
+
// (auto-sync / Sync Now / CLI). Best-effort; never affects the sync.
|
|
716
|
+
const recordProgress = createSyncProgressRecorder({
|
|
717
|
+
company: company ?? null,
|
|
718
|
+
phase: "pull",
|
|
719
|
+
});
|
|
720
|
+
const emit: SyncEventEmitter = (event) => {
|
|
721
|
+
baseEmit(event);
|
|
722
|
+
recordProgress(event);
|
|
723
|
+
};
|
|
621
724
|
|
|
622
|
-
// Resolve company
|
|
623
725
|
const companyRef = company ?? resolveActiveCompany(hqRoot);
|
|
624
726
|
if (!companyRef) {
|
|
625
727
|
throw new Error(
|
|
@@ -628,184 +730,131 @@ async function syncWithOperationLockHeld(
|
|
|
628
730
|
);
|
|
629
731
|
}
|
|
630
732
|
|
|
631
|
-
|
|
632
|
-
let ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
633
|
-
// Every company's files land under companies/{slug}/ so fanning out multiple
|
|
634
|
-
// companies into the same hqRoot doesn't cross-clobber files with overlapping
|
|
635
|
-
// S3 keys (e.g. every company has a .hq/manifest.json). Remote keys stay
|
|
636
|
-
// company-relative; the prefix lives only on disk.
|
|
637
|
-
// In personalMode the journal slug + S3 keys are person-relative (e.g. "docs/foo.md");
|
|
638
|
-
// the local target is `hqRoot` directly, NOT `<hqRoot>/companies/<personSlug>/`. This
|
|
639
|
-
// keeps round-trip parity with the Rust personal first-push (Step 7) which sources
|
|
640
|
-
// `<hqRoot>/docs/foo.md`.
|
|
733
|
+
const ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
641
734
|
const companyRoot = options.personalMode === true
|
|
642
735
|
? hqRoot
|
|
643
736
|
: path.join(hqRoot, "companies", ctx.slug);
|
|
644
737
|
const shouldSync = createIgnoreFilter(hqRoot);
|
|
645
738
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
646
739
|
const startedAt = new Date().toISOString();
|
|
647
|
-
|
|
648
|
-
// the legacy `personal` file exists (mass re-download/etag churn). Seeding
|
|
649
|
-
// here — inside the engine — covers every consumer (sync-runner already
|
|
650
|
-
// seeds; hq-cli historically didn't, which split the vault's bookkeeping
|
|
651
|
-
// across two journal files and re-flagged synced files as conflicts).
|
|
740
|
+
|
|
652
741
|
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
|
|
653
|
-
// Migrate v1 → v2 in place so the scope-shrink / pull-record machinery has
|
|
654
|
-
// its fields, and GC any tombstones past the 30-day retention window before
|
|
655
|
-
// we re-evaluate orphans (so a long-pruned path can re-download cleanly).
|
|
656
742
|
const journal = migrateToV2(readJournal(journalSlug));
|
|
657
743
|
gcTombstones(journal, Date.now());
|
|
658
744
|
|
|
659
|
-
// ── Effective download scope (US-005) ─────────────────────────────────────
|
|
660
|
-
// `all` → prefixSet `[""]`, which `isCoveredByAny` treats as "covers
|
|
661
|
-
// everything" — so the download filter and the scope-shrink
|
|
662
|
-
// comparison both become no-ops, preserving legacy full-bucket
|
|
663
|
-
// behavior bit-for-bit.
|
|
664
|
-
// `shared`/`custom` → the coalesced, company-relative prefix set the runner
|
|
665
|
-
// resolved. An empty set means "nothing in scope" → download
|
|
666
|
-
// nothing (the runner falls back to `all` on resolution errors, so
|
|
667
|
-
// empty here is an intentional "nothing shared", never a failure).
|
|
668
745
|
const syncMode: SyncMode = options.syncMode ?? "all";
|
|
669
|
-
const currentPrefixSet =
|
|
746
|
+
const currentPrefixSet: string[] =
|
|
670
747
|
syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
|
|
671
|
-
// Authorship guard input (scope-shrink): the caller's own Cognito sub,
|
|
672
|
-
// injected by the entry point (the runner sources it from its decoded
|
|
673
|
-
// idToken claims — the same sub stamped onto uploads as `created-by-sub`).
|
|
674
|
-
// Undefined degrades safely: own-author files lose their special shield, but
|
|
675
|
-
// the `protectUnknownAuthors` conservative path below still prevents a
|
|
676
|
-
// routine sync from deleting anything it can't prove is foreign.
|
|
677
|
-
const callerSub = options.callerSub;
|
|
678
|
-
|
|
679
|
-
let filesDownloaded = 0;
|
|
680
|
-
let bytesDownloaded = 0;
|
|
681
|
-
let filesSkipped = 0;
|
|
682
|
-
let conflicts = 0;
|
|
683
|
-
let filesTombstoned = 0;
|
|
684
|
-
let filesOutOfScope = 0;
|
|
685
|
-
const conflictPaths: string[] = [];
|
|
686
748
|
|
|
687
|
-
// List all remote files (IAM session policy filters at the AWS layer)
|
|
688
749
|
const remoteFiles = await listRemoteFiles(ctx);
|
|
689
|
-
|
|
690
|
-
// Fetch the company's FILE_TOMBSTONE records so the planner can suppress
|
|
691
|
-
// resurrection of an intentionally-deleted object (delete-resync). Done in
|
|
692
|
-
// parallel intent with the LIST above conceptually, but kept serial here for
|
|
693
|
-
// a clean read of `ctx`; best-effort — a failed read degrades to an empty map
|
|
694
|
-
// (no suppression), preserving the pre-fix behavior. ctx.uid is the verified
|
|
695
|
-
// companyUid the tombstone rows are keyed under.
|
|
696
|
-
//
|
|
697
|
-
// SKIP for the personal vault: its `ctx.uid` is a personUid (`prs_…`), but
|
|
698
|
-
// `GET /v1/files/tombstones?company=…` is COMPANY-scoped server-side
|
|
699
|
-
// (findCallerWithMembership), so a personal-vault request resolves
|
|
700
|
-
// `company=prs_…` to no membership and is correctly rejected with
|
|
701
|
-
// `403 "No active membership for caller in company prs_…"`. That 403 is
|
|
702
|
-
// benign for the pull (it already degrades to the empty map below), but
|
|
703
|
-
// hq-pro captures EVERY one as a Sentry warning — the per-personal-vault
|
|
704
|
-
// no-membership cluster (one Sentry issue per signed-in user). Personal-vault
|
|
705
|
-
// delete-resync was never a committed feature and there is no person-scoped
|
|
706
|
-
// tombstone path, so for the personal target we skip the fetch and use an
|
|
707
|
-
// empty map — byte-for-byte the current degraded behavior, minus the 403 spam.
|
|
708
|
-
// FUTURE FOLLOW-UP (not built here): if personal-vault delete-resync is
|
|
709
|
-
// wanted, it needs a real person-scoped tombstone endpoint + client read.
|
|
710
|
-
const tombstones =
|
|
750
|
+
const fileTombstones =
|
|
711
751
|
options.personalMode === true
|
|
712
752
|
? new Map<string, CompanyTombstone>()
|
|
713
753
|
: await fetchCompanyTombstones(vaultConfig, ctx.uid);
|
|
714
754
|
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
755
|
+
return {
|
|
756
|
+
options,
|
|
757
|
+
companyRef,
|
|
758
|
+
vaultConfig,
|
|
759
|
+
hqRoot,
|
|
760
|
+
emit,
|
|
761
|
+
ctx,
|
|
721
762
|
companyRoot,
|
|
722
763
|
shouldSync,
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
764
|
+
journalSlug,
|
|
765
|
+
startedAt,
|
|
766
|
+
journal,
|
|
767
|
+
remoteFiles,
|
|
768
|
+
syncMode,
|
|
726
769
|
currentPrefixSet,
|
|
727
|
-
|
|
770
|
+
fileTombstones,
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
function planPull(run: PullRunContext): PullPlan {
|
|
775
|
+
return computePullPlan(
|
|
776
|
+
run.remoteFiles,
|
|
777
|
+
run.journal,
|
|
778
|
+
run.companyRoot,
|
|
779
|
+
run.shouldSync,
|
|
780
|
+
run.options.personalMode === true,
|
|
781
|
+
run.options.includeLocalCompanies === true,
|
|
782
|
+
run.options.teamSyncedSlugs ?? null,
|
|
783
|
+
run.currentPrefixSet,
|
|
784
|
+
run.fileTombstones,
|
|
728
785
|
);
|
|
786
|
+
}
|
|
729
787
|
|
|
788
|
+
function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
|
|
730
789
|
emit({
|
|
731
790
|
type: "plan",
|
|
732
791
|
filesToDownload: plan.filesToDownload,
|
|
733
792
|
bytesToDownload: plan.bytesToDownload,
|
|
734
|
-
// sync() is pull-only; push counts are sourced from share()'s plan event.
|
|
735
793
|
filesToUpload: 0,
|
|
736
794
|
bytesToUpload: 0,
|
|
737
795
|
filesToSkip: plan.filesToSkip,
|
|
738
796
|
filesToConflict: plan.filesToConflict,
|
|
739
|
-
// Authoritative FILE_TOMBSTONE suppressions (delete-resync) are the only
|
|
740
|
-
// deletes known at plan time; the journal-vs-LIST tombstones are
|
|
741
|
-
// HEAD-verified later and surfaced via the final filesTombstoned count.
|
|
742
797
|
filesToDelete: plan.filesToTombstoneDelete,
|
|
743
798
|
});
|
|
799
|
+
}
|
|
744
800
|
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
// AWS layer never narrows the pull. This client-side shrink is what makes
|
|
760
|
-
// `hq sync mode shared` actually stick across re-syncs for an owner.
|
|
761
|
-
const lastRecord = lastPullRecord(journal, ctx.uid);
|
|
762
|
-
// A missing record, or a v1-migrated record with an empty prefixSet, means
|
|
763
|
-
// "no recorded scope" → treat the last scope as full-bucket `all` (`[""]`),
|
|
764
|
-
// per the PullRecord.prefixSet contract in types.ts.
|
|
801
|
+
function createPullCounters(): PullCounters {
|
|
802
|
+
return {
|
|
803
|
+
filesDownloaded: 0,
|
|
804
|
+
bytesDownloaded: 0,
|
|
805
|
+
filesSkipped: 0,
|
|
806
|
+
conflicts: 0,
|
|
807
|
+
filesTombstoned: 0,
|
|
808
|
+
filesOutOfScope: 0,
|
|
809
|
+
conflictPaths: [],
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
|
|
814
|
+
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
765
815
|
const lastPrefixSet =
|
|
766
816
|
lastRecord && lastRecord.prefixSet.length > 0
|
|
767
817
|
? lastRecord.prefixSet
|
|
768
818
|
: [""];
|
|
769
819
|
const shrinkPlan = buildScopeShrinkPlan({
|
|
770
|
-
journal,
|
|
771
|
-
hqRoot: companyRoot,
|
|
820
|
+
journal: run.journal,
|
|
821
|
+
hqRoot: run.companyRoot,
|
|
772
822
|
lastPrefixSet,
|
|
773
|
-
currentPrefixSet,
|
|
774
|
-
callerSub,
|
|
775
|
-
// Automatic pull: never auto-prune content the caller authored, and never
|
|
776
|
-
// make a destructive guess about unknown-author (legacy) orphans. The
|
|
777
|
-
// explicit `hq sync narrow` ritual opts out of the unknown-author shield.
|
|
823
|
+
currentPrefixSet: run.currentPrefixSet,
|
|
824
|
+
callerSub: run.options.callerSub,
|
|
778
825
|
protectUnknownAuthors: true,
|
|
779
826
|
});
|
|
780
|
-
|
|
781
|
-
// interactive flag, so it must never throw on a shrink — it self-heals
|
|
782
|
-
// non-destructively (dirty kept on disk + un-tracked, clean quarantined).
|
|
783
|
-
// A foreground `hq sync` ("block", the default) keeps the protective gate
|
|
784
|
-
// but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
|
|
785
|
-
// bypasses the bulk-prune cap (quarantine is non-destructive, so a large
|
|
786
|
-
// recovery move is safe). DEV-1768.
|
|
787
|
-
const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
|
|
827
|
+
const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
|
|
788
828
|
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
789
829
|
const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
|
|
790
|
-
const effectiveForce = options.forceScopeShrink === true || autoRecover;
|
|
830
|
+
const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
|
|
831
|
+
|
|
832
|
+
return {
|
|
833
|
+
lastRecord,
|
|
834
|
+
shrinkPlan,
|
|
835
|
+
autoRecover,
|
|
836
|
+
adviceContext,
|
|
837
|
+
effectiveForce,
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
function executeScopeShrink(
|
|
842
|
+
run: PullRunContext,
|
|
843
|
+
scopePlan: ScopeShrinkPlanState,
|
|
844
|
+
): ScopeShrinkRun {
|
|
845
|
+
const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
|
|
791
846
|
|
|
792
847
|
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
793
848
|
throw new ScopeShrinkBlockedError(
|
|
794
|
-
ctx.uid,
|
|
849
|
+
run.ctx.uid,
|
|
795
850
|
lastRecord?.syncMode ?? "unknown",
|
|
796
|
-
syncMode,
|
|
851
|
+
run.syncMode,
|
|
797
852
|
shrinkPlan.dirty,
|
|
798
853
|
shrinkPlan.clean,
|
|
799
854
|
adviceContext,
|
|
800
855
|
);
|
|
801
856
|
}
|
|
802
|
-
|
|
803
|
-
// a single foreground sync. A deliberate large narrow goes through
|
|
804
|
-
// `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
|
|
805
|
-
// raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
|
|
806
|
-
// under auto-recover — quarantine is non-destructive so a big recovery is
|
|
807
|
-
// safe, and the runner has no way to act on a thrown cap. The engine moves
|
|
808
|
-
// nothing when it throws here.
|
|
857
|
+
|
|
809
858
|
const autoPruneCap = resolveAutoPruneCap();
|
|
810
859
|
if (
|
|
811
860
|
!effectiveForce &&
|
|
@@ -813,40 +862,32 @@ async function syncWithOperationLockHeld(
|
|
|
813
862
|
shrinkPlan.clean.length > autoPruneCap
|
|
814
863
|
) {
|
|
815
864
|
throw new ScopeShrinkLargePruneError(
|
|
816
|
-
ctx.uid,
|
|
817
|
-
syncMode,
|
|
865
|
+
run.ctx.uid,
|
|
866
|
+
run.syncMode,
|
|
818
867
|
shrinkPlan.clean.length,
|
|
819
868
|
autoPruneCap,
|
|
820
869
|
adviceContext,
|
|
821
870
|
);
|
|
822
871
|
}
|
|
823
|
-
|
|
824
|
-
// recoverable), never silently deleted — a background sync purging local
|
|
825
|
-
// files unannounced was DEV-1768 fix #3. The quarantine root lives under the
|
|
826
|
-
// real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
|
|
827
|
-
// files don't round-trip back through S3.
|
|
872
|
+
|
|
828
873
|
const scopeQuarantineRoot = path.join(
|
|
829
|
-
hqRoot,
|
|
874
|
+
run.hqRoot,
|
|
830
875
|
".hq",
|
|
831
876
|
"scope-quarantine",
|
|
832
|
-
journalSlug,
|
|
877
|
+
run.journalSlug,
|
|
833
878
|
);
|
|
834
879
|
const shrinkResult = applyScopeShrink({
|
|
835
|
-
journal,
|
|
880
|
+
journal: run.journal,
|
|
836
881
|
plan: shrinkPlan,
|
|
837
|
-
hqRoot: companyRoot,
|
|
882
|
+
hqRoot: run.companyRoot,
|
|
838
883
|
forceScopeShrink: effectiveForce,
|
|
839
884
|
reason: "scope_shrink",
|
|
840
885
|
cleanDisposition: "quarantine",
|
|
841
886
|
quarantineRoot: scopeQuarantineRoot,
|
|
842
887
|
});
|
|
843
|
-
|
|
844
|
-
// silent. Quarantined clean files render as `deleted: true` (removed from the
|
|
845
|
-
// working tree, recoverable in quarantine); dirty files KEPT on disk render
|
|
846
|
-
// as a non-deletion notice so the operator knows they were un-tracked, not
|
|
847
|
-
// removed. The Rust menubar parser already handles `deleted: true`.
|
|
888
|
+
|
|
848
889
|
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
849
|
-
emit({
|
|
890
|
+
run.emit({
|
|
850
891
|
type: "progress",
|
|
851
892
|
path: relPath,
|
|
852
893
|
bytes: 0,
|
|
@@ -855,7 +896,7 @@ async function syncWithOperationLockHeld(
|
|
|
855
896
|
});
|
|
856
897
|
}
|
|
857
898
|
for (const relPath of shrinkResult.removedPaths) {
|
|
858
|
-
emit({
|
|
899
|
+
run.emit({
|
|
859
900
|
type: "progress",
|
|
860
901
|
path: relPath,
|
|
861
902
|
bytes: 0,
|
|
@@ -864,7 +905,7 @@ async function syncWithOperationLockHeld(
|
|
|
864
905
|
});
|
|
865
906
|
}
|
|
866
907
|
for (const relPath of shrinkResult.dirtyKeptPaths) {
|
|
867
|
-
emit({
|
|
908
|
+
run.emit({
|
|
868
909
|
type: "progress",
|
|
869
910
|
path: relPath,
|
|
870
911
|
bytes: 0,
|
|
@@ -872,547 +913,410 @@ async function syncWithOperationLockHeld(
|
|
|
872
913
|
"scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
|
|
873
914
|
});
|
|
874
915
|
}
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
shrinkResult
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
const TRANSFER_CONCURRENCY = (() => {
|
|
900
|
-
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
901
|
-
if (raw === undefined || raw === "") return 16;
|
|
902
|
-
const parsed = Number.parseInt(raw, 10);
|
|
903
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
904
|
-
})();
|
|
905
|
-
|
|
906
|
-
// First pass: serial walk for non-download outcomes (skips + conflicts).
|
|
907
|
-
// Conflicts may set `aborted = true` and short-circuit the whole pull;
|
|
908
|
-
// we detect that and skip the parallel pass. Download items are
|
|
909
|
-
// collected into `downloadItems[]` for the pool pass below.
|
|
910
|
-
const downloadItems: Array<typeof plan.items[number] & { action: "download" }> = [];
|
|
911
|
-
let aborted = false;
|
|
912
|
-
let abortResult: SyncResult | null = null;
|
|
916
|
+
|
|
917
|
+
return {
|
|
918
|
+
shrinkPlan,
|
|
919
|
+
shrinkResult,
|
|
920
|
+
scopeOrphansRemoved:
|
|
921
|
+
shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
|
|
922
|
+
};
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
async function refreshRunContextIfExpiring(
|
|
926
|
+
run: PullRunContext,
|
|
927
|
+
): Promise<void> {
|
|
928
|
+
if (isExpiringSoon(run.ctx.expiresAt)) {
|
|
929
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
async function executeConflictExecutor(
|
|
934
|
+
run: PullRunContext,
|
|
935
|
+
plan: PullPlan,
|
|
936
|
+
scopeRun: ScopeShrinkRun,
|
|
937
|
+
counters: PullCounters,
|
|
938
|
+
): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
|
|
939
|
+
const downloadItems: PullDownloadItem[] = [];
|
|
913
940
|
|
|
914
941
|
for (const item of plan.items) {
|
|
915
|
-
if (aborted) break;
|
|
916
942
|
if (
|
|
917
943
|
item.action === "skip-ignored" ||
|
|
918
944
|
item.action === "skip-personal-mode" ||
|
|
919
945
|
item.action === "skip-unchanged" ||
|
|
920
946
|
item.action === "skip-local-only"
|
|
921
947
|
) {
|
|
922
|
-
filesSkipped++;
|
|
948
|
+
counters.filesSkipped++;
|
|
923
949
|
continue;
|
|
924
950
|
}
|
|
925
951
|
if (item.action === "skip-excluded-policy") {
|
|
926
|
-
// Policy-excluded items count separately from `filesSkipped` so the
|
|
927
|
-
// pull result mirrors the push side's `filesExcludedByPolicy`
|
|
928
|
-
// counter — `filesSkipped` stays a measure of "unchanged on this
|
|
929
|
-
// run", not a catch-all for everything we didn't download.
|
|
930
952
|
continue;
|
|
931
953
|
}
|
|
932
954
|
if (item.action === "skip-out-of-scope") {
|
|
933
|
-
|
|
934
|
-
// axis so `filesSkipped` keeps meaning "unchanged on this run" — these
|
|
935
|
-
// are "deliberately not downloaded because of your sync scope".
|
|
936
|
-
filesOutOfScope++;
|
|
955
|
+
counters.filesOutOfScope++;
|
|
937
956
|
continue;
|
|
938
957
|
}
|
|
939
|
-
|
|
940
958
|
if (item.action === "tombstone-delete") {
|
|
941
|
-
|
|
942
|
-
// is present but a tombstone marks the key intentionally deleted and it is
|
|
943
|
-
// not a newer re-create. Delete any local copy and drop the journal entry
|
|
944
|
-
// so it stays gone — the mirror of the journal-vs-LIST tombstone executor
|
|
945
|
-
// below, but WITHOUT the HEAD-verify (the remote object is present by
|
|
946
|
-
// definition; the FILE_TOMBSTONE is the deletion authority). The planner
|
|
947
|
-
// already routed any divergent local copy to `conflict`, so a local file
|
|
948
|
-
// reaching here matches the deleted baseline and is safe to remove.
|
|
949
|
-
const tombstoneKey = item.remoteFile.key;
|
|
950
|
-
// Same Windows-backslash landmine guard as the journal-tombstone executor:
|
|
951
|
-
// a malformed key must never reach fs.unlinkSync (path.join collapses the
|
|
952
|
-
// backslashes onto a REAL POSIX file). Traversal keys are likewise
|
|
953
|
-
// refused before any local filesystem or journal mutation.
|
|
954
|
-
const tombstonePath = resolveContainedVaultPath(companyRoot, tombstoneKey);
|
|
955
|
-
if (tombstonePath === null) continue;
|
|
956
|
-
try {
|
|
957
|
-
const lstat = fs.lstatSync(tombstonePath);
|
|
958
|
-
if (tombstoneTargetDiverged(journal, tombstoneKey, tombstonePath, lstat)) {
|
|
959
|
-
continue;
|
|
960
|
-
}
|
|
961
|
-
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
962
|
-
fs.unlinkSync(tombstonePath);
|
|
963
|
-
}
|
|
964
|
-
// A directory at the key: don't recursively rm-rf the operator's dir;
|
|
965
|
-
// just drop the journal entry (safe-by-default, same as the other path).
|
|
966
|
-
} catch (err: unknown) {
|
|
967
|
-
const code =
|
|
968
|
-
err && typeof err === "object" && "code" in err
|
|
969
|
-
? (err as { code?: string }).code
|
|
970
|
-
: undefined;
|
|
971
|
-
// ENOENT → local already absent (the common case: a fresh machine that
|
|
972
|
-
// never held the file, or a prior pull already removed it) → drop the
|
|
973
|
-
// journal entry and converge. Other errors (EACCES/EPERM/…) leave the
|
|
974
|
-
// file in place; surface and KEEP the journal entry so the next sync
|
|
975
|
-
// retries rather than forgetting the delete.
|
|
976
|
-
if (code !== "ENOENT") {
|
|
977
|
-
emit({
|
|
978
|
-
type: "error",
|
|
979
|
-
path: tombstoneKey,
|
|
980
|
-
message: `tombstone-suppress unlink failed: ${
|
|
981
|
-
err instanceof Error ? err.message : String(err)
|
|
982
|
-
}`,
|
|
983
|
-
});
|
|
984
|
-
continue;
|
|
985
|
-
}
|
|
986
|
-
}
|
|
987
|
-
removeEntry(journal, tombstoneKey);
|
|
988
|
-
filesTombstoned++;
|
|
989
|
-
emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
959
|
+
executeFileTombstoneDelete(run, item, counters);
|
|
990
960
|
continue;
|
|
991
961
|
}
|
|
992
|
-
|
|
993
962
|
if (item.action === "download") {
|
|
994
963
|
downloadItems.push(item);
|
|
995
964
|
continue;
|
|
996
965
|
}
|
|
997
966
|
|
|
998
|
-
const
|
|
967
|
+
const abortResult = await executeConflictItem(
|
|
968
|
+
run,
|
|
969
|
+
plan,
|
|
970
|
+
scopeRun,
|
|
971
|
+
counters,
|
|
972
|
+
downloadItems,
|
|
973
|
+
item,
|
|
974
|
+
);
|
|
975
|
+
if (abortResult) {
|
|
976
|
+
return { downloadItems, abortResult };
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
return { downloadItems, abortResult: null };
|
|
981
|
+
}
|
|
999
982
|
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
983
|
+
function executeFileTombstoneDelete(
|
|
984
|
+
run: PullRunContext,
|
|
985
|
+
item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
|
|
986
|
+
counters: PullCounters,
|
|
987
|
+
): void {
|
|
988
|
+
const tombstoneKey = item.remoteFile.key;
|
|
989
|
+
const tombstonePath = resolveContainedVaultPath(run.companyRoot, tombstoneKey);
|
|
990
|
+
if (tombstonePath === null) return;
|
|
991
|
+
try {
|
|
992
|
+
const lstat = fs.lstatSync(tombstonePath);
|
|
993
|
+
if (tombstoneTargetDiverged(run.journal, tombstoneKey, tombstonePath, lstat)) {
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
997
|
+
fs.unlinkSync(tombstonePath);
|
|
1004
998
|
}
|
|
999
|
+
} catch (err: unknown) {
|
|
1000
|
+
const code =
|
|
1001
|
+
err && typeof err === "object" && "code" in err
|
|
1002
|
+
? (err as { code?: string }).code
|
|
1003
|
+
: undefined;
|
|
1004
|
+
if (code !== "ENOENT") {
|
|
1005
|
+
run.emit({
|
|
1006
|
+
type: "error",
|
|
1007
|
+
path: tombstoneKey,
|
|
1008
|
+
message: `tombstone-suppress unlink failed: ${
|
|
1009
|
+
err instanceof Error ? err.message : String(err)
|
|
1010
|
+
}`,
|
|
1011
|
+
});
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
removeEntry(run.journal, tombstoneKey);
|
|
1016
|
+
counters.filesTombstoned++;
|
|
1017
|
+
run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
1018
|
+
}
|
|
1005
1019
|
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
);
|
|
1039
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
1040
|
-
const conflictKey = toPosixKey(path.relative(companyRoot, conflictAbs));
|
|
1020
|
+
async function executeConflictItem(
|
|
1021
|
+
run: PullRunContext,
|
|
1022
|
+
plan: PullPlan,
|
|
1023
|
+
scopeRun: ScopeShrinkRun,
|
|
1024
|
+
counters: PullCounters,
|
|
1025
|
+
downloadItems: PullDownloadItem[],
|
|
1026
|
+
item: Extract<PullPlanItem, { action: "conflict" }>,
|
|
1027
|
+
): Promise<SyncResult | null> {
|
|
1028
|
+
const { remoteFile, localPath } = item;
|
|
1029
|
+
|
|
1030
|
+
await refreshRunContextIfExpiring(run);
|
|
1031
|
+
|
|
1032
|
+
const detectedAt = new Date().toISOString();
|
|
1033
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
1034
|
+
const originalRelative = path.relative(run.hqRoot, localPath);
|
|
1035
|
+
const conflictRelative = buildConflictPath(
|
|
1036
|
+
originalRelative,
|
|
1037
|
+
detectedAt,
|
|
1038
|
+
machineId,
|
|
1039
|
+
);
|
|
1040
|
+
const conflictAbs = path.join(run.hqRoot, conflictRelative);
|
|
1041
|
+
const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
|
|
1042
|
+
|
|
1043
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
|
|
1044
|
+
counters.filesSkipped++;
|
|
1045
|
+
run.emit({
|
|
1046
|
+
type: "error",
|
|
1047
|
+
path: remoteFile.key,
|
|
1048
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
1049
|
+
});
|
|
1050
|
+
return null;
|
|
1051
|
+
}
|
|
1041
1052
|
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1053
|
+
let remoteFetched = false;
|
|
1054
|
+
let converged = false;
|
|
1055
|
+
try {
|
|
1056
|
+
const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
|
|
1057
|
+
remoteFetched = true;
|
|
1058
|
+
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
1059
|
+
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
1060
|
+
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
1061
|
+
converged = remoteHash === item.localHash;
|
|
1062
|
+
} catch (probeErr) {
|
|
1063
|
+
run.emit({
|
|
1064
|
+
type: "error",
|
|
1065
|
+
path: remoteFile.key,
|
|
1066
|
+
message: `conflict convergence probe failed: ${
|
|
1067
|
+
probeErr instanceof Error ? probeErr.message : String(probeErr)
|
|
1068
|
+
}`,
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
1051
1071
|
|
|
1052
|
-
|
|
1053
|
-
|
|
1072
|
+
if (converged) {
|
|
1073
|
+
if (remoteFetched) {
|
|
1054
1074
|
try {
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
// (symlink-aware) so the two hashes are directly comparable. A
|
|
1059
|
-
// symlink record round-trips to a symlink on disk; hashing its
|
|
1060
|
-
// target string matches `hashSymlinkTarget(localPath)`.
|
|
1061
|
-
const remoteHash = fs.lstatSync(conflictAbs).isSymbolicLink()
|
|
1062
|
-
? hashSymlinkTarget(fs.readlinkSync(conflictAbs))
|
|
1063
|
-
: (downloaded.contentHash ?? hashFile(conflictAbs));
|
|
1064
|
-
converged = remoteHash === item.localHash;
|
|
1065
|
-
} catch (probeErr) {
|
|
1066
|
-
// Couldn't fetch or hash the remote — fail safe by falling through to
|
|
1067
|
-
// the conventional conflict path (converged stays false). No mirror
|
|
1068
|
-
// is on disk in this case.
|
|
1069
|
-
emit({
|
|
1070
|
-
type: "error",
|
|
1071
|
-
path: remoteFile.key,
|
|
1072
|
-
message: `conflict convergence probe failed: ${
|
|
1073
|
-
probeErr instanceof Error ? probeErr.message : String(probeErr)
|
|
1074
|
-
}`,
|
|
1075
|
-
});
|
|
1075
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
1076
|
+
} catch {
|
|
1077
|
+
/* best-effort cleanup; a stray identical mirror is harmless */
|
|
1076
1078
|
}
|
|
1079
|
+
}
|
|
1080
|
+
updateEntry(
|
|
1081
|
+
run.journal,
|
|
1082
|
+
remoteFile.key,
|
|
1083
|
+
item.localHash,
|
|
1084
|
+
item.localSize,
|
|
1085
|
+
"down",
|
|
1086
|
+
remoteFile.etag,
|
|
1087
|
+
item.localMtime.getTime(),
|
|
1088
|
+
);
|
|
1089
|
+
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1090
|
+
counters.filesSkipped++;
|
|
1091
|
+
return null;
|
|
1092
|
+
}
|
|
1077
1093
|
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
remoteFile.key,
|
|
1092
|
-
item.localHash,
|
|
1093
|
-
item.localSize,
|
|
1094
|
-
"down",
|
|
1095
|
-
remoteFile.etag,
|
|
1096
|
-
item.localMtime.getTime(),
|
|
1097
|
-
);
|
|
1098
|
-
emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1099
|
-
filesSkipped++;
|
|
1100
|
-
continue;
|
|
1101
|
-
}
|
|
1094
|
+
counters.conflicts++;
|
|
1095
|
+
counters.conflictPaths.push(remoteFile.key);
|
|
1096
|
+
|
|
1097
|
+
const resolution = await resolveConflict(
|
|
1098
|
+
{
|
|
1099
|
+
path: remoteFile.key,
|
|
1100
|
+
localHash: item.localHash,
|
|
1101
|
+
remoteModified: remoteFile.lastModified,
|
|
1102
|
+
localModified: item.localMtime,
|
|
1103
|
+
direction: "pull",
|
|
1104
|
+
},
|
|
1105
|
+
run.options.onConflict,
|
|
1106
|
+
);
|
|
1102
1107
|
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1108
|
+
run.emit({
|
|
1109
|
+
type: "conflict",
|
|
1110
|
+
path: remoteFile.key,
|
|
1111
|
+
direction: "pull",
|
|
1112
|
+
resolution,
|
|
1113
|
+
});
|
|
1106
1114
|
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1115
|
+
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
1116
|
+
if (remoteFetched) {
|
|
1117
|
+
try {
|
|
1118
|
+
appendConflictEntry(run.hqRoot, {
|
|
1119
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
1120
|
+
originalPath: originalRelative,
|
|
1121
|
+
conflictPath: conflictRelative,
|
|
1122
|
+
detectedAt,
|
|
1123
|
+
side: "pull",
|
|
1124
|
+
machineId,
|
|
1110
1125
|
localHash: item.localHash,
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
emit({
|
|
1122
|
-
type: "conflict",
|
|
1123
|
-
path: remoteFile.key,
|
|
1124
|
-
direction: "pull",
|
|
1125
|
-
resolution,
|
|
1126
|
-
});
|
|
1127
|
-
|
|
1128
|
-
// The remote bytes were already fetched to `conflictAbs` by the
|
|
1129
|
-
// convergence probe. For "keep"/"skip" they become the
|
|
1130
|
-
// `<original>.conflict-<ts>-<machine>.<ext>` inspection mirror — just
|
|
1131
|
-
// index it (no second download). For "abort" (user gave up) and
|
|
1132
|
-
// "overwrite" (cloud bytes are about to replace local) the mirror is
|
|
1133
|
-
// redundant, so discard it. Best-effort: failure here only emits an
|
|
1134
|
-
// error, doesn't break the sync.
|
|
1135
|
-
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
1136
|
-
if (remoteFetched) {
|
|
1137
|
-
try {
|
|
1138
|
-
appendConflictEntry(hqRoot, {
|
|
1139
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1140
|
-
originalPath: originalRelative,
|
|
1141
|
-
conflictPath: conflictRelative,
|
|
1142
|
-
detectedAt,
|
|
1143
|
-
side: "pull",
|
|
1144
|
-
machineId,
|
|
1145
|
-
localHash: item.localHash,
|
|
1146
|
-
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
1147
|
-
});
|
|
1148
|
-
} catch (mirrorErr) {
|
|
1149
|
-
emit({
|
|
1150
|
-
type: "error",
|
|
1151
|
-
path: remoteFile.key,
|
|
1152
|
-
message: `conflict mirror index write failed: ${
|
|
1153
|
-
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1154
|
-
}`,
|
|
1155
|
-
});
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
// If the probe download failed (!remoteFetched) there is no mirror on
|
|
1159
|
-
// disk; the probe already emitted the error. The conflict is still
|
|
1160
|
-
// surfaced and journal-stamped below so it doesn't re-fire silently.
|
|
1161
|
-
} else if (remoteFetched) {
|
|
1162
|
-
try {
|
|
1163
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
1164
|
-
} catch {
|
|
1165
|
-
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
1166
|
-
}
|
|
1167
|
-
}
|
|
1168
|
-
|
|
1169
|
-
if (resolution === "abort") {
|
|
1170
|
-
emit({ type: "new-files", files: [] });
|
|
1171
|
-
writeJournal(journalSlug, journal);
|
|
1172
|
-
aborted = true;
|
|
1173
|
-
abortResult = {
|
|
1174
|
-
filesDownloaded,
|
|
1175
|
-
bytesDownloaded,
|
|
1176
|
-
filesSkipped,
|
|
1177
|
-
conflicts,
|
|
1178
|
-
conflictPaths,
|
|
1179
|
-
aborted: true,
|
|
1180
|
-
newFiles: plan.newFiles,
|
|
1181
|
-
newFilesCount: plan.newFilesCount,
|
|
1182
|
-
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1183
|
-
// Abort short-circuits before the tombstone loop runs; report
|
|
1184
|
-
// 0 so the field shape stays stable for consumers that
|
|
1185
|
-
// destructure it.
|
|
1186
|
-
filesTombstoned: 0,
|
|
1187
|
-
// Scope-shrink ran before execution, so its counts are real even on
|
|
1188
|
-
// a conflict abort. `filesOutOfScope` reflects how far the serial
|
|
1189
|
-
// pass got before the abort; that's acceptable for an abort result.
|
|
1190
|
-
filesOutOfScope,
|
|
1191
|
-
scopeOrphansRemoved,
|
|
1192
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1193
|
-
};
|
|
1194
|
-
break;
|
|
1195
|
-
}
|
|
1196
|
-
if (resolution === "keep" || resolution === "skip") {
|
|
1197
|
-
filesSkipped++;
|
|
1198
|
-
// Stamp the journal with the new baseline so the same conflict
|
|
1199
|
-
// doesn't re-fire on every subsequent sync. After "keep", local
|
|
1200
|
-
// wins — the user has accepted that the cloud version we just
|
|
1201
|
-
// mirrored is what cloud is at this etag, and they don't want
|
|
1202
|
-
// it. Recording (current localHash + current remoteEtag) tells
|
|
1203
|
-
// the next sync "no change on either side" until something new
|
|
1204
|
-
// diverges. Without this, both `localChanged` and `remoteChanged`
|
|
1205
|
-
// stay true forever and the conflict is sticky.
|
|
1206
|
-
// Stamp from planner-captured size (symlink-aware), NOT
|
|
1207
|
-
// statSync — which would follow a dangling symlink and
|
|
1208
|
-
// throw ENOENT, get swallowed, and leave the journal
|
|
1209
|
-
// stale so this conflict would re-fire on every sync
|
|
1210
|
-
// forever. localSize is sourced from the same lstat that
|
|
1211
|
-
// computed localMtime + localHash above.
|
|
1212
|
-
updateEntry(
|
|
1213
|
-
journal,
|
|
1214
|
-
remoteFile.key,
|
|
1215
|
-
item.localHash,
|
|
1216
|
-
item.localSize,
|
|
1217
|
-
"down",
|
|
1218
|
-
remoteFile.etag,
|
|
1219
|
-
item.localMtime.getTime(),
|
|
1220
|
-
);
|
|
1221
|
-
continue;
|
|
1126
|
+
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
1127
|
+
});
|
|
1128
|
+
} catch (mirrorErr) {
|
|
1129
|
+
run.emit({
|
|
1130
|
+
type: "error",
|
|
1131
|
+
path: remoteFile.key,
|
|
1132
|
+
message: `conflict mirror index write failed: ${
|
|
1133
|
+
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1134
|
+
}`,
|
|
1135
|
+
});
|
|
1222
1136
|
}
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
remoteFile,
|
|
1230
|
-
localPath,
|
|
1231
|
-
isNew: false,
|
|
1232
|
-
});
|
|
1233
|
-
continue;
|
|
1137
|
+
}
|
|
1138
|
+
} else if (remoteFetched) {
|
|
1139
|
+
try {
|
|
1140
|
+
fs.rmSync(conflictAbs, { force: true });
|
|
1141
|
+
} catch {
|
|
1142
|
+
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
1234
1143
|
}
|
|
1235
1144
|
}
|
|
1236
1145
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1146
|
+
if (resolution === "abort") {
|
|
1147
|
+
run.emit({ type: "new-files", files: [] });
|
|
1148
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1149
|
+
return {
|
|
1150
|
+
filesDownloaded: counters.filesDownloaded,
|
|
1151
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
1152
|
+
filesSkipped: counters.filesSkipped,
|
|
1153
|
+
conflicts: counters.conflicts,
|
|
1154
|
+
conflictPaths: counters.conflictPaths,
|
|
1155
|
+
aborted: true,
|
|
1156
|
+
newFiles: plan.newFiles,
|
|
1157
|
+
newFilesCount: plan.newFilesCount,
|
|
1158
|
+
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1159
|
+
filesTombstoned: 0,
|
|
1160
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
1161
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1162
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1163
|
+
};
|
|
1243
1164
|
}
|
|
1244
1165
|
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
// On a large initial pull this is the difference between ~ceil(N/100)
|
|
1256
|
-
// presign calls and N (which would 429 past the 100-req/hr limit). No-op
|
|
1257
|
-
// on the S3 SDK transport; best-effort (failure falls back to per-file).
|
|
1258
|
-
await primeObjectTransport(
|
|
1259
|
-
ctx,
|
|
1260
|
-
"get",
|
|
1261
|
-
downloadItems.map((d) => d.remoteFile.key),
|
|
1166
|
+
if (resolution === "keep" || resolution === "skip") {
|
|
1167
|
+
counters.filesSkipped++;
|
|
1168
|
+
updateEntry(
|
|
1169
|
+
run.journal,
|
|
1170
|
+
remoteFile.key,
|
|
1171
|
+
item.localHash,
|
|
1172
|
+
item.localSize,
|
|
1173
|
+
"down",
|
|
1174
|
+
remoteFile.etag,
|
|
1175
|
+
item.localMtime.getTime(),
|
|
1262
1176
|
);
|
|
1177
|
+
return null;
|
|
1178
|
+
}
|
|
1263
1179
|
|
|
1264
|
-
|
|
1265
|
-
|
|
1180
|
+
downloadItems.push({
|
|
1181
|
+
action: "download",
|
|
1182
|
+
remoteFile,
|
|
1183
|
+
localPath,
|
|
1184
|
+
isNew: false,
|
|
1185
|
+
});
|
|
1186
|
+
return null;
|
|
1187
|
+
}
|
|
1266
1188
|
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1189
|
+
async function executeDownloadExecutor(
|
|
1190
|
+
run: PullRunContext,
|
|
1191
|
+
downloadItems: PullDownloadItem[],
|
|
1192
|
+
transferConcurrency: number,
|
|
1193
|
+
counters: PullCounters,
|
|
1194
|
+
): Promise<void> {
|
|
1195
|
+
if (downloadItems.length === 0) return;
|
|
1271
1196
|
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1197
|
+
await primeObjectTransport(
|
|
1198
|
+
run.ctx,
|
|
1199
|
+
"get",
|
|
1200
|
+
downloadItems.map((d) => d.remoteFile.key),
|
|
1201
|
+
);
|
|
1277
1202
|
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1203
|
+
const queue = [...downloadItems];
|
|
1204
|
+
const inFlight: Set<Promise<unknown>> = new Set();
|
|
1205
|
+
const workerErrors: Error[] = [];
|
|
1206
|
+
|
|
1207
|
+
while (queue.length > 0 || inFlight.size > 0) {
|
|
1208
|
+
while (inFlight.size < transferConcurrency && queue.length > 0) {
|
|
1209
|
+
const downloadItem = queue.shift()!;
|
|
1210
|
+
const p: Promise<void> = downloadOne(run, downloadItem, counters)
|
|
1211
|
+
.catch((err: unknown) => {
|
|
1212
|
+
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1213
|
+
})
|
|
1214
|
+
.finally(() => {
|
|
1215
|
+
inFlight.delete(p);
|
|
1284
1216
|
});
|
|
1285
|
-
|
|
1286
|
-
|
|
1217
|
+
inFlight.add(p);
|
|
1218
|
+
}
|
|
1219
|
+
if (inFlight.size > 0) {
|
|
1220
|
+
await Promise.race(Array.from(inFlight));
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1287
1223
|
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
const createdBySub = metadata?.["created-by-sub"];
|
|
1298
|
-
|
|
1299
|
-
// Symlink records materialize as real symlinks on disk. lstat
|
|
1300
|
-
// (does not follow) lets us detect that case so the journal stamp
|
|
1301
|
-
// mirrors what the push side would emit on the next tick:
|
|
1302
|
-
// hash = sha256(readlink target string)
|
|
1303
|
-
// size = 0
|
|
1304
|
-
// Without this check, hashFile would follow the link and stamp the
|
|
1305
|
-
// target file's contents — a value the next push would never
|
|
1306
|
-
// produce — which makes skipUnchanged perpetually re-upload every
|
|
1307
|
-
// symlink, defeating the point of the gate.
|
|
1308
|
-
const localLstat = fs.lstatSync(localPath);
|
|
1309
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1310
|
-
const hash = isLocalSymlink
|
|
1311
|
-
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1312
|
-
: (contentHash ?? hashFile(localPath));
|
|
1313
|
-
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
1314
|
-
// Capture the listing's ETag so subsequent syncs can detect remote
|
|
1315
|
-
// drift independently of mtime drift. Stamp mtimeMs from localLstat
|
|
1316
|
-
// (5.36.0) so the next push planner's lstat fast-path can skip the
|
|
1317
|
-
// SHA256 for this file without reading its bytes.
|
|
1318
|
-
//
|
|
1319
|
-
// 5.37.0 ordering invariant: downloadFile applies hq-mtime via
|
|
1320
|
-
// utimesSync AFTER its byte write but BEFORE returning, and this
|
|
1321
|
-
// lstat runs AFTER downloadFile resolves — so localLstat.mtimeMs
|
|
1322
|
-
// already reflects the source-stamped mtime, not the wall-clock
|
|
1323
|
-
// write-time. The journal therefore matches what the next push's
|
|
1324
|
-
// lstat fast-path will see, and the file is correctly skipped on
|
|
1325
|
-
// re-sync instead of being hashed every tick. Do not move this
|
|
1326
|
-
// lstat earlier; do not stamp the journal from any pre-download
|
|
1327
|
-
// mtime.
|
|
1328
|
-
updateEntry(
|
|
1329
|
-
journal,
|
|
1330
|
-
remoteFile.key,
|
|
1331
|
-
hash,
|
|
1332
|
-
size,
|
|
1333
|
-
"down",
|
|
1334
|
-
remoteFile.etag,
|
|
1335
|
-
localLstat.mtimeMs,
|
|
1336
|
-
createdBySub,
|
|
1337
|
-
);
|
|
1224
|
+
if (workerErrors.length > 0) {
|
|
1225
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1226
|
+
const first = workerErrors[0]!;
|
|
1227
|
+
if (workerErrors.length > 1) {
|
|
1228
|
+
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1229
|
+
}
|
|
1230
|
+
throw first;
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1338
1233
|
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
path: remoteFile.key,
|
|
1346
|
-
bytes: size,
|
|
1347
|
-
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
1348
|
-
...(author ? { author } : {}),
|
|
1349
|
-
});
|
|
1234
|
+
async function downloadOne(
|
|
1235
|
+
run: PullRunContext,
|
|
1236
|
+
downloadItem: PullDownloadItem,
|
|
1237
|
+
counters: PullCounters,
|
|
1238
|
+
): Promise<void> {
|
|
1239
|
+
const { remoteFile, localPath } = downloadItem;
|
|
1350
1240
|
|
|
1351
|
-
|
|
1352
|
-
bytesDownloaded += size;
|
|
1353
|
-
} catch (err) {
|
|
1354
|
-
// STS session policy may deny access to some paths — this is expected
|
|
1355
|
-
// for guest members with allowedPrefixes
|
|
1356
|
-
if (isAccessDenied(err)) {
|
|
1357
|
-
filesSkipped++;
|
|
1358
|
-
} else {
|
|
1359
|
-
emit({
|
|
1360
|
-
type: "error",
|
|
1361
|
-
path: remoteFile.key,
|
|
1362
|
-
message: err instanceof Error ? err.message : String(err),
|
|
1363
|
-
});
|
|
1364
|
-
}
|
|
1365
|
-
}
|
|
1366
|
-
};
|
|
1241
|
+
await refreshRunContextIfExpiring(run);
|
|
1367
1242
|
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
// fully drains — see workerErrors throw below.
|
|
1378
|
-
const workerErrors: Error[] = [];
|
|
1379
|
-
while (queue.length > 0 || inFlight.size > 0) {
|
|
1380
|
-
while (inFlight.size < TRANSFER_CONCURRENCY && queue.length > 0) {
|
|
1381
|
-
const downloadItem = queue.shift()!;
|
|
1382
|
-
const p: Promise<void> = downloadOne(downloadItem)
|
|
1383
|
-
.catch((err: unknown) => {
|
|
1384
|
-
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1385
|
-
})
|
|
1386
|
-
.finally(() => {
|
|
1387
|
-
inFlight.delete(p);
|
|
1388
|
-
});
|
|
1389
|
-
inFlight.add(p);
|
|
1390
|
-
}
|
|
1391
|
-
if (inFlight.size > 0) {
|
|
1392
|
-
// Wait for at least one in-flight task to settle before topping up
|
|
1393
|
-
// the pool. allSettled-style semantics via Promise.race — the
|
|
1394
|
-
// .catch wrap above guarantees no worker promise can reject.
|
|
1395
|
-
await Promise.race(Array.from(inFlight));
|
|
1396
|
-
}
|
|
1397
|
-
}
|
|
1243
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
1244
|
+
counters.filesSkipped++;
|
|
1245
|
+
run.emit({
|
|
1246
|
+
type: "error",
|
|
1247
|
+
path: remoteFile.key,
|
|
1248
|
+
message: "download skipped: local parent escaped the sync root",
|
|
1249
|
+
});
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1398
1252
|
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1253
|
+
try {
|
|
1254
|
+
const { metadata, contentHash, contentSize } = await downloadFile(
|
|
1255
|
+
run.ctx,
|
|
1256
|
+
remoteFile.key,
|
|
1257
|
+
localPath,
|
|
1258
|
+
);
|
|
1259
|
+
const author = metadata?.["created-by"] ?? null;
|
|
1260
|
+
const createdBySub = metadata?.["created-by-sub"];
|
|
1261
|
+
|
|
1262
|
+
const localLstat = fs.lstatSync(localPath);
|
|
1263
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1264
|
+
const hash = isLocalSymlink
|
|
1265
|
+
? hashSymlinkTarget(fs.readlinkSync(localPath))
|
|
1266
|
+
: (contentHash ?? hashFile(localPath));
|
|
1267
|
+
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
1268
|
+
|
|
1269
|
+
updateEntry(
|
|
1270
|
+
run.journal,
|
|
1271
|
+
remoteFile.key,
|
|
1272
|
+
hash,
|
|
1273
|
+
size,
|
|
1274
|
+
"down",
|
|
1275
|
+
remoteFile.etag,
|
|
1276
|
+
localLstat.mtimeMs,
|
|
1277
|
+
createdBySub,
|
|
1278
|
+
);
|
|
1279
|
+
|
|
1280
|
+
const priorEntry = getEntry(run.journal, remoteFile.key);
|
|
1281
|
+
const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
|
|
1282
|
+
run.emit({
|
|
1283
|
+
type: "progress",
|
|
1284
|
+
path: remoteFile.key,
|
|
1285
|
+
bytes: size,
|
|
1286
|
+
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
1287
|
+
...(author ? { author } : {}),
|
|
1288
|
+
});
|
|
1289
|
+
|
|
1290
|
+
counters.filesDownloaded++;
|
|
1291
|
+
counters.bytesDownloaded += size;
|
|
1292
|
+
} catch (err) {
|
|
1293
|
+
if (isAccessDenied(err)) {
|
|
1294
|
+
counters.filesSkipped++;
|
|
1295
|
+
} else {
|
|
1296
|
+
run.emit({
|
|
1297
|
+
type: "error",
|
|
1298
|
+
path: remoteFile.key,
|
|
1299
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1300
|
+
});
|
|
1409
1301
|
}
|
|
1410
1302
|
}
|
|
1303
|
+
}
|
|
1411
1304
|
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1305
|
+
async function emitAndReportNewFiles(
|
|
1306
|
+
run: PullRunContext,
|
|
1307
|
+
plan: PullPlan,
|
|
1308
|
+
): Promise<void> {
|
|
1415
1309
|
const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
|
|
1310
|
+
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
1311
|
+
// created-by HEADs below reuse the cache instead of each minting its own
|
|
1312
|
+
// presign. Without this, a big catch-up pull (hundreds of new files) bursts
|
|
1313
|
+
// the presign endpoint, trips the circuit breaker, and every enrichment HEAD
|
|
1314
|
+
// then fails. Mirrors the tombstone HEAD-verify pre-prime.
|
|
1315
|
+
await primeObjectTransport(
|
|
1316
|
+
run.ctx,
|
|
1317
|
+
"get",
|
|
1318
|
+
plan.newFiles.map((nf) => nf.path),
|
|
1319
|
+
);
|
|
1416
1320
|
const HEAD_CONCURRENCY = 5;
|
|
1417
1321
|
for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
|
|
1418
1322
|
const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
|
|
@@ -1420,13 +1324,11 @@ async function syncWithOperationLockHeld(
|
|
|
1420
1324
|
batch.map(async (nf) => {
|
|
1421
1325
|
let addedBy: string | null = null;
|
|
1422
1326
|
try {
|
|
1423
|
-
const head = await headRemoteFile(ctx, nf.path);
|
|
1327
|
+
const head = await headRemoteFile(run.ctx, nf.path);
|
|
1424
1328
|
if (head?.metadata?.["created-by"]) {
|
|
1425
1329
|
addedBy = head.metadata["created-by"];
|
|
1426
1330
|
}
|
|
1427
1331
|
} catch (headErr) {
|
|
1428
|
-
// Best-effort: log to console (Sentry captures via global handler)
|
|
1429
|
-
// and fall through with addedBy = null.
|
|
1430
1332
|
try {
|
|
1431
1333
|
console.error(
|
|
1432
1334
|
`[hq-sync] HeadObject failed for ${nf.path}: ${
|
|
@@ -1442,220 +1344,150 @@ async function syncWithOperationLockHeld(
|
|
|
1442
1344
|
);
|
|
1443
1345
|
enrichedNewFiles.push(...results);
|
|
1444
1346
|
}
|
|
1445
|
-
emit({ type: "new-files", files: enrichedNewFiles });
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
type: "error",
|
|
1487
|
-
path: key,
|
|
1488
|
-
message: `tombstone HEAD verify failed (deferring): ${
|
|
1489
|
-
err instanceof Error ? err.message : String(err)
|
|
1490
|
-
}`,
|
|
1491
|
-
});
|
|
1492
|
-
return null;
|
|
1493
|
-
}
|
|
1494
|
-
}),
|
|
1495
|
-
);
|
|
1496
|
-
for (const k of results) {
|
|
1497
|
-
if (k !== null) verified.push(k);
|
|
1498
|
-
}
|
|
1347
|
+
run.emit({ type: "new-files", files: enrichedNewFiles });
|
|
1348
|
+
await reportNewFilesToNotify(
|
|
1349
|
+
run.vaultConfig,
|
|
1350
|
+
run.ctx.uid,
|
|
1351
|
+
run.ctx.slug,
|
|
1352
|
+
enrichedNewFiles,
|
|
1353
|
+
);
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
async function verifyPlannedJournalTombstones(
|
|
1357
|
+
run: PullRunContext,
|
|
1358
|
+
plan: PullPlan,
|
|
1359
|
+
): Promise<void> {
|
|
1360
|
+
if (plan.tombstones.length === 0) return;
|
|
1361
|
+
|
|
1362
|
+
await primeObjectTransport(run.ctx, "get", plan.tombstones);
|
|
1363
|
+
|
|
1364
|
+
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
1365
|
+
const verified: string[] = [];
|
|
1366
|
+
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
1367
|
+
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
1368
|
+
const results = await Promise.all(
|
|
1369
|
+
batch.map(async (key) => {
|
|
1370
|
+
try {
|
|
1371
|
+
const head = await headRemoteFile(run.ctx, key);
|
|
1372
|
+
return head === null ? key : null;
|
|
1373
|
+
} catch (err) {
|
|
1374
|
+
if (isAccessDenied(err)) return null;
|
|
1375
|
+
run.emit({
|
|
1376
|
+
type: "error",
|
|
1377
|
+
path: key,
|
|
1378
|
+
message: `tombstone HEAD verify failed (deferring): ${
|
|
1379
|
+
err instanceof Error ? err.message : String(err)
|
|
1380
|
+
}`,
|
|
1381
|
+
});
|
|
1382
|
+
return null;
|
|
1383
|
+
}
|
|
1384
|
+
}),
|
|
1385
|
+
);
|
|
1386
|
+
for (const k of results) {
|
|
1387
|
+
if (k !== null) verified.push(k);
|
|
1499
1388
|
}
|
|
1500
|
-
plan.tombstones = verified;
|
|
1501
1389
|
}
|
|
1390
|
+
plan.tombstones = verified;
|
|
1391
|
+
}
|
|
1502
1392
|
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
// the journal and the next run retries.
|
|
1393
|
+
function executeJournalTombstoneDeletes(
|
|
1394
|
+
run: PullRunContext,
|
|
1395
|
+
plan: PullPlan,
|
|
1396
|
+
counters: PullCounters,
|
|
1397
|
+
): void {
|
|
1509
1398
|
for (const key of plan.tombstones) {
|
|
1510
|
-
|
|
1511
|
-
// fs.unlinkSync or journal mutation for a path outside the sync root.
|
|
1512
|
-
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
1399
|
+
const localPath = resolveContainedVaultPath(run.companyRoot, key);
|
|
1513
1400
|
if (localPath === null) continue;
|
|
1514
1401
|
let removedSomething = false;
|
|
1515
1402
|
try {
|
|
1516
1403
|
const lstat = fs.lstatSync(localPath);
|
|
1517
|
-
if (tombstoneTargetDiverged(journal, key, localPath, lstat)) {
|
|
1404
|
+
if (tombstoneTargetDiverged(run.journal, key, localPath, lstat)) {
|
|
1518
1405
|
continue;
|
|
1519
1406
|
}
|
|
1520
1407
|
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
1521
1408
|
fs.unlinkSync(localPath);
|
|
1522
1409
|
removedSomething = true;
|
|
1523
1410
|
} else if (lstat.isDirectory()) {
|
|
1524
|
-
// A dir at a key
|
|
1525
|
-
// state. Don't recursively rm-rf the operator's dir; just drop
|
|
1526
|
-
// the journal entry so we converge with reality.
|
|
1411
|
+
// A dir at a key is converged by dropping only the journal entry.
|
|
1527
1412
|
}
|
|
1528
1413
|
} catch (err: unknown) {
|
|
1529
1414
|
const code =
|
|
1530
1415
|
err && typeof err === "object" && "code" in err
|
|
1531
1416
|
? (err as { code?: string }).code
|
|
1532
1417
|
: undefined;
|
|
1533
|
-
// ENOENT → local already gone; safe to drop the journal entry.
|
|
1534
|
-
// Other errors (EACCES/EPERM/EBUSY/etc.) leave the local file in
|
|
1535
|
-
// place — if we dropped the journal entry anyway, the pull side
|
|
1536
|
-
// would forget the peer's delete and a later push could re-upload
|
|
1537
|
-
// the still-present local file, silently undoing the peer's delete.
|
|
1538
|
-
// Surface the error and KEEP the journal entry so the next sync
|
|
1539
|
-
// retries the unlink after the operator fixes the permission.
|
|
1540
1418
|
if (code !== "ENOENT") {
|
|
1541
|
-
emit({
|
|
1419
|
+
run.emit({
|
|
1542
1420
|
type: "error",
|
|
1543
1421
|
path: key,
|
|
1544
1422
|
message: `tombstone unlink failed: ${
|
|
1545
1423
|
err instanceof Error ? err.message : String(err)
|
|
1546
1424
|
}`,
|
|
1547
1425
|
});
|
|
1548
|
-
// Skip removeEntry / filesTombstoned / progress event — the
|
|
1549
|
-
// tombstone hasn't actually been honored. Next sync retries.
|
|
1550
1426
|
continue;
|
|
1551
1427
|
}
|
|
1552
1428
|
}
|
|
1553
|
-
removeEntry(journal, key);
|
|
1554
|
-
filesTombstoned++;
|
|
1555
|
-
emit({
|
|
1429
|
+
removeEntry(run.journal, key);
|
|
1430
|
+
counters.filesTombstoned++;
|
|
1431
|
+
run.emit({
|
|
1556
1432
|
type: "progress",
|
|
1557
1433
|
path: key,
|
|
1558
1434
|
bytes: 0,
|
|
1559
1435
|
deleted: true,
|
|
1560
|
-
// Suffix differentiates a tombstone from a normal delete in the
|
|
1561
|
-
// tty stream — matches the push-side `defaultConsoleLogger`
|
|
1562
|
-
// tombstone surface in share.ts.
|
|
1563
1436
|
message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
|
|
1564
1437
|
});
|
|
1565
1438
|
}
|
|
1439
|
+
}
|
|
1566
1440
|
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1441
|
+
function finalizePullRun(
|
|
1442
|
+
run: PullRunContext,
|
|
1443
|
+
plan: PullPlan,
|
|
1444
|
+
scopeRun: ScopeShrinkRun,
|
|
1445
|
+
counters: PullCounters,
|
|
1446
|
+
): SyncResult {
|
|
1447
|
+
appendPullRecord(run.journal, {
|
|
1572
1448
|
pullId: generatePullId(),
|
|
1573
|
-
companyUid: ctx.uid,
|
|
1574
|
-
startedAt,
|
|
1449
|
+
companyUid: run.ctx.uid,
|
|
1450
|
+
startedAt: run.startedAt,
|
|
1575
1451
|
completedAt: new Date().toISOString(),
|
|
1576
|
-
syncMode,
|
|
1577
|
-
prefixSet: currentPrefixSet,
|
|
1578
|
-
scopeChangeDetected: shrinkPlan.scopeChangeDetected,
|
|
1579
|
-
orphansRemoved: scopeOrphansRemoved,
|
|
1580
|
-
orphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1452
|
+
syncMode: run.syncMode,
|
|
1453
|
+
prefixSet: run.currentPrefixSet,
|
|
1454
|
+
scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
|
|
1455
|
+
orphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1456
|
+
orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1581
1457
|
});
|
|
1582
1458
|
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
// last file change, which is misleading.
|
|
1587
|
-
journal.lastSync = new Date().toISOString();
|
|
1588
|
-
writeJournal(journalSlug, journal);
|
|
1589
|
-
|
|
1590
|
-
// When the pull actually changed on-disk sources (new files, tombstoned
|
|
1591
|
-
// removals, or scope-orphan cleanups), refresh the generated skill wrappers,
|
|
1592
|
-
// personal-overlay mirrors, and workers registry. reindex is idempotent and
|
|
1593
|
-
// best-effort — it must never fail a sync, and is skipped on no-op syncs
|
|
1594
|
-
// (the common daemon case) and when the caller opts out via skipReindex.
|
|
1459
|
+
run.journal.lastSync = new Date().toISOString();
|
|
1460
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1461
|
+
|
|
1595
1462
|
const changedOnDisk =
|
|
1596
|
-
filesDownloaded > 0 ||
|
|
1597
|
-
filesTombstoned > 0 ||
|
|
1598
|
-
scopeOrphansRemoved > 0;
|
|
1599
|
-
if (!options.skipReindex && changedOnDisk) {
|
|
1463
|
+
counters.filesDownloaded > 0 ||
|
|
1464
|
+
counters.filesTombstoned > 0 ||
|
|
1465
|
+
scopeRun.scopeOrphansRemoved > 0;
|
|
1466
|
+
if (!run.options.skipReindex && changedOnDisk) {
|
|
1600
1467
|
try {
|
|
1601
|
-
|
|
1602
|
-
// lock; reindex re-acquiring would refuse against our own live PID.
|
|
1603
|
-
reindex({ repoRoot: hqRoot, skipLock: true });
|
|
1468
|
+
reindex({ repoRoot: run.hqRoot, skipLock: true });
|
|
1604
1469
|
} catch {
|
|
1605
1470
|
// best-effort: a post-sync refresh failure never fails the sync
|
|
1606
1471
|
}
|
|
1607
1472
|
}
|
|
1608
1473
|
|
|
1609
1474
|
return {
|
|
1610
|
-
filesDownloaded,
|
|
1611
|
-
bytesDownloaded,
|
|
1612
|
-
filesSkipped,
|
|
1613
|
-
conflicts,
|
|
1614
|
-
conflictPaths,
|
|
1475
|
+
filesDownloaded: counters.filesDownloaded,
|
|
1476
|
+
bytesDownloaded: counters.bytesDownloaded,
|
|
1477
|
+
filesSkipped: counters.filesSkipped,
|
|
1478
|
+
conflicts: counters.conflicts,
|
|
1479
|
+
conflictPaths: counters.conflictPaths,
|
|
1615
1480
|
aborted: false,
|
|
1616
1481
|
newFiles: plan.newFiles,
|
|
1617
1482
|
newFilesCount: plan.newFilesCount,
|
|
1618
1483
|
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1619
|
-
filesTombstoned,
|
|
1620
|
-
filesOutOfScope,
|
|
1621
|
-
scopeOrphansRemoved,
|
|
1622
|
-
scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
|
|
1484
|
+
filesTombstoned: counters.filesTombstoned,
|
|
1485
|
+
filesOutOfScope: counters.filesOutOfScope,
|
|
1486
|
+
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1487
|
+
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1623
1488
|
};
|
|
1624
1489
|
}
|
|
1625
1490
|
|
|
1626
|
-
/**
|
|
1627
|
-
* Resolve active company from .hq/config.json.
|
|
1628
|
-
*/
|
|
1629
|
-
function resolveActiveCompany(hqRoot: string): string | undefined {
|
|
1630
|
-
const configPath = path.join(hqRoot, ".hq", "config.json");
|
|
1631
|
-
if (fs.existsSync(configPath)) {
|
|
1632
|
-
try {
|
|
1633
|
-
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
1634
|
-
return config.activeCompany ?? config.companySlug;
|
|
1635
|
-
} catch {
|
|
1636
|
-
// Ignore parse errors
|
|
1637
|
-
}
|
|
1638
|
-
}
|
|
1639
|
-
return undefined;
|
|
1640
|
-
}
|
|
1641
|
-
|
|
1642
|
-
/**
|
|
1643
|
-
* Returns true when the remote object appears to have moved since the
|
|
1644
|
-
* journal entry's last-recorded sync. Prefers ETag equality; falls back to
|
|
1645
|
-
* `lastModified > syncedAt` for legacy entries written before remoteEtag
|
|
1646
|
-
* was tracked. Conservative on tie (`<=` skews "remote unchanged").
|
|
1647
|
-
*/
|
|
1648
|
-
function hasRemoteChanged(
|
|
1649
|
-
remote: { lastModified: Date; etag: string },
|
|
1650
|
-
entry: { syncedAt: string; remoteEtag?: string },
|
|
1651
|
-
): boolean {
|
|
1652
|
-
if (entry.remoteEtag) {
|
|
1653
|
-
return normalizeEtag(remote.etag) !== entry.remoteEtag;
|
|
1654
|
-
}
|
|
1655
|
-
const syncedAt = new Date(entry.syncedAt).getTime();
|
|
1656
|
-
return remote.lastModified.getTime() > syncedAt;
|
|
1657
|
-
}
|
|
1658
|
-
|
|
1659
1491
|
/**
|
|
1660
1492
|
* Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
|
|
1661
1493
|
* written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
|
|
@@ -2355,16 +2187,6 @@ function computePullPlan(
|
|
|
2355
2187
|
};
|
|
2356
2188
|
}
|
|
2357
2189
|
|
|
2358
|
-
/**
|
|
2359
|
-
* Check if an error is an S3 access denied (expected for filtered guests).
|
|
2360
|
-
*/
|
|
2361
|
-
function isAccessDenied(err: unknown): boolean {
|
|
2362
|
-
if (err && typeof err === "object" && "name" in err) {
|
|
2363
|
-
return err.name === "AccessDenied" || err.name === "Forbidden";
|
|
2364
|
-
}
|
|
2365
|
-
return false;
|
|
2366
|
-
}
|
|
2367
|
-
|
|
2368
2190
|
/**
|
|
2369
2191
|
* Default human-readable event rendering. Preserves the exact output format
|
|
2370
2192
|
* that `hq sync` emitted before SyncProgressEvent was introduced, so callers
|