@indigoai-us/hq-cloud 6.11.11 → 6.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-company.d.ts +35 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.js +290 -0
- package/dist/bin/sync-runner-company.js.map +1 -0
- package/dist/bin/sync-runner-events.d.ts +12 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -0
- package/dist/bin/sync-runner-events.js +12 -0
- package/dist/bin/sync-runner-events.js.map +1 -0
- package/dist/bin/sync-runner-planning.d.ts +53 -0
- package/dist/bin/sync-runner-planning.d.ts.map +1 -0
- package/dist/bin/sync-runner-planning.js +59 -0
- package/dist/bin/sync-runner-planning.js.map +1 -0
- package/dist/bin/sync-runner-rollup.d.ts +24 -0
- package/dist/bin/sync-runner-rollup.d.ts.map +1 -0
- package/dist/bin/sync-runner-rollup.js +46 -0
- package/dist/bin/sync-runner-rollup.js.map +1 -0
- package/dist/bin/sync-runner-telemetry.d.ts +5 -0
- package/dist/bin/sync-runner-telemetry.d.ts.map +1 -0
- package/dist/bin/sync-runner-telemetry.js +5 -0
- package/dist/bin/sync-runner-telemetry.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts +17 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-loop.js +372 -0
- package/dist/bin/sync-runner-watch-loop.js.map +1 -0
- package/dist/bin/sync-runner-watch-routes.d.ts +25 -0
- package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -0
- package/dist/bin/sync-runner-watch-routes.js +74 -0
- package/dist/bin/sync-runner-watch-routes.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +5 -54
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +76 -978
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +265 -11
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/reindex.d.ts.map +1 -1
- package/dist/cli/reindex.js +34 -17
- package/dist/cli/reindex.js.map +1 -1
- package/dist/cli/reindex.test.js +39 -5
- package/dist/cli/reindex.test.js.map +1 -1
- package/dist/cli/rescue-classify-ordering.test.js +75 -0
- package/dist/cli/rescue-classify-ordering.test.js.map +1 -1
- package/dist/cli/rescue-core.d.ts +45 -0
- package/dist/cli/rescue-core.d.ts.map +1 -1
- package/dist/cli/rescue-core.js +320 -170
- package/dist/cli/rescue-core.js.map +1 -1
- package/dist/cli/share.d.ts +2 -1
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +276 -660
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/share.test.js +30 -0
- package/dist/cli/share.test.js.map +1 -1
- package/dist/cli/sync.d.ts +28 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +541 -748
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +382 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/cognito-auth.d.ts.map +1 -1
- package/dist/cognito-auth.js +55 -10
- package/dist/cognito-auth.js.map +1 -1
- package/dist/cognito-auth.test.js +61 -0
- package/dist/cognito-auth.test.js.map +1 -1
- package/dist/daemon-worker.d.ts +2 -2
- package/dist/daemon-worker.js +3 -3
- package/dist/daemon-worker.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +93 -6
- package/dist/journal.js.map +1 -1
- package/dist/journal.test.js +59 -0
- package/dist/journal.test.js.map +1 -1
- package/dist/machine-auth.test.js +60 -2
- package/dist/machine-auth.test.js.map +1 -1
- package/dist/object-io.d.ts +37 -1
- package/dist/object-io.d.ts.map +1 -1
- package/dist/object-io.js +149 -30
- package/dist/object-io.js.map +1 -1
- package/dist/object-io.test.js +121 -0
- package/dist/object-io.test.js.map +1 -1
- package/dist/operation-lock.d.ts +8 -8
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +99 -32
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +51 -4
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/personal-vault.d.ts.map +1 -1
- package/dist/personal-vault.js +8 -2
- package/dist/personal-vault.js.map +1 -1
- package/dist/personal-vault.test.js +34 -0
- package/dist/personal-vault.test.js.map +1 -1
- package/dist/prefix-coalesce.d.ts +20 -9
- package/dist/prefix-coalesce.d.ts.map +1 -1
- package/dist/prefix-coalesce.js +124 -28
- package/dist/prefix-coalesce.js.map +1 -1
- package/dist/prefix-coalesce.test.js +57 -2
- package/dist/prefix-coalesce.test.js.map +1 -1
- package/dist/remote-pull.d.ts +8 -3
- package/dist/remote-pull.d.ts.map +1 -1
- package/dist/remote-pull.js +85 -16
- package/dist/remote-pull.js.map +1 -1
- package/dist/remote-pull.test.js +213 -2
- package/dist/remote-pull.test.js.map +1 -1
- package/dist/s3.d.ts +2 -0
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +197 -116
- package/dist/s3.js.map +1 -1
- package/dist/s3.test.js +109 -0
- package/dist/s3.test.js.map +1 -1
- package/dist/scope-shrink.d.ts +3 -2
- package/dist/scope-shrink.d.ts.map +1 -1
- package/dist/scope-shrink.js +1 -1
- package/dist/scope-shrink.js.map +1 -1
- package/dist/skill-telemetry.d.ts +1 -1
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +69 -9
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +86 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/event-sync.d.ts +6 -0
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +34 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +73 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/metrics.d.ts +17 -1
- package/dist/sync/metrics.d.ts.map +1 -1
- package/dist/sync/metrics.js +32 -1
- package/dist/sync/metrics.js.map +1 -1
- package/dist/sync/metrics.test.js +74 -1
- package/dist/sync/metrics.test.js.map +1 -1
- package/dist/sync/pull-scope.d.ts.map +1 -1
- package/dist/sync/pull-scope.js +15 -7
- package/dist/sync/pull-scope.js.map +1 -1
- package/dist/sync/push-receiver.d.ts +12 -5
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +45 -17
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +67 -1
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync-core.d.ts +27 -0
- package/dist/sync-core.d.ts.map +1 -0
- package/dist/sync-core.js +54 -0
- package/dist/sync-core.js.map +1 -0
- package/dist/telemetry.d.ts +1 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +59 -6
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +74 -0
- package/dist/telemetry.test.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/vault-client.d.ts.map +1 -1
- package/dist/vault-client.js +284 -36
- package/dist/vault-client.js.map +1 -1
- package/dist/vault-client.test.js +59 -0
- package/dist/vault-client.test.js.map +1 -1
- package/dist/watcher.d.ts +38 -20
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +155 -143
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +103 -0
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
- package/src/bin/sync-runner-company.ts +350 -0
- package/src/bin/sync-runner-events.ts +25 -0
- package/src/bin/sync-runner-planning.ts +121 -0
- package/src/bin/sync-runner-rollup.ts +72 -0
- package/src/bin/sync-runner-telemetry.ts +8 -0
- package/src/bin/sync-runner-watch-loop.ts +443 -0
- package/src/bin/sync-runner-watch-routes.ts +86 -0
- package/src/bin/sync-runner.test.ts +298 -11
- package/src/bin/sync-runner.ts +99 -1054
- package/src/cli/reindex.test.ts +41 -3
- package/src/cli/reindex.ts +35 -19
- package/src/cli/rescue-classify-ordering.test.ts +81 -0
- package/src/cli/rescue-core.ts +400 -165
- package/src/cli/share.test.ts +38 -0
- package/src/cli/share.ts +420 -693
- package/src/cli/sync.test.ts +460 -1
- package/src/cli/sync.ts +788 -825
- package/src/cognito-auth.test.ts +77 -0
- package/src/cognito-auth.ts +73 -11
- package/src/daemon-worker.ts +3 -3
- package/src/index.ts +8 -0
- package/src/journal.test.ts +72 -0
- package/src/journal.ts +95 -8
- package/src/machine-auth.test.ts +64 -2
- package/src/object-io.test.ts +142 -0
- package/src/object-io.ts +183 -31
- package/src/operation-lock.test.ts +63 -4
- package/src/operation-lock.ts +99 -31
- package/src/personal-vault.test.ts +42 -0
- package/src/personal-vault.ts +8 -2
- package/src/prefix-coalesce.test.ts +71 -1
- package/src/prefix-coalesce.ts +155 -30
- package/src/remote-pull.test.ts +235 -1
- package/src/remote-pull.ts +106 -18
- package/src/s3.test.ts +126 -0
- package/src/s3.ts +237 -122
- package/src/scope-shrink.ts +6 -3
- package/src/skill-telemetry.test.ts +109 -0
- package/src/skill-telemetry.ts +82 -14
- package/src/sync/event-sync.test.ts +75 -0
- package/src/sync/event-sync.ts +54 -1
- package/src/sync/metrics.test.ts +81 -0
- package/src/sync/metrics.ts +59 -4
- package/src/sync/pull-scope.ts +23 -7
- package/src/sync/push-receiver.test.ts +73 -1
- package/src/sync/push-receiver.ts +56 -20
- package/src/sync-core.ts +58 -0
- package/src/telemetry.test.ts +85 -0
- package/src/telemetry.ts +69 -6
- package/src/types.ts +8 -0
- package/src/vault-client.test.ts +74 -0
- package/src/vault-client.ts +395 -43
- package/src/watcher.test.ts +117 -0
- package/src/watcher.ts +215 -174
package/src/cli/share.ts
CHANGED
|
@@ -46,7 +46,17 @@ import {
|
|
|
46
46
|
fetchCompanyTombstones,
|
|
47
47
|
type CompanyTombstone,
|
|
48
48
|
} from "./tombstones.js";
|
|
49
|
-
import {
|
|
49
|
+
import {
|
|
50
|
+
isCoveredByAny,
|
|
51
|
+
isDirInScope,
|
|
52
|
+
type ScopePrefixInput,
|
|
53
|
+
} from "../prefix-coalesce.js";
|
|
54
|
+
import {
|
|
55
|
+
hasRemoteChanged,
|
|
56
|
+
isAccessDenied,
|
|
57
|
+
resolveActiveCompany,
|
|
58
|
+
resolveTransferConcurrency,
|
|
59
|
+
} from "../sync-core.js";
|
|
50
60
|
import {
|
|
51
61
|
buildConflictId,
|
|
52
62
|
buildConflictPath,
|
|
@@ -637,7 +647,7 @@ export interface ShareOptions {
|
|
|
637
647
|
* filter — full access. An empty array means "no granted prefixes" → every
|
|
638
648
|
* path is out of scope (mirrors the pull side's `isCoveredByAny([])`).
|
|
639
649
|
*/
|
|
640
|
-
prefixSet?:
|
|
650
|
+
prefixSet?: ScopePrefixInput[];
|
|
641
651
|
/**
|
|
642
652
|
* Pre-fetched FILE_TOMBSTONE map (POSIX key → tombstone) for the push-side
|
|
643
653
|
* delete-resync consult. When omitted, share() fetches it itself via
|
|
@@ -732,21 +742,6 @@ export interface ShareResult {
|
|
|
732
742
|
aborted: boolean;
|
|
733
743
|
}
|
|
734
744
|
|
|
735
|
-
/**
|
|
736
|
-
* Is this error the S3/STS "access denied" class? Expected when a scoped
|
|
737
|
-
* member/guest credential touches a key outside its granted ACL prefixes
|
|
738
|
-
* (the server's `SCOPE_EXCEEDS_PARENT` surfaces as a 403 AccessDenied /
|
|
739
|
-
* Forbidden). Mirrors the pull-side `isAccessDenied` in sync.ts so the push
|
|
740
|
-
* leg can treat a stray out-of-scope key as a skip rather than a fatal throw.
|
|
741
|
-
*/
|
|
742
|
-
function isAccessDenied(err: unknown): boolean {
|
|
743
|
-
if (err && typeof err === "object" && "name" in err) {
|
|
744
|
-
const name = (err as { name?: unknown }).name;
|
|
745
|
-
return name === "AccessDenied" || name === "Forbidden";
|
|
746
|
-
}
|
|
747
|
-
return false;
|
|
748
|
-
}
|
|
749
|
-
|
|
750
745
|
/**
|
|
751
746
|
* A conditional-write fence rejection — the SDK's 412 (`name:
|
|
752
747
|
* "PreconditionFailed"`) or the presigned transport's mirror of it. Means
|
|
@@ -778,7 +773,7 @@ function isPreconditionFailed(err: unknown): boolean {
|
|
|
778
773
|
function wrapFilterWithScope(
|
|
779
774
|
underlying: (absPath: string, isDir?: boolean) => boolean,
|
|
780
775
|
syncRoot: string,
|
|
781
|
-
prefixSet: readonly
|
|
776
|
+
prefixSet: readonly ScopePrefixInput[],
|
|
782
777
|
onScopeExcluded: (rel: string) => void,
|
|
783
778
|
): (absPath: string, isDir?: boolean) => boolean {
|
|
784
779
|
return (absPath: string, isDir?: boolean) => {
|
|
@@ -800,23 +795,99 @@ function wrapFilterWithScope(
|
|
|
800
795
|
* Share local file(s) to the entity vault.
|
|
801
796
|
*/
|
|
802
797
|
export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
798
|
+
const run = await createPushRunContext(options);
|
|
799
|
+
const counters = createShareCounters();
|
|
800
|
+
const filesRefusedStalePaths: string[] = [];
|
|
801
|
+
const conflictPaths: string[] = [];
|
|
802
|
+
|
|
803
|
+
const plans = await buildSharePlans(run);
|
|
804
|
+
const uploadResult = await executeUploads(
|
|
805
|
+
run,
|
|
806
|
+
plans.pushPlan,
|
|
807
|
+
counters,
|
|
808
|
+
conflictPaths,
|
|
809
|
+
);
|
|
810
|
+
if (uploadResult.aborted) {
|
|
811
|
+
return buildShareResult(
|
|
812
|
+
run,
|
|
813
|
+
counters,
|
|
814
|
+
filesRefusedStalePaths,
|
|
815
|
+
uploadResult.abortFlightConflictPaths,
|
|
816
|
+
true,
|
|
817
|
+
);
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
await executeDeletes(
|
|
821
|
+
run,
|
|
822
|
+
plans.deletePlan,
|
|
823
|
+
plans.decommissionPlan,
|
|
824
|
+
counters,
|
|
825
|
+
filesRefusedStalePaths,
|
|
826
|
+
);
|
|
827
|
+
finalizeShareJournal(run);
|
|
828
|
+
throwUploadWorkerErrors(uploadResult.workerErrors);
|
|
829
|
+
|
|
830
|
+
return buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths, false);
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
type DeletePolicy = NonNullable<ShareOptions["propagateDeletePolicy"]>;
|
|
834
|
+
type ShareEmit = (event: SyncProgressEvent) => void;
|
|
835
|
+
type UploadPlanItem = Extract<PushPlanItem, { action: "upload" }>;
|
|
836
|
+
type JournalFileEntry = SyncJournal["files"][string];
|
|
837
|
+
|
|
838
|
+
interface PushRunContext {
|
|
839
|
+
options: ShareOptions;
|
|
840
|
+
paths: string[];
|
|
841
|
+
message?: string;
|
|
842
|
+
onConflict?: ConflictStrategy;
|
|
843
|
+
vaultConfig?: VaultServiceConfig;
|
|
844
|
+
entityContext?: EntityContext;
|
|
845
|
+
hqRoot: string;
|
|
846
|
+
skipUnchanged?: boolean;
|
|
847
|
+
propagateDeletes?: boolean;
|
|
848
|
+
propagateDeletePolicy: DeletePolicy;
|
|
849
|
+
emit: ShareEmit;
|
|
850
|
+
companyRef: string;
|
|
851
|
+
ctx: EntityContext;
|
|
852
|
+
syncRoot: string;
|
|
853
|
+
shouldSync: (filePath: string, isDir?: boolean) => boolean;
|
|
854
|
+
journalSlug: string;
|
|
855
|
+
journal: SyncJournal;
|
|
856
|
+
excludedSet: Set<string>;
|
|
857
|
+
excludedById: Record<string, number>;
|
|
858
|
+
scopeExcludedSet: Set<string>;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
interface ShareCounters {
|
|
862
|
+
filesUploaded: number;
|
|
863
|
+
bytesUploaded: number;
|
|
864
|
+
filesSkipped: number;
|
|
865
|
+
filesDeleted: number;
|
|
866
|
+
filesTombstoned: number;
|
|
867
|
+
filesRefusedStale: number;
|
|
868
|
+
filesSuppressedByTombstone: number;
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
interface SharePlans {
|
|
872
|
+
pushPlan: PushPlan;
|
|
873
|
+
deletePlan: DeletePlan;
|
|
874
|
+
decommissionPlan: string[];
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
interface UploadExecutionResult {
|
|
878
|
+
aborted: boolean;
|
|
879
|
+
abortFlightConflictPaths: string[];
|
|
880
|
+
workerErrors: Error[];
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
const REFUSED_STALE_PATH_CAP = 50;
|
|
884
|
+
|
|
885
|
+
async function createPushRunContext(options: ShareOptions): Promise<PushRunContext> {
|
|
803
886
|
const { paths, company, message, onConflict, vaultConfig, entityContext, hqRoot, skipUnchanged, propagateDeletes } = options;
|
|
804
|
-
|
|
805
|
-
// is on but the caller hasn't pinned a policy. Staged-default rollout
|
|
806
|
-
// (see CHANGELOG / PR for hq-cloud 5.24.0): 5.24 ships the currency-gated
|
|
807
|
-
// CODE PATH plus the conflict-mirror exclusion (which is policy-
|
|
808
|
-
// independent and immediately stops new litter), but holds the default
|
|
809
|
-
// flip to a later release after soak. Opt into the safer policy now via
|
|
810
|
-
// `propagateDeletePolicy: "currency-gated"` (explicit) or
|
|
811
|
-
// `HQ_SYNC_DELETE_POLICY=currency-gated` (env, honored by sync-runner).
|
|
812
|
-
// The default flip to `"currency-gated"` is scheduled for 5.25.0.
|
|
813
|
-
let propagateDeletePolicy: "currency-gated" | "owned-only" | "all" =
|
|
887
|
+
let propagateDeletePolicy: DeletePolicy =
|
|
814
888
|
options.propagateDeletePolicy ?? "owned-only";
|
|
815
889
|
const emit = options.onEvent ?? defaultConsoleLogger;
|
|
816
890
|
|
|
817
|
-
// Exactly-one-of contract: either we vend (vaultConfig) or the caller did
|
|
818
|
-
// (entityContext). Both supplied is ambiguous (which credentials win?), and
|
|
819
|
-
// neither leaves us with no way to talk to S3.
|
|
820
891
|
if (vaultConfig && entityContext) {
|
|
821
892
|
throw new Error(
|
|
822
893
|
"share() requires exactly one of `vaultConfig` or `entityContext`, not both. " +
|
|
@@ -830,9 +901,6 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
830
901
|
);
|
|
831
902
|
}
|
|
832
903
|
|
|
833
|
-
// Resolve company — slug, UID, or from active config. When the caller
|
|
834
|
-
// provided a pre-resolved entityContext, prefer its slug as the canonical
|
|
835
|
-
// ref (the caller already knows what entity these creds are for).
|
|
836
904
|
const companyRef =
|
|
837
905
|
company ?? entityContext?.slug ?? resolveActiveCompany(hqRoot);
|
|
838
906
|
if (!companyRef) {
|
|
@@ -842,67 +910,18 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
842
910
|
);
|
|
843
911
|
}
|
|
844
912
|
|
|
845
|
-
|
|
846
|
-
// 1. vaultConfig provided → resolveEntityContext does the lookup + STS vend
|
|
847
|
-
// (cached + auto-refreshable mid-run).
|
|
848
|
-
// 2. entityContext provided → use it directly. No lookup, no vending,
|
|
849
|
-
// no auto-refresh (we have no Cognito token to re-vend with).
|
|
850
|
-
// Caller is responsible for vending credentials with enough TTL to
|
|
851
|
-
// cover the run; if they under-vend, the AWS SDK surfaces ExpiredToken
|
|
852
|
-
// naturally on the first failing PUT.
|
|
853
|
-
let ctx: EntityContext = entityContext
|
|
913
|
+
const ctx: EntityContext = entityContext
|
|
854
914
|
? entityContext
|
|
855
915
|
: await resolveEntityContext(companyRef, vaultConfig!);
|
|
856
916
|
|
|
857
|
-
// Personal-vault policy correction (6.0.1). The `owned-only` rule encodes a
|
|
858
|
-
// multi-user curation premise — "don't tombstone peer-uploaded content even
|
|
859
|
-
// if my journal says I pulled it" — which is meaningful when several humans
|
|
860
|
-
// share a company bucket (a behind machine's first sync must not erase
|
|
861
|
-
// recent uploads from peers). On a personal vault that premise collapses:
|
|
862
|
-
// every file is the same human's content, just routed through different
|
|
863
|
-
// machines, and `direction: "down"` only means "uploaded from my laptop,
|
|
864
|
-
// pulled by my EC2" — it never means "uploaded by someone else." With
|
|
865
|
-
// `owned-only` in effect, `rm <file>` followed by `hq sync` silently fails
|
|
866
|
-
// to propagate the delete, leaving permanent vault litter (the May-27
|
|
867
|
-
// `personal/.obsidian/*.drift-*` files were diagnosed exactly this way).
|
|
868
|
-
// The etag-based `currency-gated` policy already captures the only safety
|
|
869
|
-
// intent that survives the single-user case ("don't tombstone if remote
|
|
870
|
-
// drifted since I last synced"); coerce to it here so the policy is right
|
|
871
|
-
// regardless of which caller's default landed. Explicit `"all"` is
|
|
872
|
-
// preserved — it's the emergency-reconcile opt-out and the caller has
|
|
873
|
-
// already asserted intent.
|
|
874
917
|
if (ctx.uid.startsWith("prs_") && propagateDeletePolicy === "owned-only") {
|
|
875
918
|
propagateDeletePolicy = "currency-gated";
|
|
876
919
|
}
|
|
877
920
|
|
|
878
|
-
// Remote keys are company-relative; the on-disk scoping prefix is
|
|
879
|
-
// companies/{slug}/. Anything outside this folder gets skipped to avoid
|
|
880
|
-
// leaking cross-company state into the vault.
|
|
881
|
-
//
|
|
882
|
-
// In personalMode the syncRoot is `hqRoot` itself — remote keys are
|
|
883
|
-
// hq-root-relative to match the Rust personal first-push (which uploads
|
|
884
|
-
// every non-excluded top-level dir under ~/HQ). The exclusion list is
|
|
885
|
-
// enforced upstream by the runner; share() just trusts `paths`.
|
|
886
921
|
const syncRoot = options.personalMode === true
|
|
887
922
|
? hqRoot
|
|
888
923
|
: path.join(hqRoot, "companies", ctx.slug);
|
|
889
924
|
|
|
890
|
-
// Personal-vault default exclusions (introduced in 5.25): wrap the base
|
|
891
|
-
// ignore filter so paths matching `PERSONAL_VAULT_DEFAULT_EXCLUSIONS` are
|
|
892
|
-
// rejected before they upload OR enter the delete plan. Refuses & warns —
|
|
893
|
-
// an already-leaked remote object stays put as an orphan; a separate one-
|
|
894
|
-
// shot purge handles legacy litter.
|
|
895
|
-
//
|
|
896
|
-
// Out-of-policy hits are deduplicated in `excludedSet` so the same path
|
|
897
|
-
// hitting the filter from both the upload walk and the delete-plan walk
|
|
898
|
-
// counts once. `excludedById` powers the per-rule breakdown on the
|
|
899
|
-
// `personal-vault-out-of-policy` event so UI can render which class
|
|
900
|
-
// (secret / machine-local / scratch / …) did the work.
|
|
901
|
-
//
|
|
902
|
-
// Company-mode syncs skip this wrap entirely — company vaults have their
|
|
903
|
-
// own first-push protection (settings/, data/, workers/, .git/) defined
|
|
904
|
-
// in hq-sync's Rust util/ignore.rs, and a company may legitimately ship
|
|
905
|
-
// `output/` or `.env*` paths inside its `companies/{slug}/data/` folder.
|
|
906
925
|
const ignoreFilter = createIgnoreFilter(hqRoot);
|
|
907
926
|
const excludedSet = new Set<string>();
|
|
908
927
|
const excludedById: Record<string, number> = {};
|
|
@@ -911,12 +930,6 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
911
930
|
excludedSet.add(rel);
|
|
912
931
|
excludedById[match.id] = (excludedById[match.id] ?? 0) + 1;
|
|
913
932
|
};
|
|
914
|
-
// ACL scope filter (member/guest scoped push). The vended child credential
|
|
915
|
-
// is scoped to `options.prefixSet`; any candidate outside those prefixes
|
|
916
|
-
// would draw the server's correct 403 SCOPE_EXCEEDS_PARENT on PUT and abort
|
|
917
|
-
// the whole company. Pre-filter the plan to the granted subset instead —
|
|
918
|
-
// the push-side analogue of the pull leg's `skip-out-of-scope` (US-005).
|
|
919
|
-
// `undefined` = owner/`all` → no scope filter (full access).
|
|
920
933
|
const scopeExcludedSet = new Set<string>();
|
|
921
934
|
const onScopeExcluded = (rel: string) => {
|
|
922
935
|
scopeExcludedSet.add(rel);
|
|
@@ -928,124 +941,93 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
928
941
|
? wrapFilterWithScope(baseFilter, syncRoot, options.prefixSet, onScopeExcluded)
|
|
929
942
|
: baseFilter;
|
|
930
943
|
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
931
|
-
// Seed the canonical personal-vault journal from the legacy `personal` file
|
|
932
|
-
// exactly once — engine-side so every consumer (sync-runner, hq-cli) gets
|
|
933
|
-
// it; see the matching guard in sync.ts.
|
|
934
944
|
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
|
|
935
945
|
const journal = readJournal(journalSlug);
|
|
936
946
|
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
947
|
+
return {
|
|
948
|
+
options,
|
|
949
|
+
paths,
|
|
950
|
+
message,
|
|
951
|
+
onConflict,
|
|
952
|
+
vaultConfig,
|
|
953
|
+
entityContext,
|
|
954
|
+
hqRoot,
|
|
955
|
+
skipUnchanged,
|
|
956
|
+
propagateDeletes,
|
|
957
|
+
propagateDeletePolicy,
|
|
958
|
+
emit,
|
|
959
|
+
companyRef,
|
|
960
|
+
ctx,
|
|
961
|
+
syncRoot,
|
|
962
|
+
shouldSync,
|
|
963
|
+
journalSlug,
|
|
964
|
+
journal,
|
|
965
|
+
excludedSet,
|
|
966
|
+
excludedById,
|
|
967
|
+
scopeExcludedSet,
|
|
968
|
+
};
|
|
969
|
+
}
|
|
956
970
|
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
971
|
+
function createShareCounters(): ShareCounters {
|
|
972
|
+
return {
|
|
973
|
+
filesUploaded: 0,
|
|
974
|
+
bytesUploaded: 0,
|
|
975
|
+
filesSkipped: 0,
|
|
976
|
+
filesDeleted: 0,
|
|
977
|
+
filesTombstoned: 0,
|
|
978
|
+
filesRefusedStale: 0,
|
|
979
|
+
filesSuppressedByTombstone: 0,
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
async function buildSharePlans(run: PushRunContext): Promise<SharePlans> {
|
|
984
|
+
const filesToShare = collectFiles(
|
|
985
|
+
run.paths,
|
|
986
|
+
run.hqRoot,
|
|
987
|
+
run.syncRoot,
|
|
988
|
+
run.shouldSync,
|
|
989
|
+
);
|
|
990
|
+
const pushPlan = computePushPlan(
|
|
991
|
+
filesToShare,
|
|
992
|
+
run.journal,
|
|
993
|
+
run.skipUnchanged === true,
|
|
994
|
+
);
|
|
995
|
+
const deleteScopeRoots = run.propagateDeletes === true
|
|
996
|
+
? resolveDeleteScopeRoots(run.paths, run.hqRoot, run.syncRoot)
|
|
973
997
|
: [];
|
|
974
|
-
const deletePlan: DeletePlan = propagateDeletes === true
|
|
998
|
+
const deletePlan: DeletePlan = run.propagateDeletes === true
|
|
975
999
|
? await computeDeletePlan(
|
|
976
|
-
journal,
|
|
977
|
-
syncRoot,
|
|
1000
|
+
run.journal,
|
|
1001
|
+
run.syncRoot,
|
|
978
1002
|
deleteScopeRoots,
|
|
979
|
-
shouldSync,
|
|
980
|
-
propagateDeletePolicy,
|
|
981
|
-
ctx,
|
|
1003
|
+
run.shouldSync,
|
|
1004
|
+
run.propagateDeletePolicy,
|
|
1005
|
+
run.ctx,
|
|
982
1006
|
)
|
|
983
1007
|
: { toDelete: [], toTombstone: [], refusedStale: [] };
|
|
984
|
-
|
|
985
|
-
// Decommission plan: journal entries under explicit prefixes the caller
|
|
986
|
-
// has asserted no longer belong in this bucket (typically a promoted
|
|
987
|
-
// company's `companies/{slug}/` keys in the personal bucket). Independent
|
|
988
|
-
// of `propagateDeletes` and DOES NOT require the local file to be missing
|
|
989
|
-
// — the caller is making a stronger claim than "local says delete this".
|
|
990
|
-
// Honors the same owned-only safety policy so a misconfigured caller
|
|
991
|
-
// can never erase content the journal records as pulled from elsewhere.
|
|
992
|
-
// Dedupes against `deletePlan` so a key in both plans is only processed
|
|
993
|
-
// once (DeleteObject is idempotent on S3 but the journal-write would
|
|
994
|
-
// race a no-op pass through the loop body).
|
|
995
1008
|
const decommissionPlan =
|
|
996
|
-
(options.decommissionPrefixes ?? []).length > 0
|
|
1009
|
+
(run.options.decommissionPrefixes ?? []).length > 0
|
|
997
1010
|
? computeDecommissionPlan(
|
|
998
|
-
journal,
|
|
999
|
-
options.decommissionPrefixes ?? [],
|
|
1000
|
-
propagateDeletePolicy,
|
|
1001
|
-
// Dedup against `toDelete` (decommission and propagate-delete
|
|
1002
|
-
// would both issue DeleteObject — single call wins) and against
|
|
1003
|
-
// `toTombstone` (the remote is already 404; the tombstone loop
|
|
1004
|
-
// drops the journal entry without a network call — decommission
|
|
1005
|
-
// yields, both for efficiency and to avoid emitting two
|
|
1006
|
-
// "deleted" events for the same key).
|
|
1007
|
-
//
|
|
1008
|
-
// We do NOT dedup against `refusedStale`. A key whose remote
|
|
1009
|
-
// ETag drifted (peer wrote a newer version) but which decommission
|
|
1010
|
-
// claims should still be removed — the caller has asserted this
|
|
1011
|
-
// key doesn't belong in this bucket regardless of peer activity.
|
|
1012
|
-
// Under owned-only (default) `computeDecommissionPlan`'s
|
|
1013
|
-
// direction:'up' filter already excludes peer-written entries;
|
|
1014
|
-
// under policy:'all' the caller has opted out of that safety
|
|
1015
|
-
// anyway. The refusedStale loop below filters out keys we're
|
|
1016
|
-
// about to decommission to avoid emitting a spurious "kept on
|
|
1017
|
-
// remote" event for content we're deleting.
|
|
1011
|
+
run.journal,
|
|
1012
|
+
run.options.decommissionPrefixes ?? [],
|
|
1013
|
+
run.propagateDeletePolicy,
|
|
1018
1014
|
new Set([...deletePlan.toDelete, ...deletePlan.toTombstone]),
|
|
1019
1015
|
)
|
|
1020
1016
|
: [];
|
|
1021
1017
|
|
|
1022
|
-
emit({
|
|
1018
|
+
run.emit({
|
|
1023
1019
|
type: "plan",
|
|
1024
|
-
// share() is push-only; pull counts are sourced from sync()'s plan event.
|
|
1025
1020
|
filesToDownload: 0,
|
|
1026
1021
|
bytesToDownload: 0,
|
|
1027
|
-
filesToUpload:
|
|
1028
|
-
bytesToUpload:
|
|
1029
|
-
filesToSkip:
|
|
1030
|
-
// Push conflicts require a remote HEAD; we don't yet do that in Stage 1,
|
|
1031
|
-
// so this stays 0. V1.5 (single LIST) will let us classify them up-front.
|
|
1022
|
+
filesToUpload: pushPlan.filesToUpload,
|
|
1023
|
+
bytesToUpload: pushPlan.bytesToUpload,
|
|
1024
|
+
filesToSkip: pushPlan.filesToSkip,
|
|
1032
1025
|
filesToConflict: 0,
|
|
1033
|
-
// Reported count is the deletes we're actually going to issue — does NOT
|
|
1034
|
-
// include tombstones (no S3 call) or refused-stale (no journal change).
|
|
1035
|
-
// Refusals surface as their own event stream so consumers that care can
|
|
1036
|
-
// render a "kept on remote: N" line separately. `decommissionPlan` adds
|
|
1037
|
-
// to this count because every decommission entry IS an issued
|
|
1038
|
-
// DeleteObject (different intent than propagate-deletes, same network
|
|
1039
|
-
// effect).
|
|
1040
1026
|
filesToDelete: deletePlan.toDelete.length + decommissionPlan.length,
|
|
1041
1027
|
});
|
|
1042
1028
|
|
|
1043
|
-
// Bulk-asymmetry summary event. Emitted once if the circuit-breaker
|
|
1044
|
-
// tripped inside `computeDeletePlan` — see DeletePlan.bulkAsymmetry doc.
|
|
1045
|
-
// Per-key refusal events fire later in the refusedStale loop with
|
|
1046
|
-
// reason: "bulk-asymmetry" so the UI can also show the affected paths.
|
|
1047
1029
|
if (deletePlan.bulkAsymmetry) {
|
|
1048
|
-
emit({
|
|
1030
|
+
run.emit({
|
|
1049
1031
|
type: "delete-refused-bulk-asymmetry",
|
|
1050
1032
|
candidates: deletePlan.bulkAsymmetry.candidates,
|
|
1051
1033
|
inScope: deletePlan.bulkAsymmetry.inScope,
|
|
@@ -1054,213 +1036,111 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1054
1036
|
});
|
|
1055
1037
|
}
|
|
1056
1038
|
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
// Abort handling: when any item's conflict resolution is "abort", we
|
|
1068
|
-
// set `aborted = true` so the pool stops queueing new items, drain
|
|
1069
|
-
// in-flight cleanly, and short-circuit to the abort return. In-flight
|
|
1070
|
-
// PUTs that already issued will complete (S3 doesn't have client-side
|
|
1071
|
-
// cancellation in this code path); their results are still recorded on
|
|
1072
|
-
// the journal so the next sync's planner doesn't re-fire them.
|
|
1073
|
-
// Interactive-mode prompts: when `onConflict` is unset the per-item conflict
|
|
1074
|
-
// path calls resolveConflict()'s readline prompt on process.stdin, and two
|
|
1075
|
-
// pool workers prompting at once would race for the terminal and interleave
|
|
1076
|
-
// answers. The 5.36.x guard solved this by forcing the WHOLE pool to
|
|
1077
|
-
// concurrency=1 — which made an interactive `hq sync now` crawl even when
|
|
1078
|
-
// zero conflicts existed (every transfer serialized just in case one might
|
|
1079
|
-
// prompt). Instead, keep full env-tunable concurrency and serialize ONLY the
|
|
1080
|
-
// prompt (see `resolveConflictSerialized` below): at most one prompt awaits
|
|
1081
|
-
// input at a time while transfers stay parallel.
|
|
1082
|
-
const TRANSFER_CONCURRENCY = (() => {
|
|
1083
|
-
const raw = process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
|
|
1084
|
-
if (raw === undefined || raw === "") return 16;
|
|
1085
|
-
const parsed = Number.parseInt(raw, 10);
|
|
1086
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 16;
|
|
1087
|
-
})();
|
|
1088
|
-
|
|
1089
|
-
// Chained lock around the (possibly interactive) conflict prompt. Each
|
|
1090
|
-
// resolveConflict() runs only after the previous one settles, so concurrent
|
|
1091
|
-
// pool workers never prompt over each other on stdin — without dropping the
|
|
1092
|
-
// transfer pool's parallelism. A rejected prompt must not wedge the chain,
|
|
1093
|
-
// so the link swallows errors (the original promise still rejects to its
|
|
1094
|
-
// awaiter). In non-interactive mode resolveConflict applies the configured
|
|
1095
|
-
// strategy without reading stdin, so the lock adds no real serialization.
|
|
1039
|
+
return { pushPlan, deletePlan, decommissionPlan };
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
async function executeUploads(
|
|
1043
|
+
run: PushRunContext,
|
|
1044
|
+
pushPlan: PushPlan,
|
|
1045
|
+
counters: ShareCounters,
|
|
1046
|
+
conflictPaths: string[],
|
|
1047
|
+
): Promise<UploadExecutionResult> {
|
|
1048
|
+
const TRANSFER_CONCURRENCY = resolveTransferConcurrency();
|
|
1096
1049
|
let conflictPromptChain: Promise<unknown> = Promise.resolve();
|
|
1097
1050
|
const resolveConflictSerialized = (
|
|
1098
1051
|
info: Parameters<typeof resolveConflict>[0],
|
|
1099
1052
|
): ReturnType<typeof resolveConflict> => {
|
|
1100
|
-
const
|
|
1101
|
-
|
|
1053
|
+
const resolution = conflictPromptChain.then(() =>
|
|
1054
|
+
resolveConflict(info, run.onConflict),
|
|
1055
|
+
);
|
|
1056
|
+
conflictPromptChain = resolution.then(
|
|
1102
1057
|
() => undefined,
|
|
1103
1058
|
() => undefined,
|
|
1104
1059
|
);
|
|
1105
|
-
return
|
|
1060
|
+
return resolution;
|
|
1106
1061
|
};
|
|
1107
1062
|
|
|
1108
|
-
// Push-side FILE_TOMBSTONE consult (delete-resync) — symmetric to the pull
|
|
1109
|
-
// planner's suppression in sync.ts. An authoritative delete
|
|
1110
|
-
// (`hq files delete <prefix>`) writes a FILE_TOMBSTONE and removes the S3
|
|
1111
|
-
// object; the pull side already honors it. But the PUSH side did not: a behind
|
|
1112
|
-
// peer who still holds the deleted file locally would re-upload it here, and
|
|
1113
|
-
// because the re-uploaded object post-dates the tombstone, the pull planner's
|
|
1114
|
-
// timestamp-only re-create heuristic (`isRemoteRecreateAfterTombstone`) treats
|
|
1115
|
-
// it as a genuine re-create and resurrects the key for EVERYONE — defeating the
|
|
1116
|
-
// authoritative delete. Consult the tombstones so a stale-baseline upload is
|
|
1117
|
-
// skipped at the source.
|
|
1118
|
-
//
|
|
1119
|
-
// Source: an injected `fileTombstones` (a sync run can hand the push leg the
|
|
1120
|
-
// map it already fetched for the pull leg), else a self-fetch for COMPANY
|
|
1121
|
-
// vaults that have a `vaultConfig`. Personal vaults have no company tombstones
|
|
1122
|
-
// (the pull side skips them too), and `entityContext`-only callers have no
|
|
1123
|
-
// auth to fetch with — both degrade to an empty map (no suppression), the
|
|
1124
|
-
// safe/legacy direction.
|
|
1125
1063
|
const fileTombstones: Map<string, CompanyTombstone> =
|
|
1126
|
-
options.fileTombstones ??
|
|
1127
|
-
(!ctx.uid.startsWith("prs_") && vaultConfig
|
|
1128
|
-
? await fetchCompanyTombstones(vaultConfig, ctx.uid)
|
|
1064
|
+
run.options.fileTombstones ??
|
|
1065
|
+
(!run.ctx.uid.startsWith("prs_") && run.vaultConfig
|
|
1066
|
+
? await fetchCompanyTombstones(run.vaultConfig, run.ctx.uid)
|
|
1129
1067
|
: new Map<string, CompanyTombstone>());
|
|
1130
1068
|
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
const uploadItems: Array<typeof plan.items[number] & { action: "upload" }> = [];
|
|
1134
|
-
for (const item of plan.items) {
|
|
1069
|
+
const uploadItems: UploadPlanItem[] = [];
|
|
1070
|
+
for (const item of pushPlan.items) {
|
|
1135
1071
|
if (item.action === "skip-size-limit") {
|
|
1136
|
-
emit({
|
|
1072
|
+
run.emit({
|
|
1137
1073
|
type: "error",
|
|
1138
1074
|
path: item.relativePath,
|
|
1139
1075
|
message: "file exceeds size limit",
|
|
1140
1076
|
});
|
|
1141
|
-
filesSkipped++;
|
|
1077
|
+
counters.filesSkipped++;
|
|
1142
1078
|
continue;
|
|
1143
1079
|
}
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
filesSuppressedByTombstone++;
|
|
1159
|
-
emit({
|
|
1160
|
-
type: "upload-suppressed-tombstone",
|
|
1161
|
-
path: item.relativePath,
|
|
1162
|
-
deletedAt: ts.deletedAt,
|
|
1163
|
-
});
|
|
1164
|
-
continue;
|
|
1080
|
+
if (item.action === "upload") {
|
|
1081
|
+
if (fileTombstones.size > 0) {
|
|
1082
|
+
const ts = fileTombstones.get(toPosixKey(item.relativePath));
|
|
1083
|
+
if (ts !== undefined) {
|
|
1084
|
+
const entry = run.journal.files[item.relativePath];
|
|
1085
|
+
if (entry && entry.hash === item.localHash) {
|
|
1086
|
+
counters.filesSuppressedByTombstone++;
|
|
1087
|
+
run.emit({
|
|
1088
|
+
type: "upload-suppressed-tombstone",
|
|
1089
|
+
path: item.relativePath,
|
|
1090
|
+
deletedAt: ts.deletedAt,
|
|
1091
|
+
});
|
|
1092
|
+
continue;
|
|
1093
|
+
}
|
|
1165
1094
|
}
|
|
1166
1095
|
}
|
|
1096
|
+
uploadItems.push(item);
|
|
1097
|
+
continue;
|
|
1167
1098
|
}
|
|
1168
|
-
if (item.
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
// never alters the content hash, so this stays a pure no-op skip. The
|
|
1174
|
-
// journal is persisted unconditionally by writeJournal at the end of the
|
|
1175
|
-
// run, so this survives even when nothing uploads.
|
|
1176
|
-
if (item.restamp) {
|
|
1177
|
-
const existing = journal.files[item.relativePath];
|
|
1178
|
-
if (existing && existing.hash) {
|
|
1179
|
-
existing.mtimeMs = item.restamp.mtimeMs;
|
|
1180
|
-
existing.size = item.restamp.size;
|
|
1181
|
-
}
|
|
1099
|
+
if (item.restamp) {
|
|
1100
|
+
const existing = run.journal.files[item.relativePath];
|
|
1101
|
+
if (existing && existing.hash) {
|
|
1102
|
+
existing.mtimeMs = item.restamp.mtimeMs;
|
|
1103
|
+
existing.size = item.restamp.size;
|
|
1182
1104
|
}
|
|
1183
|
-
filesSkipped++;
|
|
1184
|
-
continue;
|
|
1185
1105
|
}
|
|
1186
|
-
|
|
1106
|
+
counters.filesSkipped++;
|
|
1187
1107
|
}
|
|
1188
1108
|
|
|
1189
|
-
// Batch pre-mint PUT URLs (+ the created-at HEADs) for the whole upload set,
|
|
1190
|
-
// signing the SAME metadata the pool below computes so each task replays the
|
|
1191
|
-
// cached headers and skips its own presign. Turns an N-file push from ~N
|
|
1192
|
-
// presign calls into ceil(N/1000) GET + ceil(N/1000) PUT — keeping a bulk
|
|
1193
|
-
// push under the 100/hr limit. No-op on the S3 SDK transport; best-effort.
|
|
1194
1109
|
await primeUploads(
|
|
1195
|
-
ctx,
|
|
1110
|
+
run.ctx,
|
|
1196
1111
|
uploadItems.map((it) => ({
|
|
1197
1112
|
key: it.relativePath,
|
|
1198
1113
|
localPath: it.absolutePath,
|
|
1199
1114
|
isSymlink: it.kind === "symlink",
|
|
1200
|
-
author: options.author,
|
|
1115
|
+
author: run.options.author,
|
|
1201
1116
|
})),
|
|
1202
1117
|
);
|
|
1118
|
+
// Warm the GET presigns the per-item conflict HEAD (remoteMeta) reuses, so a
|
|
1119
|
+
// large upload set doesn't mint one presign per HEAD and burst/trip the
|
|
1120
|
+
// presign breaker. Mirrors the new-files + tombstone pre-primes on the pull.
|
|
1121
|
+
await primeObjectTransport(
|
|
1122
|
+
run.ctx,
|
|
1123
|
+
"get",
|
|
1124
|
+
uploadItems.map((it) => it.relativePath),
|
|
1125
|
+
);
|
|
1203
1126
|
|
|
1204
|
-
// Phase B: parallel upload pool. Each task runs the full per-item flow
|
|
1205
|
-
// (HEAD + conflict + PUT + journal stamp + emit). Aborts flip the
|
|
1206
|
-
// shared `aborted` flag and the pool stops draining the queue; tasks
|
|
1207
|
-
// already in flight complete normally.
|
|
1208
1127
|
let aborted = false;
|
|
1209
1128
|
let abortFlightConflictPaths: string[] = [];
|
|
1210
1129
|
|
|
1211
|
-
const processUploadItem = async (
|
|
1212
|
-
item: typeof uploadItems[number],
|
|
1213
|
-
): Promise<void> => {
|
|
1130
|
+
const processUploadItem = async (item: UploadPlanItem): Promise<void> => {
|
|
1214
1131
|
if (aborted) return;
|
|
1215
1132
|
const { absolutePath, relativePath, localHash } = item;
|
|
1216
1133
|
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
// from, so we let the AWS SDK surface ExpiredToken naturally on the
|
|
1220
|
-
// PUT below if the caller under-vended.
|
|
1221
|
-
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
1222
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1134
|
+
if (run.vaultConfig && isExpiringSoon(run.ctx.expiresAt)) {
|
|
1135
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
1223
1136
|
}
|
|
1224
1137
|
|
|
1225
|
-
// Check for remote conflict — refuse to overwrite newer remote version.
|
|
1226
|
-
//
|
|
1227
|
-
// A real conflict requires BOTH sides to have moved since the last sync.
|
|
1228
|
-
// The previous predicate only checked `journalEntry.hash !== localHash`,
|
|
1229
|
-
// which mislabelled every local edit as a conflict and (combined with
|
|
1230
|
-
// `--on-conflict keep`) silently dropped the user's edit. We now compare
|
|
1231
|
-
// the current remote ETag against the one captured at last sync; when
|
|
1232
|
-
// missing (legacy entries), we fall back to the same `lastModified >
|
|
1233
|
-
// syncedAt` heuristic the pull side uses.
|
|
1234
|
-
//
|
|
1235
|
-
// Bug #7 (data-loss class — see workspace/reports/hq-cloud-5.33.0-
|
|
1236
|
-
// deep-test.md): for a path with NO prior journal entry (first push
|
|
1237
|
-
// from this machine), the localChanged/remoteChanged predicates above
|
|
1238
|
-
// both evaluate FALSE (their guards require `!!journalEntry`). Push
|
|
1239
|
-
// fell through to an unconditional PUT, silently clobbering any
|
|
1240
|
-
// peer's content already at that key. The verification report's V7
|
|
1241
|
-
// isolated this — the bug is independent of \`--on-conflict\` mode;
|
|
1242
|
-
// it's keyed on "do I have a prior journal entry?" not on the flag.
|
|
1243
|
-
//
|
|
1244
|
-
// Fresh-collision branch: when remoteMeta exists and there's no
|
|
1245
|
-
// journal entry, hash the local body (MD5 for parity with S3's
|
|
1246
|
-
// single-part etag) and compare. Match → no conflict, silently skip
|
|
1247
|
-
// the PUT (the bytes are already there). Mismatch → treat as a
|
|
1248
|
-
// conflict in the same shared branch below.
|
|
1249
|
-
// Defense-in-depth for the scoped-push 403: the `prefixSet` filter above
|
|
1250
|
-
// should already have dropped any out-of-scope key from the plan, but a
|
|
1251
|
-
// grant that changed mid-run, a pinned prefix outside the grant, or
|
|
1252
|
-
// prefix-coalesce imprecision can still leave an out-of-scope key here.
|
|
1253
|
-
// This HEAD sits OUTSIDE the per-file PUT try/catch below, so a thrown
|
|
1254
|
-
// 403 used to bubble to `workerErrors` and abort the ENTIRE company with
|
|
1255
|
-
// a generic message and exit 2. Catch the access-denied class, surface
|
|
1256
|
-
// the offending PATH clearly, and skip just this key — the rest of the
|
|
1257
|
-
// company still syncs. Non-access-denied errors re-throw unchanged.
|
|
1258
1138
|
let remoteMeta: Awaited<ReturnType<typeof headRemoteFile>>;
|
|
1259
1139
|
try {
|
|
1260
|
-
remoteMeta = await headRemoteFile(ctx, relativePath);
|
|
1140
|
+
remoteMeta = await headRemoteFile(run.ctx, relativePath);
|
|
1261
1141
|
} catch (headErr) {
|
|
1262
1142
|
if (isAccessDenied(headErr)) {
|
|
1263
|
-
emit({
|
|
1143
|
+
run.emit({
|
|
1264
1144
|
type: "error",
|
|
1265
1145
|
path: relativePath,
|
|
1266
1146
|
message:
|
|
@@ -1273,19 +1153,15 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1273
1153
|
throw headErr;
|
|
1274
1154
|
}
|
|
1275
1155
|
if (remoteMeta) {
|
|
1276
|
-
const journalEntry = journal.files[relativePath];
|
|
1156
|
+
const journalEntry = run.journal.files[relativePath];
|
|
1277
1157
|
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
1278
1158
|
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
|
|
1279
1159
|
|
|
1280
1160
|
let isFreshCollision = false;
|
|
1281
1161
|
let multipartConverged = false;
|
|
1282
|
-
if (!journalEntry && item.kind === "
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
// skip the check entirely — the wire body shape (\`hq-symlink:\`
|
|
1286
|
-
// prefix + target) isn't a pure byte mirror and would mis-
|
|
1287
|
-
// classify; symlink overwrites are rare and an audit pass after
|
|
1288
|
-
// the broader bug-cleanup wave can extend coverage if needed.
|
|
1162
|
+
if (!journalEntry && item.kind === "symlink") {
|
|
1163
|
+
isFreshCollision = true;
|
|
1164
|
+
} else if (!journalEntry && item.kind === "file") {
|
|
1289
1165
|
const remoteEtagNormalized = normalizeEtag(remoteMeta.etag);
|
|
1290
1166
|
const isMultipart = /-\d+$/.test(remoteEtagNormalized);
|
|
1291
1167
|
if (!isMultipart) {
|
|
@@ -1294,38 +1170,16 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1294
1170
|
if (localMd5 !== remoteEtagNormalized) {
|
|
1295
1171
|
isFreshCollision = true;
|
|
1296
1172
|
}
|
|
1297
|
-
// Match → bytes are already there; fall through to upload
|
|
1298
|
-
// path which is idempotent (S3 will overwrite with identical
|
|
1299
|
-
// content + carry our metadata). Cheap, no behavior change.
|
|
1300
1173
|
} else {
|
|
1301
|
-
// Multipart remote etag is \`<md5>-<partCount>\`, NOT a usable
|
|
1302
|
-
// content hash, so — unlike the single-part branch — we cannot
|
|
1303
|
-
// decide collision-vs-identical from the etag alone. The old
|
|
1304
|
-
// behavior assumed a collision here, which minted a FALSE
|
|
1305
|
-
// conflict for the most common fresh-install case: re-pushing a
|
|
1306
|
-
// byte-identical \`core/\` scaffold file whose remote copy happened
|
|
1307
|
-
// to be multipart-uploaded. Every fresh install that hit an
|
|
1308
|
-
// already-populated bucket therefore came up "with conflicts".
|
|
1309
|
-
//
|
|
1310
|
-
// Instead, fetch the remote bytes once and compare content
|
|
1311
|
-
// hashes directly — the same convergence guard the pull side
|
|
1312
|
-
// uses (sync.ts). Identical content is NOT a conflict. On any
|
|
1313
|
-
// fetch/hash failure we fail safe to "conflict" (false positives
|
|
1314
|
-
// prompt the operator; false negatives risk clobbering a peer).
|
|
1315
1174
|
const remoteDiffers = await remoteContentDiffers(
|
|
1316
|
-
ctx,
|
|
1175
|
+
run.ctx,
|
|
1317
1176
|
relativePath,
|
|
1318
1177
|
localHash,
|
|
1319
|
-
hqRoot,
|
|
1178
|
+
run.hqRoot,
|
|
1320
1179
|
);
|
|
1321
1180
|
if (remoteDiffers) {
|
|
1322
1181
|
isFreshCollision = true;
|
|
1323
1182
|
} else {
|
|
1324
|
-
// Byte-identical multipart object already present. Seed the
|
|
1325
|
-
// journal baseline from the remote so the next sync sees no
|
|
1326
|
-
// change on either side, and skip the redundant PUT —
|
|
1327
|
-
// re-uploading would needlessly rewrite remote and churn its
|
|
1328
|
-
// etag from multipart to single-part.
|
|
1329
1183
|
multipartConverged = true;
|
|
1330
1184
|
}
|
|
1331
1185
|
}
|
|
@@ -1334,7 +1188,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1334
1188
|
if (multipartConverged) {
|
|
1335
1189
|
const lstat = fs.lstatSync(absolutePath);
|
|
1336
1190
|
updateEntry(
|
|
1337
|
-
journal,
|
|
1191
|
+
run.journal,
|
|
1338
1192
|
relativePath,
|
|
1339
1193
|
localHash,
|
|
1340
1194
|
lstat.size,
|
|
@@ -1342,8 +1196,8 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1342
1196
|
remoteMeta.etag,
|
|
1343
1197
|
lstat.mtimeMs,
|
|
1344
1198
|
);
|
|
1345
|
-
emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
1346
|
-
filesSkipped++;
|
|
1199
|
+
run.emit({ type: "reconciled", path: relativePath, direction: "push" });
|
|
1200
|
+
counters.filesSkipped++;
|
|
1347
1201
|
return;
|
|
1348
1202
|
}
|
|
1349
1203
|
|
|
@@ -1357,7 +1211,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1357
1211
|
direction: "push",
|
|
1358
1212
|
});
|
|
1359
1213
|
|
|
1360
|
-
emit({
|
|
1214
|
+
run.emit({
|
|
1361
1215
|
type: "conflict",
|
|
1362
1216
|
path: relativePath,
|
|
1363
1217
|
direction: "push",
|
|
@@ -1365,124 +1219,51 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1365
1219
|
});
|
|
1366
1220
|
|
|
1367
1221
|
if (resolution === "abort") {
|
|
1368
|
-
// Flip the shared aborted flag — the pool drainer below sees this
|
|
1369
|
-
// and stops queueing new items. In-flight tasks complete normally
|
|
1370
|
-
// (S3 PUTs have no client-side cancel here). The outer abort
|
|
1371
|
-
// return is built after the pool drains.
|
|
1372
1222
|
aborted = true;
|
|
1373
1223
|
abortFlightConflictPaths = [...conflictPaths];
|
|
1374
1224
|
return;
|
|
1375
1225
|
}
|
|
1376
1226
|
if (resolution === "keep" || resolution === "skip") {
|
|
1377
|
-
// Bug #7 mirror branch: when the resolution is keep/skip on a
|
|
1378
|
-
// FRESH collision (no prior journal entry), download the
|
|
1379
|
-
// remote bytes to \`<orig>.conflict-<ts>-<short>\` so both
|
|
1380
|
-
// versions survive on disk. Mirrors the pull-side mirror-write
|
|
1381
|
-
// routine in sync.ts exactly. Skipped for stale-journal
|
|
1382
|
-
// conflicts (the pre-Bug-#7 codepath) — those already produce
|
|
1383
|
-
// a pull-side mirror on the next sync cycle.
|
|
1384
1227
|
if (isFreshCollision) {
|
|
1385
|
-
|
|
1386
|
-
const detectedAt = new Date().toISOString();
|
|
1387
|
-
const machineId = readShortMachineId(hqRoot);
|
|
1388
|
-
const originalRelative = path.relative(hqRoot, absolutePath);
|
|
1389
|
-
const conflictRelative = buildConflictPath(
|
|
1390
|
-
originalRelative,
|
|
1391
|
-
detectedAt,
|
|
1392
|
-
machineId,
|
|
1393
|
-
);
|
|
1394
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
1395
|
-
await downloadFile(ctx, relativePath, conflictAbs);
|
|
1396
|
-
appendConflictEntry(hqRoot, {
|
|
1397
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1398
|
-
originalPath: originalRelative,
|
|
1399
|
-
conflictPath: conflictRelative,
|
|
1400
|
-
detectedAt,
|
|
1401
|
-
side: "push",
|
|
1402
|
-
machineId,
|
|
1403
|
-
localHash,
|
|
1404
|
-
remoteHash: normalizeEtag(remoteMeta.etag),
|
|
1405
|
-
});
|
|
1406
|
-
} catch (mirrorErr) {
|
|
1407
|
-
emit({
|
|
1408
|
-
type: "error",
|
|
1409
|
-
path: relativePath,
|
|
1410
|
-
message: `conflict mirror write failed: ${
|
|
1411
|
-
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1412
|
-
}`,
|
|
1413
|
-
});
|
|
1414
|
-
}
|
|
1228
|
+
await writePushConflictMirror(run, item, normalizeEtag(remoteMeta.etag));
|
|
1415
1229
|
}
|
|
1416
|
-
filesSkipped++;
|
|
1230
|
+
counters.filesSkipped++;
|
|
1417
1231
|
return;
|
|
1418
1232
|
}
|
|
1419
|
-
// "overwrite" falls through to upload
|
|
1420
1233
|
}
|
|
1421
1234
|
}
|
|
1422
1235
|
|
|
1423
|
-
// Re-check abort flag right before the PUT — if a peer task aborted
|
|
1424
|
-
// while we were waiting on HEAD, skip the PUT entirely. This is
|
|
1425
|
-
// belt-and-suspenders alongside the queue-drain check; without it,
|
|
1426
|
-
// up to TRANSFER_CONCURRENCY in-flight uploads could still issue PUTs
|
|
1427
|
-
// after the user signaled abort.
|
|
1428
1236
|
if (aborted) return;
|
|
1429
1237
|
|
|
1430
|
-
// Conditional-write fence (storage-level backstop for the entire
|
|
1431
|
-
// stale-clobber class). Every PUT asserts the remote state this pass
|
|
1432
|
-
// just inspected:
|
|
1433
|
-
// - remote exists → If-Match on the observed etag ("replace exactly
|
|
1434
|
-
// what I HEAD'd"). Closes the HEAD→PUT TOCTOU: a peer's write
|
|
1435
|
-
// landing in the window makes S3 itself reject with 412 instead of
|
|
1436
|
-
// this machine silently regressing the object.
|
|
1437
|
-
// - remote absent → If-None-Match:* ("create only"). If the HEAD was
|
|
1438
|
-
// wrong about absence (any transport/state bug — the 2026-06-10..12
|
|
1439
|
-
// regression storm's shape), the PUT 412s instead of clobbering.
|
|
1440
|
-
// Enforced natively on the SDK transport; on the presigned transport it
|
|
1441
|
-
// activates when files-presign signs the headers (see object-io.ts).
|
|
1442
1238
|
const precondition: PutPrecondition = remoteMeta
|
|
1443
1239
|
? { ifMatch: remoteMeta.etag }
|
|
1444
1240
|
: { ifNoneMatch: "*" };
|
|
1445
1241
|
|
|
1446
|
-
// Upload — symlinks go through uploadSymlink (zero-byte body + target
|
|
1447
|
-
// metadata), regular files through uploadFile (file contents). The
|
|
1448
|
-
// discriminator is item.kind set by computePushPlan; both branches
|
|
1449
|
-
// converge on the same journal/event update path below. Factored into a
|
|
1450
|
-
// closure so the 412 "overwrite" resolution below can re-run it without
|
|
1451
|
-
// the fence after explicit user consent.
|
|
1452
1242
|
const performUpload = async (pc: PutPrecondition | undefined): Promise<void> => {
|
|
1453
1243
|
const isSymlinkUpload = item.kind === "symlink";
|
|
1454
|
-
// Capture lstat post-upload so size + mtimeMs stamped into the
|
|
1455
|
-
// journal reflect the bytes we actually shipped. lstat (not stat)
|
|
1456
|
-
// so a symlink stamps the link's own mtime, not the target's —
|
|
1457
|
-
// matches the fast-path's lstat comparison so the next sync can
|
|
1458
|
-
// skip without dereferencing.
|
|
1459
1244
|
const lstat = fs.lstatSync(absolutePath);
|
|
1460
1245
|
const size = isSymlinkUpload ? 0 : lstat.size;
|
|
1461
1246
|
const mtimeMs = lstat.mtimeMs;
|
|
1462
1247
|
|
|
1463
1248
|
const { etag } = isSymlinkUpload
|
|
1464
|
-
? await uploadSymlink(ctx, item.target, relativePath, options.author, pc)
|
|
1465
|
-
: await uploadFile(ctx, absolutePath, relativePath, options.author, pc);
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
journal.files[relativePath] = {
|
|
1474
|
-
...journal.files[relativePath],
|
|
1475
|
-
message,
|
|
1476
|
-
} as typeof journal.files[string] & { message: string };
|
|
1249
|
+
? await uploadSymlink(run.ctx, item.target, relativePath, run.options.author, pc)
|
|
1250
|
+
: await uploadFile(run.ctx, absolutePath, relativePath, run.options.author, pc);
|
|
1251
|
+
|
|
1252
|
+
updateEntry(run.journal, relativePath, localHash, size, "up", etag, mtimeMs);
|
|
1253
|
+
if (run.message) {
|
|
1254
|
+
run.journal.files[relativePath] = {
|
|
1255
|
+
...run.journal.files[relativePath],
|
|
1256
|
+
message: run.message,
|
|
1257
|
+
} as JournalFileEntry & { message: string };
|
|
1477
1258
|
}
|
|
1478
1259
|
|
|
1479
|
-
filesUploaded++;
|
|
1480
|
-
bytesUploaded += size;
|
|
1481
|
-
emit({
|
|
1260
|
+
counters.filesUploaded++;
|
|
1261
|
+
counters.bytesUploaded += size;
|
|
1262
|
+
run.emit({
|
|
1482
1263
|
type: "progress",
|
|
1483
1264
|
path: relativePath,
|
|
1484
1265
|
bytes: size,
|
|
1485
|
-
...(message ? { message } : {}),
|
|
1266
|
+
...(run.message ? { message: run.message } : {}),
|
|
1486
1267
|
});
|
|
1487
1268
|
};
|
|
1488
1269
|
|
|
@@ -1490,17 +1271,13 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1490
1271
|
await performUpload(precondition);
|
|
1491
1272
|
} catch (err) {
|
|
1492
1273
|
if (isPreconditionFailed(err)) {
|
|
1493
|
-
// The fence fired: the remote moved past (If-Match) or appeared at
|
|
1494
|
-
// (If-None-Match) this key between our HEAD and the PUT — exactly
|
|
1495
|
-
// the race the fence exists to catch. Surface as a push conflict;
|
|
1496
|
-
// never silently overwrite.
|
|
1497
1274
|
conflictPaths.push(relativePath);
|
|
1498
1275
|
const resolution = await resolveConflictSerialized({
|
|
1499
1276
|
path: relativePath,
|
|
1500
1277
|
localHash,
|
|
1501
1278
|
direction: "push",
|
|
1502
1279
|
});
|
|
1503
|
-
emit({
|
|
1280
|
+
run.emit({
|
|
1504
1281
|
type: "conflict",
|
|
1505
1282
|
path: relativePath,
|
|
1506
1283
|
direction: "push",
|
|
@@ -1512,12 +1289,10 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1512
1289
|
return;
|
|
1513
1290
|
}
|
|
1514
1291
|
if (resolution === "overwrite") {
|
|
1515
|
-
// Explicit clobber consent — retry once without the fence. A
|
|
1516
|
-
// second failure falls through to the generic error emit.
|
|
1517
1292
|
try {
|
|
1518
1293
|
await performUpload(undefined);
|
|
1519
1294
|
} catch (retryErr) {
|
|
1520
|
-
emit({
|
|
1295
|
+
run.emit({
|
|
1521
1296
|
type: "error",
|
|
1522
1297
|
path: relativePath,
|
|
1523
1298
|
message:
|
|
@@ -1526,46 +1301,15 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1526
1301
|
}
|
|
1527
1302
|
return;
|
|
1528
1303
|
}
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
const originalRelative = path.relative(hqRoot, absolutePath);
|
|
1536
|
-
const conflictRelative = buildConflictPath(
|
|
1537
|
-
originalRelative,
|
|
1538
|
-
detectedAt,
|
|
1539
|
-
machineId,
|
|
1540
|
-
);
|
|
1541
|
-
const conflictAbs = path.join(hqRoot, conflictRelative);
|
|
1542
|
-
await downloadFile(ctx, relativePath, conflictAbs);
|
|
1543
|
-
appendConflictEntry(hqRoot, {
|
|
1544
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1545
|
-
originalPath: originalRelative,
|
|
1546
|
-
conflictPath: conflictRelative,
|
|
1547
|
-
detectedAt,
|
|
1548
|
-
side: "push",
|
|
1549
|
-
machineId,
|
|
1550
|
-
localHash,
|
|
1551
|
-
// remoteMeta (if any) predates the racing write that fired the
|
|
1552
|
-
// fence — record what we knew ("" when the key was believed
|
|
1553
|
-
// absent); the mirror file carries the authoritative remote bytes.
|
|
1554
|
-
remoteHash: remoteMeta ? normalizeEtag(remoteMeta.etag) : "",
|
|
1555
|
-
});
|
|
1556
|
-
} catch (mirrorErr) {
|
|
1557
|
-
emit({
|
|
1558
|
-
type: "error",
|
|
1559
|
-
path: relativePath,
|
|
1560
|
-
message: `conflict mirror write failed: ${
|
|
1561
|
-
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1562
|
-
}`,
|
|
1563
|
-
});
|
|
1564
|
-
}
|
|
1565
|
-
filesSkipped++;
|
|
1304
|
+
await writePushConflictMirror(
|
|
1305
|
+
run,
|
|
1306
|
+
item,
|
|
1307
|
+
remoteMeta ? normalizeEtag(remoteMeta.etag) : "",
|
|
1308
|
+
);
|
|
1309
|
+
counters.filesSkipped++;
|
|
1566
1310
|
return;
|
|
1567
1311
|
}
|
|
1568
|
-
emit({
|
|
1312
|
+
run.emit({
|
|
1569
1313
|
type: "error",
|
|
1570
1314
|
path: relativePath,
|
|
1571
1315
|
message: isAccessDenied(err)
|
|
@@ -1579,25 +1323,6 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1579
1323
|
}
|
|
1580
1324
|
};
|
|
1581
1325
|
|
|
1582
|
-
// Drain the upload queue with bounded concurrency. Per-file progress
|
|
1583
|
-
// events fire from inside processUploadItem at file-settle time, so
|
|
1584
|
-
// cross-file event ordering is settle-order, not plan-walk-order — the
|
|
1585
|
-
// menubar's stream parser already tolerates per-company interleave so
|
|
1586
|
-
// this is shape-compatible. allSettled-style waiting (via Promise.race
|
|
1587
|
-
// on the in-flight set) keeps the pool topped up without idling on slow
|
|
1588
|
-
// members.
|
|
1589
|
-
//
|
|
1590
|
-
// Codex P1 (5.36.x): each worker promise is wrapped in a .catch that
|
|
1591
|
-
// records the error to `workerErrors` and resolves normally — so
|
|
1592
|
-
// `Promise.race(inFlight)` can never reject and unwind the drain loop
|
|
1593
|
-
// mid-flight. Without the wrap, an unhandled rejection inside
|
|
1594
|
-
// processUploadItem (e.g. headRemoteFile or refreshEntityContext, both
|
|
1595
|
-
// of which sit outside the per-item PUT try/catch) bubbled out of the
|
|
1596
|
-
// race, abandoned still-in-flight uploads that kept mutating remote
|
|
1597
|
-
// state, and skipped writeJournal — leaving remote and journal
|
|
1598
|
-
// permanently out of sync for the next run. After the pool fully
|
|
1599
|
-
// drains we throw an aggregated error so the caller still surfaces the
|
|
1600
|
-
// failure (and the journal reflects the writes that actually landed).
|
|
1601
1326
|
const workerErrors: Error[] = [];
|
|
1602
1327
|
{
|
|
1603
1328
|
const queue = [...uploadItems];
|
|
@@ -1617,96 +1342,86 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1617
1342
|
if (inFlight.size > 0) {
|
|
1618
1343
|
await Promise.race(Array.from(inFlight));
|
|
1619
1344
|
} else {
|
|
1620
|
-
// Aborted with nothing in flight → exit the drain loop.
|
|
1621
1345
|
break;
|
|
1622
1346
|
}
|
|
1623
1347
|
}
|
|
1624
1348
|
}
|
|
1625
1349
|
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1350
|
+
return { aborted, abortFlightConflictPaths, workerErrors };
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
async function writePushConflictMirror(
|
|
1354
|
+
run: PushRunContext,
|
|
1355
|
+
item: UploadPlanItem,
|
|
1356
|
+
remoteHash: string,
|
|
1357
|
+
): Promise<void> {
|
|
1358
|
+
try {
|
|
1359
|
+
const detectedAt = new Date().toISOString();
|
|
1360
|
+
const machineId = readShortMachineId(run.hqRoot);
|
|
1361
|
+
const originalRelative = path.relative(run.hqRoot, item.absolutePath);
|
|
1362
|
+
const conflictRelative = buildConflictPath(
|
|
1363
|
+
originalRelative,
|
|
1364
|
+
detectedAt,
|
|
1365
|
+
machineId,
|
|
1366
|
+
);
|
|
1367
|
+
const conflictAbs = path.join(run.hqRoot, conflictRelative);
|
|
1368
|
+
if (!isMaterializationPathStillContained(run.syncRoot, conflictAbs)) {
|
|
1369
|
+
run.emit({
|
|
1370
|
+
type: "error",
|
|
1371
|
+
path: item.relativePath,
|
|
1372
|
+
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
1373
|
+
});
|
|
1374
|
+
} else {
|
|
1375
|
+
await downloadFile(run.ctx, item.relativePath, conflictAbs);
|
|
1376
|
+
appendConflictEntry(run.hqRoot, {
|
|
1377
|
+
id: buildConflictId(originalRelative, detectedAt),
|
|
1378
|
+
originalPath: originalRelative,
|
|
1379
|
+
conflictPath: conflictRelative,
|
|
1380
|
+
detectedAt,
|
|
1381
|
+
side: "push",
|
|
1382
|
+
machineId,
|
|
1383
|
+
localHash: item.localHash,
|
|
1384
|
+
remoteHash,
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
} catch (mirrorErr) {
|
|
1388
|
+
run.emit({
|
|
1389
|
+
type: "error",
|
|
1390
|
+
path: item.relativePath,
|
|
1391
|
+
message:
|
|
1392
|
+
"conflict mirror write failed: " +
|
|
1393
|
+
(mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)),
|
|
1394
|
+
});
|
|
1660
1395
|
}
|
|
1396
|
+
}
|
|
1661
1397
|
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
// entry and remote object intact — the next run retries.
|
|
1670
|
-
// Decommission keys join this bucket because their effect is
|
|
1671
|
-
// identical (DeleteObject + journal removal); the difference is
|
|
1672
|
-
// intent only, and the dedupe at plan-computation time ensures we
|
|
1673
|
-
// don't double-issue for a key both plans matched.
|
|
1674
|
-
//
|
|
1675
|
-
// 2. `toTombstone` — the remote was 404 at HEAD time (cleaned up out
|
|
1676
|
-
// of band, e.g. someone hand-deleted via console). No DeleteObject
|
|
1677
|
-
// needed; just drop the journal entry so the journal converges with
|
|
1678
|
-
// reality. Emit a synthetic `progress` event with `deleted: true`
|
|
1679
|
-
// and bytes=0 so consumers see the convergence.
|
|
1680
|
-
//
|
|
1681
|
-
// 3. `refusedStale` — under `currency-gated`, the remote's current
|
|
1682
|
-
// ETag no longer matches the journal's recorded one. Some other
|
|
1683
|
-
// device modified the file since this device last synced it. Keep
|
|
1684
|
-
// the remote intact; keep the journal entry intact. The next pull
|
|
1685
|
-
// leg of `sync now` re-pulls naturally via the existing
|
|
1686
|
-
// `hasRemoteChanged` path. Emit a dedicated event so UIs can
|
|
1687
|
-
// surface the refusal without inferring it from absence.
|
|
1688
|
-
// Batch pre-mint DELETE URLs so a large delete set is ~ceil(N/1000) presign
|
|
1689
|
-
// calls, not N. No-op on the S3 SDK transport; best-effort.
|
|
1398
|
+
async function executeDeletes(
|
|
1399
|
+
run: PushRunContext,
|
|
1400
|
+
deletePlan: DeletePlan,
|
|
1401
|
+
decommissionPlan: string[],
|
|
1402
|
+
counters: ShareCounters,
|
|
1403
|
+
filesRefusedStalePaths: string[],
|
|
1404
|
+
): Promise<void> {
|
|
1690
1405
|
const deleteKeys = [...deletePlan.toDelete, ...decommissionPlan];
|
|
1691
|
-
await primeObjectTransport(ctx, "delete", deleteKeys);
|
|
1406
|
+
await primeObjectTransport(run.ctx, "delete", deleteKeys);
|
|
1692
1407
|
for (const relativePath of deleteKeys) {
|
|
1693
|
-
if (vaultConfig && isExpiringSoon(ctx.expiresAt)) {
|
|
1694
|
-
ctx = await refreshEntityContext(companyRef, vaultConfig);
|
|
1408
|
+
if (run.vaultConfig && isExpiringSoon(run.ctx.expiresAt)) {
|
|
1409
|
+
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
1695
1410
|
}
|
|
1696
1411
|
try {
|
|
1697
|
-
const entry = journal.files[relativePath];
|
|
1412
|
+
const entry = run.journal.files[relativePath];
|
|
1698
1413
|
const size = entry?.size ?? 0;
|
|
1699
|
-
await deleteRemoteFile(ctx, relativePath);
|
|
1700
|
-
removeEntry(journal, relativePath);
|
|
1701
|
-
filesDeleted++;
|
|
1702
|
-
emit({
|
|
1414
|
+
await deleteRemoteFile(run.ctx, relativePath);
|
|
1415
|
+
removeEntry(run.journal, relativePath);
|
|
1416
|
+
counters.filesDeleted++;
|
|
1417
|
+
run.emit({
|
|
1703
1418
|
type: "progress",
|
|
1704
1419
|
path: relativePath,
|
|
1705
1420
|
bytes: size,
|
|
1706
1421
|
deleted: true,
|
|
1707
1422
|
});
|
|
1708
1423
|
} catch (err) {
|
|
1709
|
-
emit({
|
|
1424
|
+
run.emit({
|
|
1710
1425
|
type: "error",
|
|
1711
1426
|
path: relativePath,
|
|
1712
1427
|
message: err instanceof Error ? err.message : String(err),
|
|
@@ -1714,9 +1429,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1714
1429
|
}
|
|
1715
1430
|
}
|
|
1716
1431
|
for (const relativePath of deletePlan.toTombstone) {
|
|
1717
|
-
removeEntry(journal, relativePath);
|
|
1718
|
-
filesTombstoned++;
|
|
1719
|
-
emit({
|
|
1432
|
+
removeEntry(run.journal, relativePath);
|
|
1433
|
+
counters.filesTombstoned++;
|
|
1434
|
+
run.emit({
|
|
1720
1435
|
type: "progress",
|
|
1721
1436
|
path: relativePath,
|
|
1722
1437
|
bytes: 0,
|
|
@@ -1724,20 +1439,15 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1724
1439
|
message: "tombstone (remote already 404)",
|
|
1725
1440
|
});
|
|
1726
1441
|
}
|
|
1727
|
-
// Decommission overrides refusedStale: a key whose peer drifted but
|
|
1728
|
-
// which the caller has claimed via `decommissionPrefixes` is processed
|
|
1729
|
-
// by the DeleteObject loop above; skip the refusal event here so the
|
|
1730
|
-
// event stream doesn't simultaneously claim "we deleted it" and "we
|
|
1731
|
-
// kept it on remote" for the same key.
|
|
1732
1442
|
const decommissionedSet =
|
|
1733
1443
|
decommissionPlan.length > 0 ? new Set(decommissionPlan) : null;
|
|
1734
1444
|
for (const refused of deletePlan.refusedStale) {
|
|
1735
1445
|
if (decommissionedSet && decommissionedSet.has(refused.key)) continue;
|
|
1736
|
-
filesRefusedStale++;
|
|
1446
|
+
counters.filesRefusedStale++;
|
|
1737
1447
|
if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
|
|
1738
1448
|
filesRefusedStalePaths.push(refused.key);
|
|
1739
1449
|
}
|
|
1740
|
-
emit({
|
|
1450
|
+
run.emit({
|
|
1741
1451
|
type: "delete-refused-stale-etag",
|
|
1742
1452
|
path: refused.key,
|
|
1743
1453
|
journalEtag: refused.journalEtag,
|
|
@@ -1745,78 +1455,74 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
|
|
|
1745
1455
|
reason: refused.reason,
|
|
1746
1456
|
});
|
|
1747
1457
|
}
|
|
1458
|
+
}
|
|
1748
1459
|
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
writeJournal(journalSlug, journal);
|
|
1460
|
+
function finalizeShareJournal(run: PushRunContext): void {
|
|
1461
|
+
run.journal.lastSync = new Date().toISOString();
|
|
1462
|
+
writeJournal(run.journalSlug, run.journal);
|
|
1753
1463
|
|
|
1754
|
-
|
|
1755
|
-
// least one path was excluded. Sample is capped at 10 to keep the event
|
|
1756
|
-
// small (Set iteration order = insertion order, so samples are the first
|
|
1757
|
-
// ten paths encountered during the walk — deterministic, not random).
|
|
1758
|
-
if (excludedSet.size > 0) {
|
|
1464
|
+
if (run.excludedSet.size > 0) {
|
|
1759
1465
|
const samplePaths: string[] = [];
|
|
1760
|
-
for (const p of excludedSet) {
|
|
1466
|
+
for (const p of run.excludedSet) {
|
|
1761
1467
|
samplePaths.push(p);
|
|
1762
1468
|
if (samplePaths.length >= 10) break;
|
|
1763
1469
|
}
|
|
1764
|
-
emit({
|
|
1470
|
+
run.emit({
|
|
1765
1471
|
type: "personal-vault-out-of-policy",
|
|
1766
|
-
count: excludedSet.size,
|
|
1472
|
+
count: run.excludedSet.size,
|
|
1767
1473
|
samplePaths,
|
|
1768
|
-
byId: { ...excludedById },
|
|
1474
|
+
byId: { ...run.excludedById },
|
|
1769
1475
|
});
|
|
1770
1476
|
}
|
|
1771
1477
|
|
|
1772
|
-
|
|
1773
|
-
// paths fell outside the run's granted `prefixSet` and were skipped from the
|
|
1774
|
-
// upload/delete plan. Informational (NOT an error): the company still syncs
|
|
1775
|
-
// its in-scope subset and the run exits 0. Sample capped at 10 (insertion
|
|
1776
|
-
// order = walk order, deterministic).
|
|
1777
|
-
if (scopeExcludedSet.size > 0) {
|
|
1478
|
+
if (run.scopeExcludedSet.size > 0) {
|
|
1778
1479
|
const samplePaths: string[] = [];
|
|
1779
|
-
for (const p of scopeExcludedSet) {
|
|
1480
|
+
for (const p of run.scopeExcludedSet) {
|
|
1780
1481
|
samplePaths.push(p);
|
|
1781
1482
|
if (samplePaths.length >= 10) break;
|
|
1782
1483
|
}
|
|
1783
|
-
emit({
|
|
1484
|
+
run.emit({
|
|
1784
1485
|
type: "scope-excluded",
|
|
1785
|
-
count: scopeExcludedSet.size,
|
|
1486
|
+
count: run.scopeExcludedSet.size,
|
|
1786
1487
|
samplePaths,
|
|
1787
1488
|
});
|
|
1788
1489
|
}
|
|
1490
|
+
}
|
|
1789
1491
|
|
|
1790
|
-
|
|
1791
|
-
// headRemoteFile / refreshEntityContext / resolveConflict — paths
|
|
1792
|
-
// outside the per-item PUT try/catch), we deliberately let the pool
|
|
1793
|
-
// drain AND let post-pool stages (deletes, tombstones, journal write,
|
|
1794
|
-
// personal-vault summary) run to completion so journal + remote stay
|
|
1795
|
-
// converged on what actually landed. NOW surface the failure to the
|
|
1796
|
-
// caller — preserving the first error's stack so debugging works.
|
|
1797
|
-
// Aggregate count is reported in the message for visibility when
|
|
1798
|
-
// multiple workers failed.
|
|
1492
|
+
function throwUploadWorkerErrors(workerErrors: Error[]): void {
|
|
1799
1493
|
if (workerErrors.length > 0) {
|
|
1800
1494
|
const first = workerErrors[0]!;
|
|
1801
1495
|
if (workerErrors.length > 1) {
|
|
1802
|
-
first.message =
|
|
1496
|
+
first.message =
|
|
1497
|
+
first.message +
|
|
1498
|
+
" (and " +
|
|
1499
|
+
(workerErrors.length - 1) +
|
|
1500
|
+
" more upload-worker errors)";
|
|
1803
1501
|
}
|
|
1804
1502
|
throw first;
|
|
1805
1503
|
}
|
|
1504
|
+
}
|
|
1806
1505
|
|
|
1506
|
+
function buildShareResult(
|
|
1507
|
+
run: PushRunContext,
|
|
1508
|
+
counters: ShareCounters,
|
|
1509
|
+
filesRefusedStalePaths: string[],
|
|
1510
|
+
conflictPaths: string[],
|
|
1511
|
+
aborted: boolean,
|
|
1512
|
+
): ShareResult {
|
|
1807
1513
|
return {
|
|
1808
|
-
filesUploaded,
|
|
1809
|
-
bytesUploaded,
|
|
1810
|
-
filesSkipped,
|
|
1811
|
-
filesDeleted,
|
|
1812
|
-
filesTombstoned,
|
|
1813
|
-
filesRefusedStale,
|
|
1514
|
+
filesUploaded: counters.filesUploaded,
|
|
1515
|
+
bytesUploaded: counters.bytesUploaded,
|
|
1516
|
+
filesSkipped: counters.filesSkipped,
|
|
1517
|
+
filesDeleted: counters.filesDeleted,
|
|
1518
|
+
filesTombstoned: counters.filesTombstoned,
|
|
1519
|
+
filesRefusedStale: counters.filesRefusedStale,
|
|
1814
1520
|
filesRefusedStalePaths,
|
|
1815
|
-
filesSuppressedByTombstone,
|
|
1816
|
-
filesExcludedByPolicy: excludedSet.size,
|
|
1817
|
-
filesExcludedByScope: scopeExcludedSet.size,
|
|
1521
|
+
filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
|
|
1522
|
+
filesExcludedByPolicy: run.excludedSet.size,
|
|
1523
|
+
filesExcludedByScope: run.scopeExcludedSet.size,
|
|
1818
1524
|
conflictPaths,
|
|
1819
|
-
aborted
|
|
1525
|
+
aborted,
|
|
1820
1526
|
};
|
|
1821
1527
|
}
|
|
1822
1528
|
|
|
@@ -1879,22 +1585,6 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
|
|
|
1879
1585
|
}
|
|
1880
1586
|
}
|
|
1881
1587
|
|
|
1882
|
-
/**
|
|
1883
|
-
* Resolve active company from .hq/config.json or parent directory chain.
|
|
1884
|
-
*/
|
|
1885
|
-
function resolveActiveCompany(hqRoot: string): string | undefined {
|
|
1886
|
-
const configPath = path.join(hqRoot, ".hq", "config.json");
|
|
1887
|
-
if (fs.existsSync(configPath)) {
|
|
1888
|
-
try {
|
|
1889
|
-
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
1890
|
-
return config.activeCompany ?? config.companySlug;
|
|
1891
|
-
} catch {
|
|
1892
|
-
// Ignore parse errors
|
|
1893
|
-
}
|
|
1894
|
-
}
|
|
1895
|
-
return undefined;
|
|
1896
|
-
}
|
|
1897
|
-
|
|
1898
1588
|
/**
|
|
1899
1589
|
* One entry produced by collectFiles/walkDir. Files describe regular
|
|
1900
1590
|
* payloads that get hashed + size-checked + uploaded via uploadFile;
|
|
@@ -2082,6 +1772,11 @@ function isWithin(parent: string, child: string): boolean {
|
|
|
2082
1772
|
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
2083
1773
|
}
|
|
2084
1774
|
|
|
1775
|
+
function isPathWithin(parent: string, child: string): boolean {
|
|
1776
|
+
const rel = path.relative(parent, child);
|
|
1777
|
+
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
1778
|
+
}
|
|
1779
|
+
|
|
2085
1780
|
function realpathSafe(p: string): string {
|
|
2086
1781
|
try {
|
|
2087
1782
|
return fs.realpathSync.native(p);
|
|
@@ -2090,6 +1785,48 @@ function realpathSafe(p: string): string {
|
|
|
2090
1785
|
}
|
|
2091
1786
|
}
|
|
2092
1787
|
|
|
1788
|
+
function deepestExistingAncestor(start: string): string | null {
|
|
1789
|
+
let current = start;
|
|
1790
|
+
for (;;) {
|
|
1791
|
+
try {
|
|
1792
|
+
fs.lstatSync(current);
|
|
1793
|
+
return current;
|
|
1794
|
+
} catch (err: unknown) {
|
|
1795
|
+
const code =
|
|
1796
|
+
err && typeof err === "object" && "code" in err
|
|
1797
|
+
? (err as { code?: string }).code
|
|
1798
|
+
: undefined;
|
|
1799
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") return null;
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
const parent = path.dirname(current);
|
|
1803
|
+
if (parent === current) return null;
|
|
1804
|
+
current = parent;
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
function isMaterializationPathStillContained(root: string, localPath: string): boolean {
|
|
1809
|
+
const resolvedRoot = path.resolve(root);
|
|
1810
|
+
const resolvedLocal = path.resolve(localPath);
|
|
1811
|
+
if (!isPathWithin(resolvedRoot, resolvedLocal)) return false;
|
|
1812
|
+
|
|
1813
|
+
let realRoot: string;
|
|
1814
|
+
try {
|
|
1815
|
+
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
1816
|
+
} catch {
|
|
1817
|
+
return false;
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
1821
|
+
if (existingAncestor === null) return false;
|
|
1822
|
+
try {
|
|
1823
|
+
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
1824
|
+
return isPathWithin(realRoot, realAncestor);
|
|
1825
|
+
} catch {
|
|
1826
|
+
return false;
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
|
|
2093
1830
|
/**
|
|
2094
1831
|
* Containment check tailored for symlinks. Canonicalizes the link's
|
|
2095
1832
|
* PARENT DIR (which is a real dir, not the link), then compares the
|
|
@@ -2112,24 +1849,6 @@ function isWithinForLink(parent: string, linkPath: string): boolean {
|
|
|
2112
1849
|
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
|
2113
1850
|
}
|
|
2114
1851
|
|
|
2115
|
-
/**
|
|
2116
|
-
* Returns true when the remote object appears to have moved since the
|
|
2117
|
-
* journal entry's last-recorded sync. Prefers ETag equality; falls back to
|
|
2118
|
-
* `lastModified > syncedAt` for legacy entries written before remoteEtag
|
|
2119
|
-
* was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
|
|
2120
|
-
* S3-side mtime that exactly equals our syncedAt is not treated as drift.
|
|
2121
|
-
*/
|
|
2122
|
-
function hasRemoteChanged(
|
|
2123
|
-
remote: { lastModified: Date; etag: string },
|
|
2124
|
-
entry: { syncedAt: string; remoteEtag?: string },
|
|
2125
|
-
): boolean {
|
|
2126
|
-
if (entry.remoteEtag) {
|
|
2127
|
-
return normalizeEtag(remote.etag) !== entry.remoteEtag;
|
|
2128
|
-
}
|
|
2129
|
-
const syncedAt = new Date(entry.syncedAt).getTime();
|
|
2130
|
-
return remote.lastModified.getTime() > syncedAt;
|
|
2131
|
-
}
|
|
2132
|
-
|
|
2133
1852
|
/**
|
|
2134
1853
|
* Resolve each user-supplied share path to a directory under `syncRoot`,
|
|
2135
1854
|
* returning the company-relative prefix that constrains delete propagation.
|
|
@@ -2510,6 +2229,14 @@ async function computeDeletePlan(
|
|
|
2510
2229
|
// independently — one failed HEAD doesn't poison the others (errors
|
|
2511
2230
|
// propagate from the chunk's Promise.all and are surfaced by share()'s
|
|
2512
2231
|
// outer try/catch, mirroring the existing pre-share error handling).
|
|
2232
|
+
// Warm the GET presigns the HEAD pass reuses so a large candidate set doesn't
|
|
2233
|
+
// mint one presign per HEAD and trip the presign breaker. Mirrors the
|
|
2234
|
+
// new-files + tombstone HEAD-pass pre-primes.
|
|
2235
|
+
await primeObjectTransport(
|
|
2236
|
+
ctx,
|
|
2237
|
+
"get",
|
|
2238
|
+
headCandidates.map((c) => c.key),
|
|
2239
|
+
);
|
|
2513
2240
|
for (let i = 0; i < headCandidates.length; i += DELETE_PLAN_HEAD_CONCURRENCY) {
|
|
2514
2241
|
const chunk = headCandidates.slice(i, i + DELETE_PLAN_HEAD_CONCURRENCY);
|
|
2515
2242
|
const results = await Promise.all(
|