@indigoai-us/hq-cloud 6.18.21 → 6.18.22-alpha.702
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/sync.d.ts +6 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +724 -613
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +253 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/kms10-etag-safe-pull.d.ts +20 -0
- package/dist/kms10-etag-safe-pull.d.ts.map +1 -0
- package/dist/kms10-etag-safe-pull.js +37 -0
- package/dist/kms10-etag-safe-pull.js.map +1 -0
- package/dist/kms10-etag-safe-pull.test.d.ts +2 -0
- package/dist/kms10-etag-safe-pull.test.d.ts.map +1 -0
- package/dist/kms10-etag-safe-pull.test.js +53 -0
- package/dist/kms10-etag-safe-pull.test.js.map +1 -0
- package/package.json +1 -1
package/dist/cli/sync.js
CHANGED
|
@@ -30,6 +30,7 @@ import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeS
|
|
|
30
30
|
import { coalescePrefixes, compileScopeMatcher, intersectPrefixSets, pathToScopePrefix, toScopePrefixEntries, } from "../prefix-coalesce.js";
|
|
31
31
|
import { SESSION_LOGS_PUSH_ONLY_PREFIX } from "../sync/pull-scope.js";
|
|
32
32
|
import { contentHashMatchesRawSha256 } from "../sync/content-hash.js";
|
|
33
|
+
import { isKms10CopyOfJournalBase } from "../kms10-etag-safe-pull.js";
|
|
33
34
|
import { isTargetedUpsertExactListEnabled } from "../sync/targeted-upsert-exact-list.js";
|
|
34
35
|
import { coalesceDenseTargetedPullPaths } from "../sync/targeted-pull-prefixes.js";
|
|
35
36
|
import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
|
|
@@ -1109,6 +1110,10 @@ async function executeConflictExecutor(run, plan, scopeRun, counters) {
|
|
|
1109
1110
|
executeContentMatchSkip(run, item, counters);
|
|
1110
1111
|
continue;
|
|
1111
1112
|
}
|
|
1113
|
+
if (item.action === "skip-kms10-copy") {
|
|
1114
|
+
executeKms10CopySkip(run, item, counters);
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1112
1117
|
if (item.action === "skip-unreadable-link") {
|
|
1113
1118
|
// An unreadable local link must stay non-fatal: the runner treats a
|
|
1114
1119
|
// generic error event as a failed company leg. Surface the exact remote
|
|
@@ -1227,6 +1232,38 @@ function executeContentMatchSkip(run, item, counters) {
|
|
|
1227
1232
|
run.journalChangesSinceCheckpoint++;
|
|
1228
1233
|
counters.filesSkipped++;
|
|
1229
1234
|
}
|
|
1235
|
+
function executeKms10CopySkip(run, item, counters) {
|
|
1236
|
+
const key = item.remoteFile.key;
|
|
1237
|
+
if (!isDownloadWritePathStillContained(run.companyRoot, key, item.localPath)) {
|
|
1238
|
+
throw new Error(`KMS-10 copy skip escaped the sync root for ${key}`);
|
|
1239
|
+
}
|
|
1240
|
+
if (!matchesLocalSnapshot(item.localPath, item.localSnapshot)) {
|
|
1241
|
+
counters.filesSkipped++;
|
|
1242
|
+
counters.hasRetryableOutcome = true;
|
|
1243
|
+
run.emit({
|
|
1244
|
+
type: "error",
|
|
1245
|
+
diagnostic: true,
|
|
1246
|
+
path: key,
|
|
1247
|
+
message: "KMS-10 copy skip deferred because the local file changed after planning",
|
|
1248
|
+
});
|
|
1249
|
+
return;
|
|
1250
|
+
}
|
|
1251
|
+
const entry = journalRows(run.journal).get(key);
|
|
1252
|
+
const remoteEtag = normalizeEtag(item.remoteFile.etag);
|
|
1253
|
+
if (!entry ||
|
|
1254
|
+
entry.hash !== item.journalBaseHash ||
|
|
1255
|
+
entry.size !== item.remoteFile.size ||
|
|
1256
|
+
remoteEtag === "" ||
|
|
1257
|
+
remoteEtag === entry.remoteEtag) {
|
|
1258
|
+
counters.filesSkipped++;
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
// HEAD proved the remote hash matches the journal baseline. Preserve local
|
|
1262
|
+
// bytes and record only the copied remote revision.
|
|
1263
|
+
journalRows(run.journal).set(key, { ...entry, remoteEtag });
|
|
1264
|
+
run.journalChangesSinceCheckpoint++;
|
|
1265
|
+
counters.filesSkipped++;
|
|
1266
|
+
}
|
|
1230
1267
|
async function executeConflictItem(run, plan, scopeRun, counters, downloadItems, item) {
|
|
1231
1268
|
const { remoteFile, localPath } = item;
|
|
1232
1269
|
// Cloud-authoritative paths (server-regenerated: company-brief.md, board.json,
|
|
@@ -2705,6 +2742,7 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2705
2742
|
if (item.action === "download" ||
|
|
2706
2743
|
item.action === "conflict" ||
|
|
2707
2744
|
item.action === "skip-content-match" ||
|
|
2745
|
+
item.action === "skip-kms10-copy" ||
|
|
2708
2746
|
item.action === "tombstone-delete" ||
|
|
2709
2747
|
item.action === "skip-junk-key-spelling" ||
|
|
2710
2748
|
item.action === "skip-unreadable-link") {
|
|
@@ -2820,679 +2858,752 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
|
|
|
2820
2858
|
// promise/microtask work per object even though no I/O is being awaited.
|
|
2821
2859
|
const cooperativeWork = { rowsSinceYield: 0 };
|
|
2822
2860
|
let plannedRows = 0;
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
const
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
if (
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2861
|
+
const listedIterator = listedFiles[Symbol.iterator]();
|
|
2862
|
+
const pullHeadBatchSize = 16;
|
|
2863
|
+
while (true) {
|
|
2864
|
+
const remoteBatch = [];
|
|
2865
|
+
while (remoteBatch.length < pullHeadBatchSize) {
|
|
2866
|
+
const next = listedIterator.next();
|
|
2867
|
+
if (next.done)
|
|
2868
|
+
break;
|
|
2869
|
+
remoteBatch.push(next.value);
|
|
2870
|
+
}
|
|
2871
|
+
if (remoteBatch.length === 0)
|
|
2872
|
+
break;
|
|
2873
|
+
// KMS-10 copies change ETags without changing the journal content hash.
|
|
2874
|
+
// Probe bounded batches so large vaults never queue an unbounded HEAD set.
|
|
2875
|
+
const prefetchedHeads = new Map();
|
|
2876
|
+
const headCandidates = ctx === undefined
|
|
2877
|
+
? []
|
|
2878
|
+
: remoteBatch.filter((remoteFile) => {
|
|
2879
|
+
const entry = getEntry(journal, remoteFile.key);
|
|
2880
|
+
return entry !== undefined &&
|
|
2881
|
+
hasRemoteChanged(remoteFile, entry) &&
|
|
2882
|
+
remoteFile.size === entry.size &&
|
|
2883
|
+
normalizeEtag(remoteFile.etag) !== "" &&
|
|
2884
|
+
normalizeEtag(remoteFile.etag) !== entry.remoteEtag;
|
|
2885
|
+
});
|
|
2886
|
+
if (headCandidates.length > 0 && ctx !== undefined) {
|
|
2887
|
+
await Promise.all(headCandidates.map(async (remoteFile) => {
|
|
2888
|
+
try {
|
|
2889
|
+
prefetchedHeads.set(remoteFile.key, {
|
|
2890
|
+
head: await headRemoteFile(ctx, remoteFile.key),
|
|
2891
|
+
});
|
|
2892
|
+
}
|
|
2893
|
+
catch (error) {
|
|
2894
|
+
prefetchedHeads.set(remoteFile.key, { head: null, error });
|
|
2895
|
+
}
|
|
2896
|
+
}));
|
|
2897
|
+
}
|
|
2898
|
+
for (const remoteFile of remoteBatch) {
|
|
2899
|
+
onHeartbeat?.();
|
|
2900
|
+
noteCooperativeRowVisit("plan", remoteFile.key);
|
|
2901
|
+
plannedRows++;
|
|
2902
|
+
if (cooperativeCheckpoint &&
|
|
2903
|
+
plannedRows % cooperativeUnitSize() === 0) {
|
|
2904
|
+
dropJournalDecodeWindow();
|
|
2905
|
+
if (await yieldToCooperativeCheckpoint(cooperativeWork, cooperativeCheckpoint, true, plannedRows)) {
|
|
2906
|
+
onCooperativeHandoff?.();
|
|
2857
2907
|
}
|
|
2858
2908
|
}
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
remoteFile
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
}
|
|
2873
|
-
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
2874
|
-
if (localPath === null) {
|
|
2875
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
2876
|
-
continue;
|
|
2877
|
-
}
|
|
2878
|
-
// This is intentionally exact-prefix and storage-class-specific. Other
|
|
2879
|
-
// archived objects retain their established sync behavior until a separate
|
|
2880
|
-
// policy explicitly defines it. LIST retains the object in remoteKeys, so
|
|
2881
|
-
// an existing journal row is neither tombstoned nor deleted.
|
|
2882
|
-
if (remoteFile.key.startsWith("workspace/.session-logs/") &&
|
|
2883
|
-
remoteFile.storageClass === "DEEP_ARCHIVE") {
|
|
2884
|
-
items.push({ action: "skip-archived-session-log", remoteFile, localPath });
|
|
2885
|
-
continue;
|
|
2886
|
-
}
|
|
2887
|
-
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
2888
|
-
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
2889
|
-
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
2890
|
-
// staging, so legacy `.conflict-*` litter rode every sync into clean
|
|
2891
|
-
// trees. Refuse them here; the V2 verification test (direct S3 inject
|
|
2892
|
-
// bypassing the push filter entirely) is the regression contract.
|
|
2893
|
-
if (isEphemeralPath(remoteFile.key)) {
|
|
2894
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2895
|
-
continue;
|
|
2896
|
-
}
|
|
2897
|
-
// Personal-vault policy must be symmetric across both transfer legs.
|
|
2898
|
-
// The push walker refuses derived/machine-local/sensitive paths, but old
|
|
2899
|
-
// objects can remain in the bucket from clients that predate an exclusion.
|
|
2900
|
-
// Refuse those objects here too so a pull cannot rematerialize them (or,
|
|
2901
|
-
// for generated skill wrappers, fight reindex over a directory link).
|
|
2902
|
-
if (personalMode &&
|
|
2903
|
-
isPersonalVaultExcluded(remoteFile.key) &&
|
|
2904
|
-
!(allowRootMcpConfig && remoteFile.key === ".mcp.json")) {
|
|
2905
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2906
|
-
continue;
|
|
2907
|
-
}
|
|
2908
|
-
if (personalMode &&
|
|
2909
|
-
remoteFile.key.startsWith("companies/") &&
|
|
2910
|
-
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
2911
|
-
// carved INTO the personal vault on the push side
|
|
2912
|
-
// (computePersonalVaultPaths). It must round-trip on the pull leg too —
|
|
2913
|
-
// skipping it here leaves it forever unjournaled, which re-fires a
|
|
2914
|
-
// transient push-side conflict every sync (no journal baseline). Let it
|
|
2915
|
-
// fall through to download + journal like any personal file.
|
|
2916
|
-
remoteFile.key !== PERSONAL_VAULT_MANIFEST_KEY) {
|
|
2917
|
-
// Local (non-cloud) companies sync to the personal vault: keys under
|
|
2918
|
-
// `companies/{slug}/...` are legitimate content a peer machine pushed
|
|
2919
|
-
// for a company that is not cloud-backed. Allow them through EXCEPT
|
|
2920
|
-
// for slugs the operator has an active team-bucket Membership for:
|
|
2921
|
-
// those are orphan remnants from before the company became cloud-true,
|
|
2922
|
-
// and downloading them would clash with the team-bucket pull at the
|
|
2923
|
-
// same disk path.
|
|
2924
|
-
//
|
|
2925
|
-
// Symmetric to the push-side `decommissionPrefixes` logic in
|
|
2926
|
-
// share.ts — both target the same orphan class, both honor the
|
|
2927
|
-
// same `teamSyncedSlugs` set derived from the operator's live
|
|
2928
|
-
// membership at runtime.
|
|
2929
|
-
const slug = remoteFile.key.split("/")[1] ?? "";
|
|
2930
|
-
const isTeamSyncedOrphan = teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
|
|
2931
|
-
if (isTeamSyncedOrphan) {
|
|
2932
|
-
items.push({ action: "skip-personal-mode", remoteFile, localPath });
|
|
2933
|
-
continue;
|
|
2909
|
+
else if (plannedRows % decodeWindowSize() === 0) {
|
|
2910
|
+
dropJournalDecodeWindow();
|
|
2911
|
+
}
|
|
2912
|
+
const posixRemoteKey = toPosixKey(remoteFile.key);
|
|
2913
|
+
// S3 object keys are POSIX. Match the old delete-sweep membership check:
|
|
2914
|
+
// a legacy backslash journal entry counts as present only when its POSIX
|
|
2915
|
+
// form appears in the LIST, while a malformed backslash remote key does
|
|
2916
|
+
// not accidentally change that behavior.
|
|
2917
|
+
if (remoteFile.key === posixRemoteKey) {
|
|
2918
|
+
unseenJournalKeys.delete(remoteFile.key);
|
|
2919
|
+
for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
|
|
2920
|
+
unseenJournalKeys.delete(legacyKey);
|
|
2921
|
+
}
|
|
2934
2922
|
}
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
? "push-only"
|
|
2954
|
-
: "membership-scope",
|
|
2955
|
-
});
|
|
2956
|
-
continue;
|
|
2957
|
-
}
|
|
2958
|
-
// Push-only exclusion filter (US-006). A key that PASSED the inclusion check
|
|
2959
|
-
// above is STILL not downloaded if it falls under a push-only exclude prefix
|
|
2960
|
-
// (`sessions/`). This is what makes sessions push-only in EVERY mode — it
|
|
2961
|
-
// subtracts from `all` (whose `prefixSet` is `[""]`, covering everything)
|
|
2962
|
-
// and from an owner's role-wide scope alike. Empty `excludePrefixes` (the
|
|
2963
|
-
// default) makes this a no-op. Deliberately AFTER inclusion and reported on
|
|
2964
|
-
// the same `filesOutOfScope` axis. The matching local file (if a session was
|
|
2965
|
-
// previously pulled, authored locally, or pinned+materialized) is NOT pruned
|
|
2966
|
-
// here or by scope-shrink — see `planScopeShrink`, which never sees this set.
|
|
2967
|
-
if (excludeMatcher.coversPath(remoteFile.key)) {
|
|
2968
|
-
items.push({
|
|
2969
|
-
action: "skip-out-of-scope",
|
|
2970
|
-
remoteFile,
|
|
2971
|
-
localPath,
|
|
2972
|
-
reason: "push-only",
|
|
2973
|
-
});
|
|
2974
|
-
continue;
|
|
2975
|
-
}
|
|
2976
|
-
// LIST gives us no kind signal for the remote object — we don't
|
|
2977
|
-
// know whether this key is a regular file or a symlink record
|
|
2978
|
-
// until we either HEAD it (expensive — N extra calls per pull) or
|
|
2979
|
-
// download it. So probe the ignore filter with BOTH isDir hints
|
|
2980
|
-
// and include the entry if either probe passes. Without this,
|
|
2981
|
-
// a symlink record at a dir-only-allowlisted path (e.g. an
|
|
2982
|
-
// .hqinclude of `companies/*/knowledge/`) would be classified
|
|
2983
|
-
// skip-ignored and the symlinks freshly uploaded by the
|
|
2984
|
-
// push-side fix would never propagate. Mirrors the dual-hint
|
|
2985
|
-
// probe in walkDir/collectFiles. Pure path lookup, no I/O.
|
|
2986
|
-
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) {
|
|
2987
|
-
items.push({ action: "skip-ignored", remoteFile, localPath });
|
|
2988
|
-
continue;
|
|
2989
|
-
}
|
|
2990
|
-
const journalEntry = getEntry(journal, remoteFile.key);
|
|
2991
|
-
// Reindex and package wiring own symlinks under these `core/<type>`
|
|
2992
|
-
// surfaces; their canonical sources are elsewhere. A prior pull records
|
|
2993
|
-
// the remote object as a symlink, then reindex removes the generated link.
|
|
2994
|
-
// Without this classification the next pull sees only an absent local path
|
|
2995
|
-
// and downloads the same marker forever. Retain regular core files: the
|
|
2996
|
-
// journal kind is the discriminator, because LIST exposes no object type.
|
|
2997
|
-
// An unchanged ETag is also required: a remote writer can replace a legacy
|
|
2998
|
-
// symlink with canonical file content at the same key, and that replacement
|
|
2999
|
-
// must still be pulled.
|
|
3000
|
-
if (personalMode &&
|
|
3001
|
-
journalEntry?.kind === "symlink" &&
|
|
3002
|
-
isGeneratedCoreMirrorKey(remoteFile.key) &&
|
|
3003
|
-
journalEntry.remoteEtag !== undefined &&
|
|
3004
|
-
!hasRemoteChanged(remoteFile, journalEntry)) {
|
|
3005
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
3006
|
-
continue;
|
|
3007
|
-
}
|
|
3008
|
-
// Junk-spelling guard — SKILL-FARM KEYS ONLY. The amplifier is specific
|
|
3009
|
-
// to `.claude/skills/<ns>:<skill>` wrapper spellings; outside that prefix,
|
|
3010
|
-
// keys whose percent-decode fixpoints collide (`notes/a b.md` vs a genuine
|
|
3011
|
-
// `notes/a%20b.md`) are DISTINCT files that must both pull.
|
|
3012
|
-
// Within the farm: a remote key that is NOT journaled under its own
|
|
3013
|
-
// spelling but whose percent-decode fixpoint matches an already-journaled
|
|
3014
|
-
// key with a different spelling is a mixed-version twin (`%3A`/`%253A`
|
|
3015
|
-
// generations of a canonical `:` key). Materializing it would mint junk
|
|
3016
|
-
// local dirs and fork the journal into a second key family for the same
|
|
3017
|
-
// logical path — skip it (loudly, never fatally; see the
|
|
3018
|
-
// `skip-junk-key-spelling` event doc) and leave the journaled spelling
|
|
3019
|
-
// authoritative for this machine.
|
|
3020
|
-
if (journalEntry === undefined &&
|
|
3021
|
-
remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
|
|
3022
|
-
// Gate NON-canonical remote spellings only. A remote key that already
|
|
3023
|
-
// IS the canonical spelling must always be downloadable — after the
|
|
3024
|
-
// doctor collapses the vault, a machine whose journal still holds only
|
|
3025
|
-
// the junk spelling would otherwise skip the canonical object forever
|
|
3026
|
-
// (and the delete-resync twin protection keeps that stale junk entry),
|
|
3027
|
-
// deadlocking it out of every future update.
|
|
3028
|
-
canonicalVaultKeySpelling(remoteFile.key) !== remoteFile.key) {
|
|
3029
|
-
const journaledTwin = journaledKeyByCanonical.get(canonicalVaultKeySpelling(remoteFile.key));
|
|
3030
|
-
if (journaledTwin !== undefined && journaledTwin !== remoteFile.key) {
|
|
2923
|
+
if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
2924
|
+
// A remote junk spelling means its canonical journal twin is still
|
|
2925
|
+
// remotely present. Keep that twin out of the delete sweep unless an
|
|
2926
|
+
// exact FILE_TOMBSTONE is authoritative, precisely matching the old
|
|
2927
|
+
// remoteCanonicalSkillKeys protection without retaining remote keys.
|
|
2928
|
+
const canonical = canonicalVaultKeySpelling(remoteFile.key);
|
|
2929
|
+
for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
|
|
2930
|
+
if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
|
|
2931
|
+
unseenJournalKeys.delete(journaledKey);
|
|
2932
|
+
}
|
|
2933
|
+
}
|
|
2934
|
+
}
|
|
2935
|
+
const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
|
|
2936
|
+
if (invalidKey !== null) {
|
|
2937
|
+
invalidRemoteKeys.push({
|
|
2938
|
+
path: remoteFile.key,
|
|
2939
|
+
errorCode: invalidKey.code,
|
|
2940
|
+
});
|
|
3031
2941
|
items.push({
|
|
3032
|
-
action: "skip-
|
|
2942
|
+
action: "skip-excluded-policy",
|
|
3033
2943
|
remoteFile,
|
|
3034
|
-
localPath,
|
|
3035
|
-
journaledKey: journaledTwin,
|
|
2944
|
+
localPath: companyRoot,
|
|
3036
2945
|
});
|
|
3037
2946
|
continue;
|
|
3038
2947
|
}
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
// (a peer re-pushed it, or its delete-marker hasn't propagated to this
|
|
3043
|
-
// caller's view). `tombstoneSuppresses` is true when a FILE_TOMBSTONE marks
|
|
3044
|
-
// this key deleted AND the remote object is NOT a newer re-create — i.e. the
|
|
3045
|
-
// object is the stale/deleted version and must NOT be downloaded. A genuine
|
|
3046
|
-
// re-create (object newer than the tombstone) leaves this false so the
|
|
3047
|
-
// normal download/merge path runs (the tombstone is not permanent
|
|
3048
|
-
// suppression). An empty tombstone map (the default / degraded-fetch case)
|
|
3049
|
-
// makes this always false — legacy behavior preserved bit-for-bit.
|
|
3050
|
-
const tombstone = fileTombstones.get(toPosixKey(remoteFile.key));
|
|
3051
|
-
const tombstoneSuppresses = tombstone !== undefined &&
|
|
3052
|
-
!isRemoteRecreateAfterTombstone(remoteFile, tombstone);
|
|
3053
|
-
// LOCAL journal tombstone — this machine's own intentional-delete record,
|
|
3054
|
-
// independent of the (best-effort, intermittently-degraded) server
|
|
3055
|
-
// FILE_TOMBSTONE fetch above. When a clean local delete is classified, the
|
|
3056
|
-
// push leg stamps `removedAt` on the journal entry; honoring it here makes
|
|
3057
|
-
// the delete-vs-drift-restore outcome DETERMINISTIC, closing the
|
|
3058
|
-
// intermittent respawn where a degraded tombstone fetch let the pull leg
|
|
3059
|
-
// re-materialize an intentionally-deleted file. Reuses the same re-create
|
|
3060
|
-
// guard (a remote object newer than the local delete is a genuine re-create
|
|
3061
|
-
// and still downloads). Consulted ONLY on the `!localExists` re-download
|
|
3062
|
-
// path below — never to delete a file that is present locally.
|
|
3063
|
-
const localTombstoneSuppresses = journalEntry !== undefined &&
|
|
3064
|
-
isTombstone(journalEntry) &&
|
|
3065
|
-
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
3066
|
-
journalEntry.removedAt !== undefined &&
|
|
3067
|
-
!isRemoteRecreateAfterTombstone(remoteFile, {
|
|
3068
|
-
deletedAt: journalEntry.removedAt,
|
|
3069
|
-
});
|
|
3070
|
-
// Only a key that passed validation, policy, scope and ignore filtering
|
|
3071
|
-
// and is not itself a deleted object can own a local file, so only such a
|
|
3072
|
-
// key may protect a case-only twin from deletion (feedback_c8ed69d1). A
|
|
3073
|
-
// local file whose folder merely keeps an older case spelling is not
|
|
3074
|
-
// protected: when no live key owns it, the remote delete still applies.
|
|
3075
|
-
if (!tombstoneSuppresses && !localTombstoneSuppresses) {
|
|
3076
|
-
caseAliasTracker?.observeRemote(remoteFile.key);
|
|
3077
|
-
}
|
|
3078
|
-
// lstat (not existsSync/statSync) handles three cases the legacy
|
|
3079
|
-
// checks got wrong for symlinks:
|
|
3080
|
-
// 1. A valid symlink at localPath: existsSync returns true and the
|
|
3081
|
-
// planner falls through to hashFile, which follows the link and
|
|
3082
|
-
// hashes the target file's bytes — never matching the journal's
|
|
3083
|
-
// sha256(target string), so every pull would mis-classify a
|
|
3084
|
-
// pristine symlink as locally changed.
|
|
3085
|
-
// 2. A dangling symlink: existsSync returns false (existsSync
|
|
3086
|
-
// follows links) so the planner treats the entry as missing and
|
|
3087
|
-
// re-downloads, silently clobbering the user's intentional
|
|
3088
|
-
// dangling link on every run.
|
|
3089
|
-
// 3. A regular file: indistinguishable from current behaviour.
|
|
3090
|
-
let localLstat = null;
|
|
3091
|
-
let plannedLocalSnapshot = { kind: "absent" };
|
|
3092
|
-
let localPathBlockedByFileAncestor = false;
|
|
3093
|
-
try {
|
|
3094
|
-
localLstat = fs.lstatSync(localPath);
|
|
3095
|
-
}
|
|
3096
|
-
catch (err) {
|
|
3097
|
-
const code = err && typeof err === "object" && "code" in err
|
|
3098
|
-
? err.code
|
|
3099
|
-
: undefined;
|
|
3100
|
-
// ENOENT → truly absent → treated as "new file from cloud".
|
|
3101
|
-
// ENOTDIR → an ancestor of localPath is a regular file (cloud has a
|
|
3102
|
-
// dir at that key, local has a file). Pre-fix this threw
|
|
3103
|
-
// and aborted the whole company sync (Bug #10 in the
|
|
3104
|
-
// 5.33.0 deep-test verification — \`v4-dir-vs-file\` repro
|
|
3105
|
-
// wedged the personal company at \"errored\" status,
|
|
3106
|
-
// skipping every later file). Recover by classifying as
|
|
3107
|
-
// a structural collision the operator must resolve
|
|
3108
|
-
// manually, and let the rest of the company process.
|
|
3109
|
-
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
3110
|
-
throw err;
|
|
3111
|
-
}
|
|
3112
|
-
if (code === "ENOTDIR") {
|
|
3113
|
-
localPathBlockedByFileAncestor = true;
|
|
3114
|
-
}
|
|
3115
|
-
}
|
|
3116
|
-
const localExists = localLstat !== null;
|
|
3117
|
-
if (localPathBlockedByFileAncestor) {
|
|
3118
|
-
// Symmetric counterpart to the (local-dir, cloud-file) warning below.
|
|
3119
|
-
// Emit the same "manual reconciliation required" surface so the
|
|
3120
|
-
// operator sees one consistent message for both topologies; record as
|
|
3121
|
-
// skip-local-only so the file is not silently dropped from the
|
|
3122
|
-
// count and the executor never tries to lstat/write it.
|
|
3123
|
-
console.error(` Warning: an ancestor of ${remoteFile.key} exists locally as a regular file; ` +
|
|
3124
|
-
`cloud has a deeper key under that path. Skipping; manual ` +
|
|
3125
|
-
`reconciliation required (rm the conflicting local file to pull).`);
|
|
3126
|
-
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
3127
|
-
continue;
|
|
3128
|
-
}
|
|
3129
|
-
if (localExists) {
|
|
3130
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
3131
|
-
// Kind-mismatch guard: a remote LIST entry is always a single
|
|
3132
|
-
// S3 object (regular file or symlink record), but the local
|
|
3133
|
-
// path may be a real directory if the user previously had a
|
|
3134
|
-
// dir at this key. Hashing a directory throws EISDIR and
|
|
3135
|
-
// aborts the whole pull. We don't auto-replace because that
|
|
3136
|
-
// would `rm -rf` the user's directory; surface as a non-fatal
|
|
3137
|
-
// skip and warn so the operator can reconcile manually
|
|
3138
|
-
// (delete the local dir, or restructure the remote).
|
|
3139
|
-
if (localLstat.isDirectory() && !isLocalSymlink) {
|
|
3140
|
-
// Stale personal-overlay marker vs. release-shipped core directory:
|
|
3141
|
-
// a single vault object at a `core/<overlay-type>/.../<entry>` key
|
|
3142
|
-
// whose local counterpart is a real directory is an orphaned overlay
|
|
3143
|
-
// symlink marker (reindex mirror artifact) left behind after a release
|
|
3144
|
-
// shipped a real core dir of the same name. The local release-shipped
|
|
3145
|
-
// directory is authoritative and the marker can never be materialized
|
|
3146
|
-
// over it, so keep local and ignore the marker QUIETLY — the generic
|
|
3147
|
-
// "rm -rf the local directory" advice below is actively dangerous here
|
|
3148
|
-
// (it would destroy release-shipped core content to pull inert derived
|
|
3149
|
-
// state), and it recurred on every sync. feedback_ce9aeede.
|
|
3150
|
-
if (isGeneratedCoreMirrorKey(remoteFile.key)) {
|
|
3151
|
-
console.error(` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
|
|
3152
|
-
`shadowing a release-shipped core directory; keeping the local ` +
|
|
3153
|
-
`directory and ignoring the marker (no action needed).`);
|
|
3154
|
-
items.push({
|
|
3155
|
-
action: "skip-stale-overlay-marker",
|
|
3156
|
-
remoteFile,
|
|
3157
|
-
localPath,
|
|
3158
|
-
});
|
|
3159
|
-
continue;
|
|
3160
|
-
}
|
|
3161
|
-
// Directory-overlay marker vs. a locally-materialized directory:
|
|
3162
|
-
// the remote object at this key carries CHILD objects under it in the
|
|
3163
|
-
// same LIST (`remoteKeysWithChildren`), so it is a symlink/overlay
|
|
3164
|
-
// record for a directory — e.g. the DPP vault's `skills/cody-copywriter`
|
|
3165
|
-
// overlay whose 13 child objects are already materialized locally. The
|
|
3166
|
-
// marker can never be renamed over the real directory, and its children
|
|
3167
|
-
// carry the actual content and materialize via their own plan items, so
|
|
3168
|
-
// the marker is a benign, no-action skip. The generic "rm -rf the local
|
|
3169
|
-
// directory to pull" advice below would tell the operator to DELETE the
|
|
3170
|
-
// materialized (and possibly locally-authored) contents to install an
|
|
3171
|
-
// inert link, and it recurred on EVERY sync cycle. feedback_d2082110.
|
|
3172
|
-
if (remoteKeysWithChildren.has(toPosixKey(remoteFile.key))) {
|
|
3173
|
-
console.error(` Note: ${remoteFile.key} is a directory-overlay marker whose ` +
|
|
3174
|
-
`contents already exist locally as a directory; keeping the ` +
|
|
3175
|
-
`local directory and ignoring the marker (no action needed).`);
|
|
3176
|
-
items.push({
|
|
3177
|
-
action: "skip-overlay-marker-with-children",
|
|
3178
|
-
remoteFile,
|
|
3179
|
-
localPath,
|
|
3180
|
-
});
|
|
3181
|
-
continue;
|
|
3182
|
-
}
|
|
3183
|
-
console.error(` Warning: ${remoteFile.key} exists locally as a directory; ` +
|
|
3184
|
-
`cloud has a single object at this key. Skipping; manual ` +
|
|
3185
|
-
`reconciliation required (rm -rf the local directory to pull).`);
|
|
3186
|
-
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
2948
|
+
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
2949
|
+
if (localPath === null) {
|
|
2950
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
3187
2951
|
continue;
|
|
3188
2952
|
}
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
continue;
|
|
3198
|
-
}
|
|
3199
|
-
localHash = hashSymlinkTarget(target);
|
|
3200
|
-
}
|
|
3201
|
-
else if (journalEntry && journalEntryMatchesStat(journalEntry, localLstat)) {
|
|
3202
|
-
// size + mtime + ctime all match the row that recorded this hash, so
|
|
3203
|
-
// the bytes have not been written since (ctime cannot be set from
|
|
3204
|
-
// userspace — see `journalEntryMatchesStat`). Reading them would only
|
|
3205
|
-
// reproduce the recorded hash. This is the pull half of the push
|
|
3206
|
-
// planner's stat fast path; without it every local file with a remote
|
|
3207
|
-
// counterpart was hashed on every pass — about 13 GB per runner start
|
|
3208
|
-
// on a real install, a quarter of the first pass's CPU.
|
|
3209
|
-
localHash = journalEntry.hash;
|
|
2953
|
+
// This is intentionally exact-prefix and storage-class-specific. Other
|
|
2954
|
+
// archived objects retain their established sync behavior until a separate
|
|
2955
|
+
// policy explicitly defines it. LIST retains the object in remoteKeys, so
|
|
2956
|
+
// an existing journal row is neither tombstoned nor deleted.
|
|
2957
|
+
if (remoteFile.key.startsWith("workspace/.session-logs/") &&
|
|
2958
|
+
remoteFile.storageClass === "DEEP_ARCHIVE") {
|
|
2959
|
+
items.push({ action: "skip-archived-session-log", remoteFile, localPath });
|
|
2960
|
+
continue;
|
|
3210
2961
|
}
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
plannedLocalSnapshot = {
|
|
3220
|
-
kind: isLocalSymlink ? "symlink" : "file",
|
|
3221
|
-
hash: localHash,
|
|
3222
|
-
};
|
|
3223
|
-
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
|
3224
|
-
const announcedContentHash = pullContentHashes[remoteFile.key];
|
|
3225
|
-
// ── Authoritative delete reaches a machine that still holds the file ──
|
|
3226
|
-
// A FILE_TOMBSTONE marks this key deleted and the remote object is not a
|
|
3227
|
-
// newer re-create, but this machine still has a local copy. Honor the
|
|
3228
|
-
// delete — EXCEPT never destroy unsynced local work: if the local copy
|
|
3229
|
-
// diverges from the synced baseline (a post-tombstone edit, or an
|
|
3230
|
-
// untracked local file with no journal entry), surface a CONFLICT instead
|
|
3231
|
-
// of silently deleting (the safety guard the brief requires + the 3-way
|
|
3232
|
-
// merge invariant). A local copy that still matches the deleted baseline
|
|
3233
|
-
// is the stale version → delete it + drop the journal entry so it stays
|
|
3234
|
-
// gone on this machine.
|
|
3235
|
-
if (tombstoneSuppresses) {
|
|
3236
|
-
if (localChanged || !journalEntry) {
|
|
3237
|
-
items.push({
|
|
3238
|
-
action: "conflict",
|
|
3239
|
-
remoteFile,
|
|
3240
|
-
localPath,
|
|
3241
|
-
localHash,
|
|
3242
|
-
localMtime: localLstat.mtime,
|
|
3243
|
-
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3244
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3245
|
-
});
|
|
3246
|
-
}
|
|
3247
|
-
else {
|
|
3248
|
-
items.push({
|
|
3249
|
-
action: "tombstone-delete",
|
|
3250
|
-
remoteFile,
|
|
3251
|
-
localPath,
|
|
3252
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3253
|
-
});
|
|
3254
|
-
}
|
|
2962
|
+
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
2963
|
+
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
2964
|
+
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
2965
|
+
// staging, so legacy `.conflict-*` litter rode every sync into clean
|
|
2966
|
+
// trees. Refuse them here; the V2 verification test (direct S3 inject
|
|
2967
|
+
// bypassing the push filter entirely) is the regression contract.
|
|
2968
|
+
if (isEphemeralPath(remoteFile.key)) {
|
|
2969
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
3255
2970
|
continue;
|
|
3256
2971
|
}
|
|
3257
|
-
//
|
|
3258
|
-
//
|
|
3259
|
-
//
|
|
3260
|
-
//
|
|
3261
|
-
//
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
// `companies/*/projects/*/prd.json` three times on 2026-08-01/02, but
|
|
3267
|
-
// nothing about it is prd- or project-specific: it fires for ANY synced
|
|
3268
|
-
// path whose journal entry is absent. Absence is common — a file created
|
|
3269
|
-
// on this machine and on a peer under the same key before either pushed,
|
|
3270
|
-
// a journal shard reset/lost/rebuilt (`readJournal` returns an empty
|
|
3271
|
-
// `files` map for a missing shard, so EVERY divergent local file is in
|
|
3272
|
-
// scope), a run under a different `HQ_STATE_DIR`, or an entry dropped by
|
|
3273
|
-
// scope-shrink or rescue.
|
|
3274
|
-
//
|
|
3275
|
-
// No journal entry means no evidence that local and remote ever agreed,
|
|
3276
|
-
// which is exactly when an overwrite is least safe. Route it through the
|
|
3277
|
-
// conflict path instead: the executor's convergence probe fetches the
|
|
3278
|
-
// remote once and, when the bytes already match (the ordinary
|
|
3279
|
-
// fresh-machine / rebuilt-journal case), reconciles silently — no
|
|
3280
|
-
// conflict, no mirror, journal stamped. Only genuine divergence surfaces,
|
|
3281
|
-
// and there local is PRESERVED as-is with the cloud copy written beside
|
|
3282
|
-
// it under `--on-conflict keep`, or the run halts under `abort`.
|
|
3283
|
-
// Cloud-authoritative paths (board.json, ontology/, signals/, …) are
|
|
3284
|
-
// unaffected — the conflict executor short-circuits those back to a
|
|
3285
|
-
// pull-wins download before any mirror is minted.
|
|
3286
|
-
if (!journalEntry) {
|
|
3287
|
-
items.push({
|
|
3288
|
-
action: "conflict",
|
|
3289
|
-
remoteFile,
|
|
3290
|
-
localPath,
|
|
3291
|
-
localHash,
|
|
3292
|
-
localMtime: localLstat.mtime,
|
|
3293
|
-
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3294
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3295
|
-
});
|
|
2972
|
+
// Personal-vault policy must be symmetric across both transfer legs.
|
|
2973
|
+
// The push walker refuses derived/machine-local/sensitive paths, but old
|
|
2974
|
+
// objects can remain in the bucket from clients that predate an exclusion.
|
|
2975
|
+
// Refuse those objects here too so a pull cannot rematerialize them (or,
|
|
2976
|
+
// for generated skill wrappers, fight reindex over a directory link).
|
|
2977
|
+
if (personalMode &&
|
|
2978
|
+
isPersonalVaultExcluded(remoteFile.key) &&
|
|
2979
|
+
!(allowRootMcpConfig && remoteFile.key === ".mcp.json")) {
|
|
2980
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
3296
2981
|
continue;
|
|
3297
2982
|
}
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
|
|
3306
|
-
|
|
3307
|
-
|
|
3308
|
-
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
// No proof means the ordinary GET path remains authoritative.
|
|
3325
|
-
console.error(`sync: unable to verify announced content hash for ${remoteFile.key}; falling back to GET`, headError);
|
|
2983
|
+
if (personalMode &&
|
|
2984
|
+
remoteFile.key.startsWith("companies/") &&
|
|
2985
|
+
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
2986
|
+
// carved INTO the personal vault on the push side
|
|
2987
|
+
// (computePersonalVaultPaths). It must round-trip on the pull leg too —
|
|
2988
|
+
// skipping it here leaves it forever unjournaled, which re-fires a
|
|
2989
|
+
// transient push-side conflict every sync (no journal baseline). Let it
|
|
2990
|
+
// fall through to download + journal like any personal file.
|
|
2991
|
+
remoteFile.key !== PERSONAL_VAULT_MANIFEST_KEY) {
|
|
2992
|
+
// Local (non-cloud) companies sync to the personal vault: keys under
|
|
2993
|
+
// `companies/{slug}/...` are legitimate content a peer machine pushed
|
|
2994
|
+
// for a company that is not cloud-backed. Allow them through EXCEPT
|
|
2995
|
+
// for slugs the operator has an active team-bucket Membership for:
|
|
2996
|
+
// those are orphan remnants from before the company became cloud-true,
|
|
2997
|
+
// and downloading them would clash with the team-bucket pull at the
|
|
2998
|
+
// same disk path.
|
|
2999
|
+
//
|
|
3000
|
+
// Symmetric to the push-side `decommissionPrefixes` logic in
|
|
3001
|
+
// share.ts — both target the same orphan class, both honor the
|
|
3002
|
+
// same `teamSyncedSlugs` set derived from the operator's live
|
|
3003
|
+
// membership at runtime.
|
|
3004
|
+
const slug = remoteFile.key.split("/")[1] ?? "";
|
|
3005
|
+
const isTeamSyncedOrphan = teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
|
|
3006
|
+
if (isTeamSyncedOrphan) {
|
|
3007
|
+
items.push({ action: "skip-personal-mode", remoteFile, localPath });
|
|
3008
|
+
continue;
|
|
3326
3009
|
}
|
|
3327
3010
|
}
|
|
3328
|
-
|
|
3011
|
+
// Scope filter (US-005). Keys outside the effective `syncMode` prefix set
|
|
3012
|
+
// are not downloaded. `prefixSet` is `[""]` in `all` mode, which
|
|
3013
|
+
// `isCoveredByAny` treats as covering everything — so this is a no-op for
|
|
3014
|
+
// `all` and preserves the legacy full-bucket pull bit-for-bit. The
|
|
3015
|
+
// previously-downloaded counterparts of these keys (if scope just shrank)
|
|
3016
|
+
// are pruned separately by the scope-shrink pass in `sync()`.
|
|
3017
|
+
// A key can fail the inclusion check AND sit under a push-only exclude
|
|
3018
|
+
// prefix. Attribute it to `push-only` in that case: widening the
|
|
3019
|
+
// membership to `all` would move it into `prefixSet` but the exclusion
|
|
3020
|
+
// below would still withhold it, so it is NOT part of the shared-vs-all
|
|
3021
|
+
// gap and must not be advertised as fixable by raising the access level.
|
|
3022
|
+
if (!prefixMatcher.coversPath(remoteFile.key)) {
|
|
3329
3023
|
items.push({
|
|
3330
|
-
action: "skip-
|
|
3024
|
+
action: "skip-out-of-scope",
|
|
3331
3025
|
remoteFile,
|
|
3332
3026
|
localPath,
|
|
3333
|
-
|
|
3027
|
+
reason: excludeMatcher.coversPath(remoteFile.key)
|
|
3028
|
+
? "push-only"
|
|
3029
|
+
: "membership-scope",
|
|
3334
3030
|
});
|
|
3335
3031
|
continue;
|
|
3336
3032
|
}
|
|
3337
|
-
//
|
|
3338
|
-
//
|
|
3339
|
-
//
|
|
3340
|
-
|
|
3033
|
+
// Push-only exclusion filter (US-006). A key that PASSED the inclusion check
|
|
3034
|
+
// above is STILL not downloaded if it falls under a push-only exclude prefix
|
|
3035
|
+
// (`sessions/`). This is what makes sessions push-only in EVERY mode — it
|
|
3036
|
+
// subtracts from `all` (whose `prefixSet` is `[""]`, covering everything)
|
|
3037
|
+
// and from an owner's role-wide scope alike. Empty `excludePrefixes` (the
|
|
3038
|
+
// default) makes this a no-op. Deliberately AFTER inclusion and reported on
|
|
3039
|
+
// the same `filesOutOfScope` axis. The matching local file (if a session was
|
|
3040
|
+
// previously pulled, authored locally, or pinned+materialized) is NOT pruned
|
|
3041
|
+
// here or by scope-shrink — see `planScopeShrink`, which never sees this set.
|
|
3042
|
+
if (excludeMatcher.coversPath(remoteFile.key)) {
|
|
3341
3043
|
items.push({
|
|
3342
|
-
action: "
|
|
3044
|
+
action: "skip-out-of-scope",
|
|
3343
3045
|
remoteFile,
|
|
3344
3046
|
localPath,
|
|
3345
|
-
|
|
3346
|
-
// localLstat is non-null inside this `if (localExists)`
|
|
3347
|
-
// branch — the planner already lstat'd the path. Carry
|
|
3348
|
-
// its mtime forward so the executor doesn't have to
|
|
3349
|
-
// re-stat (which would follow links and crash on a
|
|
3350
|
-
// dangling symlink). Same goes for size: the conflict
|
|
3351
|
-
// executor's keep/skip branch needs to stamp the
|
|
3352
|
-
// journal with a size, and statSync on a dangling
|
|
3353
|
-
// symlink throws ENOENT (silently swallowed pre-fix),
|
|
3354
|
-
// leaving the journal stale and re-firing the conflict
|
|
3355
|
-
// every sync.
|
|
3356
|
-
localMtime: localLstat.mtime,
|
|
3357
|
-
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3358
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3047
|
+
reason: "push-only",
|
|
3359
3048
|
});
|
|
3360
3049
|
continue;
|
|
3361
3050
|
}
|
|
3362
|
-
|
|
3363
|
-
|
|
3051
|
+
// LIST gives us no kind signal for the remote object — we don't
|
|
3052
|
+
// know whether this key is a regular file or a symlink record
|
|
3053
|
+
// until we either HEAD it (expensive — N extra calls per pull) or
|
|
3054
|
+
// download it. So probe the ignore filter with BOTH isDir hints
|
|
3055
|
+
// and include the entry if either probe passes. Without this,
|
|
3056
|
+
// a symlink record at a dir-only-allowlisted path (e.g. an
|
|
3057
|
+
// .hqinclude of `companies/*/knowledge/`) would be classified
|
|
3058
|
+
// skip-ignored and the symlinks freshly uploaded by the
|
|
3059
|
+
// push-side fix would never propagate. Mirrors the dual-hint
|
|
3060
|
+
// probe in walkDir/collectFiles. Pure path lookup, no I/O.
|
|
3061
|
+
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) {
|
|
3062
|
+
items.push({ action: "skip-ignored", remoteFile, localPath });
|
|
3364
3063
|
continue;
|
|
3365
3064
|
}
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3065
|
+
const journalEntry = getEntry(journal, remoteFile.key);
|
|
3066
|
+
// Reindex and package wiring own symlinks under these `core/<type>`
|
|
3067
|
+
// surfaces; their canonical sources are elsewhere. A prior pull records
|
|
3068
|
+
// the remote object as a symlink, then reindex removes the generated link.
|
|
3069
|
+
// Without this classification the next pull sees only an absent local path
|
|
3070
|
+
// and downloads the same marker forever. Retain regular core files: the
|
|
3071
|
+
// journal kind is the discriminator, because LIST exposes no object type.
|
|
3072
|
+
// An unchanged ETag is also required: a remote writer can replace a legacy
|
|
3073
|
+
// symlink with canonical file content at the same key, and that replacement
|
|
3074
|
+
// must still be pulled.
|
|
3075
|
+
if (personalMode &&
|
|
3076
|
+
journalEntry?.kind === "symlink" &&
|
|
3077
|
+
isGeneratedCoreMirrorKey(remoteFile.key) &&
|
|
3078
|
+
journalEntry.remoteEtag !== undefined &&
|
|
3079
|
+
!hasRemoteChanged(remoteFile, journalEntry)) {
|
|
3080
|
+
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
3081
|
+
continue;
|
|
3082
|
+
}
|
|
3083
|
+
// Junk-spelling guard — SKILL-FARM KEYS ONLY. The amplifier is specific
|
|
3084
|
+
// to `.claude/skills/<ns>:<skill>` wrapper spellings; outside that prefix,
|
|
3085
|
+
// keys whose percent-decode fixpoints collide (`notes/a b.md` vs a genuine
|
|
3086
|
+
// `notes/a%20b.md`) are DISTINCT files that must both pull.
|
|
3087
|
+
// Within the farm: a remote key that is NOT journaled under its own
|
|
3088
|
+
// spelling but whose percent-decode fixpoint matches an already-journaled
|
|
3089
|
+
// key with a different spelling is a mixed-version twin (`%3A`/`%253A`
|
|
3090
|
+
// generations of a canonical `:` key). Materializing it would mint junk
|
|
3091
|
+
// local dirs and fork the journal into a second key family for the same
|
|
3092
|
+
// logical path — skip it (loudly, never fatally; see the
|
|
3093
|
+
// `skip-junk-key-spelling` event doc) and leave the journaled spelling
|
|
3094
|
+
// authoritative for this machine.
|
|
3095
|
+
if (journalEntry === undefined &&
|
|
3096
|
+
remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
|
|
3097
|
+
// Gate NON-canonical remote spellings only. A remote key that already
|
|
3098
|
+
// IS the canonical spelling must always be downloadable — after the
|
|
3099
|
+
// doctor collapses the vault, a machine whose journal still holds only
|
|
3100
|
+
// the junk spelling would otherwise skip the canonical object forever
|
|
3101
|
+
// (and the delete-resync twin protection keeps that stale junk entry),
|
|
3102
|
+
// deadlocking it out of every future update.
|
|
3103
|
+
canonicalVaultKeySpelling(remoteFile.key) !== remoteFile.key) {
|
|
3104
|
+
const journaledTwin = journaledKeyByCanonical.get(canonicalVaultKeySpelling(remoteFile.key));
|
|
3105
|
+
if (journaledTwin !== undefined && journaledTwin !== remoteFile.key) {
|
|
3375
3106
|
items.push({
|
|
3376
|
-
action: "
|
|
3107
|
+
action: "skip-junk-key-spelling",
|
|
3377
3108
|
remoteFile,
|
|
3378
3109
|
localPath,
|
|
3379
|
-
|
|
3380
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3110
|
+
journaledKey: journaledTwin,
|
|
3381
3111
|
});
|
|
3382
3112
|
continue;
|
|
3383
3113
|
}
|
|
3384
|
-
|
|
3114
|
+
}
|
|
3115
|
+
// ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
|
|
3116
|
+
// A remote object present in the LIST may be an intentionally-deleted key
|
|
3117
|
+
// (a peer re-pushed it, or its delete-marker hasn't propagated to this
|
|
3118
|
+
// caller's view). `tombstoneSuppresses` is true when a FILE_TOMBSTONE marks
|
|
3119
|
+
// this key deleted AND the remote object is NOT a newer re-create — i.e. the
|
|
3120
|
+
// object is the stale/deleted version and must NOT be downloaded. A genuine
|
|
3121
|
+
// re-create (object newer than the tombstone) leaves this false so the
|
|
3122
|
+
// normal download/merge path runs (the tombstone is not permanent
|
|
3123
|
+
// suppression). An empty tombstone map (the default / degraded-fetch case)
|
|
3124
|
+
// makes this always false — legacy behavior preserved bit-for-bit.
|
|
3125
|
+
const tombstone = fileTombstones.get(toPosixKey(remoteFile.key));
|
|
3126
|
+
const tombstoneSuppresses = tombstone !== undefined &&
|
|
3127
|
+
!isRemoteRecreateAfterTombstone(remoteFile, tombstone);
|
|
3128
|
+
// LOCAL journal tombstone — this machine's own intentional-delete record,
|
|
3129
|
+
// independent of the (best-effort, intermittently-degraded) server
|
|
3130
|
+
// FILE_TOMBSTONE fetch above. When a clean local delete is classified, the
|
|
3131
|
+
// push leg stamps `removedAt` on the journal entry; honoring it here makes
|
|
3132
|
+
// the delete-vs-drift-restore outcome DETERMINISTIC, closing the
|
|
3133
|
+
// intermittent respawn where a degraded tombstone fetch let the pull leg
|
|
3134
|
+
// re-materialize an intentionally-deleted file. Reuses the same re-create
|
|
3135
|
+
// guard (a remote object newer than the local delete is a genuine re-create
|
|
3136
|
+
// and still downloads). Consulted ONLY on the `!localExists` re-download
|
|
3137
|
+
// path below — never to delete a file that is present locally.
|
|
3138
|
+
const localTombstoneSuppresses = journalEntry !== undefined &&
|
|
3139
|
+
isTombstone(journalEntry) &&
|
|
3140
|
+
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
3141
|
+
journalEntry.removedAt !== undefined &&
|
|
3142
|
+
!isRemoteRecreateAfterTombstone(remoteFile, {
|
|
3143
|
+
deletedAt: journalEntry.removedAt,
|
|
3144
|
+
});
|
|
3145
|
+
// Only a key that passed validation, policy, scope and ignore filtering
|
|
3146
|
+
// and is not itself a deleted object can own a local file, so only such a
|
|
3147
|
+
// key may protect a case-only twin from deletion (feedback_c8ed69d1). A
|
|
3148
|
+
// local file whose folder merely keeps an older case spelling is not
|
|
3149
|
+
// protected: when no live key owns it, the remote delete still applies.
|
|
3150
|
+
if (!tombstoneSuppresses && !localTombstoneSuppresses) {
|
|
3151
|
+
caseAliasTracker?.observeRemote(remoteFile.key);
|
|
3152
|
+
}
|
|
3153
|
+
// lstat (not existsSync/statSync) handles three cases the legacy
|
|
3154
|
+
// checks got wrong for symlinks:
|
|
3155
|
+
// 1. A valid symlink at localPath: existsSync returns true and the
|
|
3156
|
+
// planner falls through to hashFile, which follows the link and
|
|
3157
|
+
// hashes the target file's bytes — never matching the journal's
|
|
3158
|
+
// sha256(target string), so every pull would mis-classify a
|
|
3159
|
+
// pristine symlink as locally changed.
|
|
3160
|
+
// 2. A dangling symlink: existsSync returns false (existsSync
|
|
3161
|
+
// follows links) so the planner treats the entry as missing and
|
|
3162
|
+
// re-downloads, silently clobbering the user's intentional
|
|
3163
|
+
// dangling link on every run.
|
|
3164
|
+
// 3. A regular file: indistinguishable from current behaviour.
|
|
3165
|
+
let localLstat = null;
|
|
3166
|
+
let plannedLocalSnapshot = { kind: "absent" };
|
|
3167
|
+
let localPathBlockedByFileAncestor = false;
|
|
3168
|
+
try {
|
|
3169
|
+
localLstat = fs.lstatSync(localPath);
|
|
3170
|
+
}
|
|
3171
|
+
catch (err) {
|
|
3172
|
+
const code = err && typeof err === "object" && "code" in err
|
|
3173
|
+
? err.code
|
|
3174
|
+
: undefined;
|
|
3175
|
+
// ENOENT → truly absent → treated as "new file from cloud".
|
|
3176
|
+
// ENOTDIR → an ancestor of localPath is a regular file (cloud has a
|
|
3177
|
+
// dir at that key, local has a file). Pre-fix this threw
|
|
3178
|
+
// and aborted the whole company sync (Bug #10 in the
|
|
3179
|
+
// 5.33.0 deep-test verification — \`v4-dir-vs-file\` repro
|
|
3180
|
+
// wedged the personal company at \"errored\" status,
|
|
3181
|
+
// skipping every later file). Recover by classifying as
|
|
3182
|
+
// a structural collision the operator must resolve
|
|
3183
|
+
// manually, and let the rest of the company process.
|
|
3184
|
+
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
3185
|
+
throw err;
|
|
3186
|
+
}
|
|
3187
|
+
if (code === "ENOTDIR") {
|
|
3188
|
+
localPathBlockedByFileAncestor = true;
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
const localExists = localLstat !== null;
|
|
3192
|
+
if (localPathBlockedByFileAncestor) {
|
|
3193
|
+
// Symmetric counterpart to the (local-dir, cloud-file) warning below.
|
|
3194
|
+
// Emit the same "manual reconciliation required" surface so the
|
|
3195
|
+
// operator sees one consistent message for both topologies; record as
|
|
3196
|
+
// skip-local-only so the file is not silently dropped from the
|
|
3197
|
+
// count and the executor never tries to lstat/write it.
|
|
3198
|
+
console.error(` Warning: an ancestor of ${remoteFile.key} exists locally as a regular file; ` +
|
|
3199
|
+
`cloud has a deeper key under that path. Skipping; manual ` +
|
|
3200
|
+
`reconciliation required (rm the conflicting local file to pull).`);
|
|
3201
|
+
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
3385
3202
|
continue;
|
|
3386
3203
|
}
|
|
3387
|
-
if (
|
|
3388
|
-
|
|
3389
|
-
//
|
|
3390
|
-
//
|
|
3391
|
-
//
|
|
3392
|
-
//
|
|
3393
|
-
//
|
|
3394
|
-
//
|
|
3395
|
-
//
|
|
3396
|
-
//
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3204
|
+
if (localExists) {
|
|
3205
|
+
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
3206
|
+
// Kind-mismatch guard: a remote LIST entry is always a single
|
|
3207
|
+
// S3 object (regular file or symlink record), but the local
|
|
3208
|
+
// path may be a real directory if the user previously had a
|
|
3209
|
+
// dir at this key. Hashing a directory throws EISDIR and
|
|
3210
|
+
// aborts the whole pull. We don't auto-replace because that
|
|
3211
|
+
// would `rm -rf` the user's directory; surface as a non-fatal
|
|
3212
|
+
// skip and warn so the operator can reconcile manually
|
|
3213
|
+
// (delete the local dir, or restructure the remote).
|
|
3214
|
+
if (localLstat.isDirectory() && !isLocalSymlink) {
|
|
3215
|
+
// Stale personal-overlay marker vs. release-shipped core directory:
|
|
3216
|
+
// a single vault object at a `core/<overlay-type>/.../<entry>` key
|
|
3217
|
+
// whose local counterpart is a real directory is an orphaned overlay
|
|
3218
|
+
// symlink marker (reindex mirror artifact) left behind after a release
|
|
3219
|
+
// shipped a real core dir of the same name. The local release-shipped
|
|
3220
|
+
// directory is authoritative and the marker can never be materialized
|
|
3221
|
+
// over it, so keep local and ignore the marker QUIETLY — the generic
|
|
3222
|
+
// "rm -rf the local directory" advice below is actively dangerous here
|
|
3223
|
+
// (it would destroy release-shipped core content to pull inert derived
|
|
3224
|
+
// state), and it recurred on every sync. feedback_ce9aeede.
|
|
3225
|
+
if (isGeneratedCoreMirrorKey(remoteFile.key)) {
|
|
3226
|
+
console.error(` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
|
|
3227
|
+
`shadowing a release-shipped core directory; keeping the local ` +
|
|
3228
|
+
`directory and ignoring the marker (no action needed).`);
|
|
3229
|
+
items.push({
|
|
3230
|
+
action: "skip-stale-overlay-marker",
|
|
3231
|
+
remoteFile,
|
|
3232
|
+
localPath,
|
|
3233
|
+
});
|
|
3234
|
+
continue;
|
|
3235
|
+
}
|
|
3236
|
+
// Directory-overlay marker vs. a locally-materialized directory:
|
|
3237
|
+
// the remote object at this key carries CHILD objects under it in the
|
|
3238
|
+
// same LIST (`remoteKeysWithChildren`), so it is a symlink/overlay
|
|
3239
|
+
// record for a directory — e.g. the DPP vault's `skills/cody-copywriter`
|
|
3240
|
+
// overlay whose 13 child objects are already materialized locally. The
|
|
3241
|
+
// marker can never be renamed over the real directory, and its children
|
|
3242
|
+
// carry the actual content and materialize via their own plan items, so
|
|
3243
|
+
// the marker is a benign, no-action skip. The generic "rm -rf the local
|
|
3244
|
+
// directory to pull" advice below would tell the operator to DELETE the
|
|
3245
|
+
// materialized (and possibly locally-authored) contents to install an
|
|
3246
|
+
// inert link, and it recurred on EVERY sync cycle. feedback_d2082110.
|
|
3247
|
+
if (remoteKeysWithChildren.has(toPosixKey(remoteFile.key))) {
|
|
3248
|
+
console.error(` Note: ${remoteFile.key} is a directory-overlay marker whose ` +
|
|
3249
|
+
`contents already exist locally as a directory; keeping the ` +
|
|
3250
|
+
`local directory and ignoring the marker (no action needed).`);
|
|
3251
|
+
items.push({
|
|
3252
|
+
action: "skip-overlay-marker-with-children",
|
|
3253
|
+
remoteFile,
|
|
3254
|
+
localPath,
|
|
3255
|
+
});
|
|
3256
|
+
continue;
|
|
3257
|
+
}
|
|
3258
|
+
console.error(` Warning: ${remoteFile.key} exists locally as a directory; ` +
|
|
3259
|
+
`cloud has a single object at this key. Skipping; manual ` +
|
|
3260
|
+
`reconciliation required (rm -rf the local directory to pull).`);
|
|
3261
|
+
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
3262
|
+
continue;
|
|
3263
|
+
}
|
|
3264
|
+
let localHash;
|
|
3265
|
+
if (isLocalSymlink) {
|
|
3266
|
+
const target = readlinkOrNull(localPath);
|
|
3267
|
+
if (target === null) {
|
|
3268
|
+
// We cannot prove whether the local link diverged, so never
|
|
3269
|
+
// overwrite or tombstone it from this plan. A later pass can retry
|
|
3270
|
+
// once the OS exposes a readable target.
|
|
3271
|
+
items.push({ action: "skip-unreadable-link", remoteFile, localPath });
|
|
3272
|
+
continue;
|
|
3273
|
+
}
|
|
3274
|
+
localHash = hashSymlinkTarget(target);
|
|
3275
|
+
}
|
|
3276
|
+
else if (journalEntry && journalEntryMatchesStat(journalEntry, localLstat)) {
|
|
3277
|
+
// size + mtime + ctime all match the row that recorded this hash, so
|
|
3278
|
+
// the bytes have not been written since (ctime cannot be set from
|
|
3279
|
+
// userspace — see `journalEntryMatchesStat`). Reading them would only
|
|
3280
|
+
// reproduce the recorded hash. This is the pull half of the push
|
|
3281
|
+
// planner's stat fast path; without it every local file with a remote
|
|
3282
|
+
// counterpart was hashed on every pass — about 13 GB per runner start
|
|
3283
|
+
// on a real install, a quarter of the first pass's CPU.
|
|
3284
|
+
localHash = journalEntry.hash;
|
|
3285
|
+
}
|
|
3286
|
+
else {
|
|
3287
|
+
// A same-size rewrite can preserve mtime, so mtime alone cannot prove
|
|
3288
|
+
// content identity, and rows without stamps carry no evidence at all.
|
|
3289
|
+
// Hash the file before deciding whether local state diverged from the
|
|
3290
|
+
// journal baseline.
|
|
3291
|
+
localHash = hashFile(localPath);
|
|
3292
|
+
}
|
|
3293
|
+
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
3294
|
+
plannedLocalSnapshot = {
|
|
3295
|
+
kind: isLocalSymlink ? "symlink" : "file",
|
|
3296
|
+
hash: localHash,
|
|
3297
|
+
};
|
|
3298
|
+
const remoteChanged = !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
|
3299
|
+
const announcedContentHash = pullContentHashes[remoteFile.key];
|
|
3300
|
+
// ── Authoritative delete reaches a machine that still holds the file ──
|
|
3301
|
+
// A FILE_TOMBSTONE marks this key deleted and the remote object is not a
|
|
3302
|
+
// newer re-create, but this machine still has a local copy. Honor the
|
|
3303
|
+
// delete — EXCEPT never destroy unsynced local work: if the local copy
|
|
3304
|
+
// diverges from the synced baseline (a post-tombstone edit, or an
|
|
3305
|
+
// untracked local file with no journal entry), surface a CONFLICT instead
|
|
3306
|
+
// of silently deleting (the safety guard the brief requires + the 3-way
|
|
3307
|
+
// merge invariant). A local copy that still matches the deleted baseline
|
|
3308
|
+
// is the stale version → delete it + drop the journal entry so it stays
|
|
3309
|
+
// gone on this machine.
|
|
3310
|
+
if (tombstoneSuppresses) {
|
|
3311
|
+
if (localChanged || !journalEntry) {
|
|
3312
|
+
items.push({
|
|
3313
|
+
action: "conflict",
|
|
3314
|
+
remoteFile,
|
|
3315
|
+
localPath,
|
|
3316
|
+
localHash,
|
|
3317
|
+
localMtime: localLstat.mtime,
|
|
3318
|
+
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3319
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
3322
|
+
else {
|
|
3323
|
+
items.push({
|
|
3324
|
+
action: "tombstone-delete",
|
|
3325
|
+
remoteFile,
|
|
3326
|
+
localPath,
|
|
3327
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3328
|
+
});
|
|
3329
|
+
}
|
|
3330
|
+
continue;
|
|
3331
|
+
}
|
|
3332
|
+
// ── Untracked-local guard (unjournaled local file) ──────────────
|
|
3333
|
+
// Both `localChanged` and `remoteChanged` are gated on `!!journalEntry`,
|
|
3334
|
+
// so a key with NO journal entry reads "clean on both sides" and falls
|
|
3335
|
+
// all the way through to a plain `download` — silently replacing a local
|
|
3336
|
+
// file that exists on disk. There is no conflict, no `.conflict-` mirror
|
|
3337
|
+
// and no conflict-index record, so the loss is invisible unless someone
|
|
3338
|
+
// happens to know what the file should contain.
|
|
3403
3339
|
//
|
|
3404
|
-
//
|
|
3405
|
-
//
|
|
3406
|
-
//
|
|
3407
|
-
//
|
|
3408
|
-
//
|
|
3409
|
-
//
|
|
3410
|
-
//
|
|
3411
|
-
//
|
|
3412
|
-
//
|
|
3413
|
-
// length, so the comparison is meaningless.
|
|
3340
|
+
// This destroyed in-progress edits to
|
|
3341
|
+
// `companies/*/projects/*/prd.json` three times on 2026-08-01/02, but
|
|
3342
|
+
// nothing about it is prd- or project-specific: it fires for ANY synced
|
|
3343
|
+
// path whose journal entry is absent. Absence is common — a file created
|
|
3344
|
+
// on this machine and on a peer under the same key before either pushed,
|
|
3345
|
+
// a journal shard reset/lost/rebuilt (`readJournal` returns an empty
|
|
3346
|
+
// `files` map for a missing shard, so EVERY divergent local file is in
|
|
3347
|
+
// scope), a run under a different `HQ_STATE_DIR`, or an entry dropped by
|
|
3348
|
+
// scope-shrink or rescue.
|
|
3414
3349
|
//
|
|
3415
|
-
//
|
|
3416
|
-
//
|
|
3417
|
-
//
|
|
3418
|
-
//
|
|
3419
|
-
//
|
|
3420
|
-
//
|
|
3421
|
-
//
|
|
3422
|
-
//
|
|
3423
|
-
|
|
3350
|
+
// No journal entry means no evidence that local and remote ever agreed,
|
|
3351
|
+
// which is exactly when an overwrite is least safe. Route it through the
|
|
3352
|
+
// conflict path instead: the executor's convergence probe fetches the
|
|
3353
|
+
// remote once and, when the bytes already match (the ordinary
|
|
3354
|
+
// fresh-machine / rebuilt-journal case), reconciles silently — no
|
|
3355
|
+
// conflict, no mirror, journal stamped. Only genuine divergence surfaces,
|
|
3356
|
+
// and there local is PRESERVED as-is with the cloud copy written beside
|
|
3357
|
+
// it under `--on-conflict keep`, or the run halts under `abort`.
|
|
3358
|
+
// Cloud-authoritative paths (board.json, ontology/, signals/, …) are
|
|
3359
|
+
// unaffected — the conflict executor short-circuits those back to a
|
|
3360
|
+
// pull-wins download before any mirror is minted.
|
|
3361
|
+
if (!journalEntry) {
|
|
3362
|
+
items.push({
|
|
3363
|
+
action: "conflict",
|
|
3364
|
+
remoteFile,
|
|
3365
|
+
localPath,
|
|
3366
|
+
localHash,
|
|
3367
|
+
localMtime: localLstat.mtime,
|
|
3368
|
+
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3369
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3370
|
+
});
|
|
3371
|
+
continue;
|
|
3372
|
+
}
|
|
3373
|
+
// An event hash and a LIST row are separate observations. Only use the
|
|
3374
|
+
// event to skip a changed ETag after HEAD proves that the exact listed
|
|
3375
|
+
// revision carries that hash; otherwise the event may be stale and the
|
|
3376
|
+
// LIST may already describe newer bytes.
|
|
3377
|
+
let listedRevisionMatchesAnnouncement = false;
|
|
3378
|
+
let kms10CopyMatchesJournalBase = false;
|
|
3379
|
+
if (ctx !== undefined &&
|
|
3380
|
+
remoteChanged &&
|
|
3424
3381
|
!isLocalSymlink &&
|
|
3425
|
-
|
|
3382
|
+
!localChanged &&
|
|
3383
|
+
announcedContentHash !== undefined &&
|
|
3384
|
+
contentHashMatchesRawSha256(announcedContentHash, journalEntry.hash) &&
|
|
3385
|
+
journalEntryMatchesStat(journalEntry, localLstat) &&
|
|
3386
|
+
remoteFile.size === localLstat.size &&
|
|
3387
|
+
normalizeEtag(remoteFile.etag) !== "" &&
|
|
3388
|
+
normalizeEtag(remoteFile.etag) !== journalEntry.remoteEtag) {
|
|
3389
|
+
try {
|
|
3390
|
+
const headResult = prefetchedHeads.get(remoteFile.key);
|
|
3391
|
+
if (headResult?.error !== undefined)
|
|
3392
|
+
throw headResult.error;
|
|
3393
|
+
const head = headResult
|
|
3394
|
+
? headResult.head
|
|
3395
|
+
: await headRemoteFile(ctx, remoteFile.key);
|
|
3396
|
+
const remoteContentHash = head?.metadata?.[FILE_CONTENT_SHA256_META_KEY];
|
|
3397
|
+
listedRevisionMatchesAnnouncement =
|
|
3398
|
+
head !== null &&
|
|
3399
|
+
normalizeEtag(head.etag) === normalizeEtag(remoteFile.etag) &&
|
|
3400
|
+
head.size === remoteFile.size &&
|
|
3401
|
+
remoteContentHash !== undefined &&
|
|
3402
|
+
contentHashMatchesRawSha256(announcedContentHash, remoteContentHash);
|
|
3403
|
+
}
|
|
3404
|
+
catch (headError) {
|
|
3405
|
+
// No proof means the ordinary GET path remains authoritative.
|
|
3406
|
+
console.error(`sync: unable to verify announced content hash for ${remoteFile.key}; falling back to GET`, headError);
|
|
3407
|
+
}
|
|
3408
|
+
}
|
|
3409
|
+
const kmsHeadResult = prefetchedHeads.get(remoteFile.key);
|
|
3410
|
+
if (kmsHeadResult !== undefined) {
|
|
3411
|
+
if (kmsHeadResult.error !== undefined) {
|
|
3412
|
+
console.error(`sync: unable to verify marked KMS-10 copy for ${remoteFile.key}; falling back to GET`, kmsHeadResult.error);
|
|
3413
|
+
}
|
|
3414
|
+
else {
|
|
3415
|
+
kms10CopyMatchesJournalBase = isKms10CopyOfJournalBase(remoteFile, kmsHeadResult.head, journalEntry.hash, journalEntry.remoteEtag ?? "", remoteFile.key);
|
|
3416
|
+
}
|
|
3417
|
+
}
|
|
3418
|
+
if (listedRevisionMatchesAnnouncement) {
|
|
3419
|
+
items.push({
|
|
3420
|
+
action: "skip-content-match",
|
|
3421
|
+
remoteFile,
|
|
3422
|
+
localPath,
|
|
3423
|
+
contentHash: announcedContentHash,
|
|
3424
|
+
});
|
|
3425
|
+
continue;
|
|
3426
|
+
}
|
|
3427
|
+
if (kms10CopyMatchesJournalBase) {
|
|
3428
|
+
if (localChanged || !journalEntryMatchesStat(journalEntry, localLstat)) {
|
|
3429
|
+
items.push({
|
|
3430
|
+
action: "skip-kms10-copy",
|
|
3431
|
+
remoteFile,
|
|
3432
|
+
localPath,
|
|
3433
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3434
|
+
journalBaseHash: journalEntry.hash,
|
|
3435
|
+
});
|
|
3436
|
+
}
|
|
3437
|
+
else {
|
|
3438
|
+
items.push({
|
|
3439
|
+
action: "skip-content-match",
|
|
3440
|
+
remoteFile,
|
|
3441
|
+
localPath,
|
|
3442
|
+
contentHash: `sha256:${journalEntry.hash}`,
|
|
3443
|
+
});
|
|
3444
|
+
}
|
|
3445
|
+
continue;
|
|
3446
|
+
}
|
|
3447
|
+
// Mirror the original 3-way merge from the inline loop. Tested by
|
|
3448
|
+
// `does NOT flag a pull conflict when only local changed since last
|
|
3449
|
+
// sync` and `detects conflicts with local changes…`.
|
|
3450
|
+
if (localChanged && remoteChanged) {
|
|
3426
3451
|
items.push({
|
|
3427
3452
|
action: "conflict",
|
|
3428
3453
|
remoteFile,
|
|
3429
3454
|
localPath,
|
|
3430
3455
|
localHash,
|
|
3456
|
+
// localLstat is non-null inside this `if (localExists)`
|
|
3457
|
+
// branch — the planner already lstat'd the path. Carry
|
|
3458
|
+
// its mtime forward so the executor doesn't have to
|
|
3459
|
+
// re-stat (which would follow links and crash on a
|
|
3460
|
+
// dangling symlink). Same goes for size: the conflict
|
|
3461
|
+
// executor's keep/skip branch needs to stamp the
|
|
3462
|
+
// journal with a size, and statSync on a dangling
|
|
3463
|
+
// symlink throws ENOENT (silently swallowed pre-fix),
|
|
3464
|
+
// leaving the journal stale and re-firing the conflict
|
|
3465
|
+
// every sync.
|
|
3431
3466
|
localMtime: localLstat.mtime,
|
|
3432
|
-
localSize: localLstat.size,
|
|
3467
|
+
localSize: isLocalSymlink ? 0 : localLstat.size,
|
|
3433
3468
|
localSnapshot: plannedLocalSnapshot,
|
|
3434
3469
|
});
|
|
3435
3470
|
continue;
|
|
3436
3471
|
}
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
|
|
3472
|
+
if (journalEntry && localChanged && !remoteChanged) {
|
|
3473
|
+
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
3474
|
+
continue;
|
|
3475
|
+
}
|
|
3476
|
+
if (journalEntry && !localChanged && !remoteChanged) {
|
|
3477
|
+
// Flavor-health repair (win32 skill farm): a link journaled while its
|
|
3478
|
+
// target was absent was minted file-flavor; once the target directory
|
|
3479
|
+
// exists the hashes still match (hash = sha256(target)) so this
|
|
3480
|
+
// branch would skip it forever, leaving the wrapper broken. Re-download
|
|
3481
|
+
// instead — the materializer's flavor-aware path recreates it 'dir'.
|
|
3482
|
+
if (isLocalSymlink &&
|
|
3483
|
+
remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
|
|
3484
|
+
win32SymlinkFlavorBroken(localPath)) {
|
|
3485
|
+
items.push({
|
|
3486
|
+
action: "download",
|
|
3487
|
+
remoteFile,
|
|
3488
|
+
localPath,
|
|
3489
|
+
isNew: false,
|
|
3490
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3491
|
+
});
|
|
3492
|
+
continue;
|
|
3493
|
+
}
|
|
3494
|
+
items.push({ action: "skip-unchanged", remoteFile, localPath });
|
|
3495
|
+
continue;
|
|
3496
|
+
}
|
|
3497
|
+
if (journalEntry && !localChanged && remoteChanged) {
|
|
3498
|
+
// ── Anti-rollback guard (silent newer→older overwrite) ───────────
|
|
3499
|
+
// Remote etag moved but local still matches the journal baseline, so
|
|
3500
|
+
// the classic 3-way merge calls this a clean `download` and overwrites
|
|
3501
|
+
// local with remote. That assumes the remote *advanced*. It can also
|
|
3502
|
+
// REGRESS: a second device (or a stale re-push) can replace the S3
|
|
3503
|
+
// object with an OLDER, SMALLER copy. `hasRemoteChanged` is pure-ETag
|
|
3504
|
+
// and cannot tell an advance from a rollback — ListObjectsV2 carries no
|
|
3505
|
+
// content hash, and `lastModified` is the upload time (newer even for
|
|
3506
|
+
// stale content), so neither distinguishes the two. A silent download
|
|
3507
|
+
// then DESTROYS newer local bytes with no conflict surfaced.
|
|
3508
|
+
//
|
|
3509
|
+
// This bit append-only bot chat logs on 2026-06-10, the day
|
|
3510
|
+
// event-driven bidirectional sync (which lets a peer device push to
|
|
3511
|
+
// S3 between this device's pulls) first rolled out: a regressed S3
|
|
3512
|
+
// object truncated freshly-appended lines, undetected by the planner.
|
|
3513
|
+
//
|
|
3514
|
+
// Guard: if the incoming remote object is strictly SMALLER than the
|
|
3515
|
+
// local file we already hold (a size regression on a tracked regular
|
|
3516
|
+
// file), do NOT silently clobber. Route through the conflict path —
|
|
3517
|
+
// the executor's convergence probe fetches the remote once and, on
|
|
3518
|
+
// genuine divergence, PRESERVES local as a `.conflict-*` mirror instead
|
|
3519
|
+
// of overwriting it (and under `--on-conflict abort` halts loudly
|
|
3520
|
+
// rather than losing data). A byte-identical remote can never be
|
|
3521
|
+
// smaller, so this never manufactures a false conflict on a no-op
|
|
3522
|
+
// re-upload. Symlinks are exempt: their on-disk size is not the target
|
|
3523
|
+
// length, so the comparison is meaningless.
|
|
3524
|
+
//
|
|
3525
|
+
// Direction gate: only guard copies this device AUTHORED (last synced
|
|
3526
|
+
// UP). A consumer copy (last synced DOWN) has no local authorship to
|
|
3527
|
+
// protect — the remote is authoritative, so a smaller remote is a
|
|
3528
|
+
// legitimate update (e.g. the gardener rewriting a shorter brief), not a
|
|
3529
|
+
// regression. Without this gate a consumer device floods `.conflict`
|
|
3530
|
+
// mirrors and stalls on stale gardener output forever (the 786-file
|
|
3531
|
+
// pile-up, 2026-06-22). `"up"` and any missing/unknown direction stay
|
|
3532
|
+
// guarded exactly as before — the 2026-06-10 chat-log protection holds.
|
|
3533
|
+
if (journalEntry.direction !== "down" &&
|
|
3534
|
+
!isLocalSymlink &&
|
|
3535
|
+
remoteFile.size < localLstat.size) {
|
|
3536
|
+
items.push({
|
|
3537
|
+
action: "conflict",
|
|
3538
|
+
remoteFile,
|
|
3539
|
+
localPath,
|
|
3540
|
+
localHash,
|
|
3541
|
+
localMtime: localLstat.mtime,
|
|
3542
|
+
localSize: localLstat.size,
|
|
3543
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3544
|
+
});
|
|
3545
|
+
continue;
|
|
3546
|
+
}
|
|
3547
|
+
// Remote genuinely advanced (same-or-larger) → normal catch-up
|
|
3548
|
+
// download below.
|
|
3549
|
+
}
|
|
3550
|
+
// No journal entry, or remote-only changed (non-shrinking) → fall
|
|
3551
|
+
// through to download.
|
|
3552
|
+
}
|
|
3553
|
+
// Suppress the re-download of an intentionally-deleted key (delete-resync).
|
|
3554
|
+
// Reaches here only for the `!localExists` case — the `localExists` branch
|
|
3555
|
+
// above already handled (and `continue`d) any suppressing tombstone, so a
|
|
3556
|
+
// tombstone that survives to here means "remote present, local absent, not a
|
|
3557
|
+
// re-create": the classic resurrection ("remote present → I'm behind →
|
|
3558
|
+
// download") the planner must NOT do. Route to `tombstone-delete` so the
|
|
3559
|
+
// executor drops any stale journal entry and the key stays gone, instead of
|
|
3560
|
+
// pulling the deleted object back in. `localTombstoneSuppresses` adds the
|
|
3561
|
+
// same protection driven by this machine's own journal tombstone, so an
|
|
3562
|
+
// intentional delete is not resurrected even when the server FILE_TOMBSTONE
|
|
3563
|
+
// fetch is degraded.
|
|
3564
|
+
if (tombstoneSuppresses || localTombstoneSuppresses) {
|
|
3565
|
+
// A persisting `local-delete` tombstone (stamped by the intentional-delete
|
|
3566
|
+
// producer below) must KEEP its journal entry — the push leg still needs
|
|
3567
|
+
// it to propagate the S3 delete. Skip the re-download without dropping the
|
|
3568
|
+
// entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
|
|
3569
|
+
if (journalEntry?.removedReason === "local-delete") {
|
|
3570
|
+
continue;
|
|
3571
|
+
}
|
|
3572
|
+
items.push({
|
|
3573
|
+
action: "tombstone-delete",
|
|
3574
|
+
remoteFile,
|
|
3575
|
+
localPath,
|
|
3576
|
+
localSnapshot: plannedLocalSnapshot,
|
|
3577
|
+
});
|
|
3578
|
+
continue;
|
|
3579
|
+
}
|
|
3580
|
+
// Intentional-delete producer: a clean, CURRENT (etag matches), non-divergent
|
|
3581
|
+
// file that is MISSING locally is the user deleting a genuinely-synced file.
|
|
3582
|
+
// A watcher intent is the strict proof used by `owned-only` (and `all`).
|
|
3583
|
+
// `currency-gated` additionally admits an intent-less candidate because a
|
|
3584
|
+
// watcherless runner can never mint one; the matching ETag still proves no
|
|
3585
|
+
// peer changed this individual object. The post-loop bulk-asymmetry decision
|
|
3586
|
+
// remains the whole-set guard for BOTH candidate classes. A stale-etag,
|
|
3587
|
+
// divergent, tombstoned, or never-journaled missing file falls through to
|
|
3588
|
+
// the normal download below.
|
|
3589
|
+
if (!localExists &&
|
|
3590
|
+
journalEntry !== undefined &&
|
|
3591
|
+
!isTombstone(journalEntry) &&
|
|
3592
|
+
!journalEntry.localDiverges &&
|
|
3593
|
+
(hasCurrentLocalDeleteIntent(journalEntry) ||
|
|
3594
|
+
propagateDeletePolicy === "currency-gated") &&
|
|
3595
|
+
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag) {
|
|
3596
|
+
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
3460
3597
|
continue;
|
|
3461
3598
|
}
|
|
3462
3599
|
items.push({
|
|
3463
|
-
action: "
|
|
3600
|
+
action: "download",
|
|
3464
3601
|
remoteFile,
|
|
3465
3602
|
localPath,
|
|
3603
|
+
isNew: !localExists,
|
|
3466
3604
|
localSnapshot: plannedLocalSnapshot,
|
|
3467
3605
|
});
|
|
3468
|
-
continue;
|
|
3469
3606
|
}
|
|
3470
|
-
// Intentional-delete producer: a clean, CURRENT (etag matches), non-divergent
|
|
3471
|
-
// file that is MISSING locally is the user deleting a genuinely-synced file.
|
|
3472
|
-
// A watcher intent is the strict proof used by `owned-only` (and `all`).
|
|
3473
|
-
// `currency-gated` additionally admits an intent-less candidate because a
|
|
3474
|
-
// watcherless runner can never mint one; the matching ETag still proves no
|
|
3475
|
-
// peer changed this individual object. The post-loop bulk-asymmetry decision
|
|
3476
|
-
// remains the whole-set guard for BOTH candidate classes. A stale-etag,
|
|
3477
|
-
// divergent, tombstoned, or never-journaled missing file falls through to
|
|
3478
|
-
// the normal download below.
|
|
3479
|
-
if (!localExists &&
|
|
3480
|
-
journalEntry !== undefined &&
|
|
3481
|
-
!isTombstone(journalEntry) &&
|
|
3482
|
-
!journalEntry.localDiverges &&
|
|
3483
|
-
(hasCurrentLocalDeleteIntent(journalEntry) ||
|
|
3484
|
-
propagateDeletePolicy === "currency-gated") &&
|
|
3485
|
-
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag) {
|
|
3486
|
-
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
3487
|
-
continue;
|
|
3488
|
-
}
|
|
3489
|
-
items.push({
|
|
3490
|
-
action: "download",
|
|
3491
|
-
remoteFile,
|
|
3492
|
-
localPath,
|
|
3493
|
-
isNew: !localExists,
|
|
3494
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3495
|
-
});
|
|
3496
3607
|
}
|
|
3497
3608
|
dropJournalDecodeWindow();
|
|
3498
3609
|
if (cooperativeCheckpoint &&
|