@indigoai-us/hq-cloud 6.15.0 → 6.15.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-mutation.d.ts +16 -0
- package/dist/bin/sync-mutation.d.ts.map +1 -0
- package/dist/bin/sync-mutation.js +60 -0
- package/dist/bin/sync-mutation.js.map +1 -0
- package/dist/bin/sync-mutation.test.d.ts +2 -0
- package/dist/bin/sync-mutation.test.d.ts.map +1 -0
- package/dist/bin/sync-mutation.test.js +165 -0
- package/dist/bin/sync-mutation.test.js.map +1 -0
- package/dist/bin/sync-runner-company.d.ts +8 -0
- package/dist/bin/sync-runner-company.d.ts.map +1 -1
- package/dist/bin/sync-runner-company.js +16 -0
- package/dist/bin/sync-runner-company.js.map +1 -1
- package/dist/bin/sync-runner-company.test.d.ts +2 -0
- package/dist/bin/sync-runner-company.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-company.test.js +36 -0
- package/dist/bin/sync-runner-company.test.js.map +1 -0
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +98 -8
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +17 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +109 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/conflict-recovery.test.d.ts +2 -0
- package/dist/cli/conflict-recovery.test.d.ts.map +1 -0
- package/dist/cli/conflict-recovery.test.js +201 -0
- package/dist/cli/conflict-recovery.test.js.map +1 -0
- package/dist/cli/conflict.d.ts +60 -0
- package/dist/cli/conflict.d.ts.map +1 -1
- package/dist/cli/conflict.js +333 -0
- package/dist/cli/conflict.js.map +1 -1
- package/dist/cli/sync.d.ts +35 -0
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +100 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +85 -1
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/skill-telemetry.d.ts +6 -0
- package/dist/skill-telemetry.d.ts.map +1 -1
- package/dist/skill-telemetry.js +14 -2
- package/dist/skill-telemetry.js.map +1 -1
- package/dist/skill-telemetry.test.js +79 -0
- package/dist/skill-telemetry.test.js.map +1 -1
- package/dist/sync/candidate-uploader.d.ts +88 -0
- package/dist/sync/candidate-uploader.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.js +212 -0
- package/dist/sync/candidate-uploader.js.map +1 -0
- package/dist/sync/candidate-uploader.test.d.ts +2 -0
- package/dist/sync/candidate-uploader.test.d.ts.map +1 -0
- package/dist/sync/candidate-uploader.test.js +132 -0
- package/dist/sync/candidate-uploader.test.js.map +1 -0
- package/dist/sync/delta-client.d.ts +73 -0
- package/dist/sync/delta-client.d.ts.map +1 -0
- package/dist/sync/delta-client.js +201 -0
- package/dist/sync/delta-client.js.map +1 -0
- package/dist/sync/delta-client.test.d.ts +2 -0
- package/dist/sync/delta-client.test.d.ts.map +1 -0
- package/dist/sync/delta-client.test.js +97 -0
- package/dist/sync/delta-client.test.js.map +1 -0
- package/dist/sync/durable-apply.d.ts +76 -0
- package/dist/sync/durable-apply.d.ts.map +1 -0
- package/dist/sync/durable-apply.js +530 -0
- package/dist/sync/durable-apply.js.map +1 -0
- package/dist/sync/durable-apply.test.d.ts +2 -0
- package/dist/sync/durable-apply.test.d.ts.map +1 -0
- package/dist/sync/durable-apply.test.js +180 -0
- package/dist/sync/durable-apply.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +33 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +149 -1
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +142 -1
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +2 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +1 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/multipart-uploader.d.ts +99 -0
- package/dist/sync/multipart-uploader.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.js +447 -0
- package/dist/sync/multipart-uploader.js.map +1 -0
- package/dist/sync/multipart-uploader.test.d.ts +2 -0
- package/dist/sync/multipart-uploader.test.d.ts.map +1 -0
- package/dist/sync/multipart-uploader.test.js +119 -0
- package/dist/sync/multipart-uploader.test.js.map +1 -0
- package/dist/sync/mutation-client.d.ts +85 -0
- package/dist/sync/mutation-client.d.ts.map +1 -0
- package/dist/sync/mutation-client.js +245 -0
- package/dist/sync/mutation-client.js.map +1 -0
- package/dist/sync/mutation-client.test.d.ts +2 -0
- package/dist/sync/mutation-client.test.d.ts.map +1 -0
- package/dist/sync/mutation-client.test.js +51 -0
- package/dist/sync/mutation-client.test.js.map +1 -0
- package/dist/sync/push-receiver.d.ts +45 -0
- package/dist/sync/push-receiver.d.ts.map +1 -1
- package/dist/sync/push-receiver.js +101 -0
- package/dist/sync/push-receiver.js.map +1 -1
- package/dist/sync/push-receiver.test.js +54 -2
- package/dist/sync/push-receiver.test.js.map +1 -1
- package/dist/sync/scope-inventory-client.d.ts +69 -0
- package/dist/sync/scope-inventory-client.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.js +210 -0
- package/dist/sync/scope-inventory-client.js.map +1 -0
- package/dist/sync/scope-inventory-client.test.d.ts +2 -0
- package/dist/sync/scope-inventory-client.test.d.ts.map +1 -0
- package/dist/sync/scope-inventory-client.test.js +94 -0
- package/dist/sync/scope-inventory-client.test.js.map +1 -0
- package/dist/sync/snapshot-client.d.ts +98 -0
- package/dist/sync/snapshot-client.d.ts.map +1 -0
- package/dist/sync/snapshot-client.js +402 -0
- package/dist/sync/snapshot-client.js.map +1 -0
- package/dist/sync/snapshot-client.test.d.ts +2 -0
- package/dist/sync/snapshot-client.test.d.ts.map +1 -0
- package/dist/sync/snapshot-client.test.js +169 -0
- package/dist/sync/snapshot-client.test.js.map +1 -0
- package/dist/sync/uploader-finalization.d.ts +97 -0
- package/dist/sync/uploader-finalization.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.js +273 -0
- package/dist/sync/uploader-finalization.js.map +1 -0
- package/dist/sync/uploader-finalization.test.d.ts +2 -0
- package/dist/sync/uploader-finalization.test.d.ts.map +1 -0
- package/dist/sync/uploader-finalization.test.js +92 -0
- package/dist/sync/uploader-finalization.test.js.map +1 -0
- package/dist/telemetry.d.ts +11 -1
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +21 -2
- package/dist/telemetry.js.map +1 -1
- package/dist/telemetry.test.js +80 -0
- package/dist/telemetry.test.js.map +1 -1
- package/package.json +6 -1
- package/.claude/policies/hq-cloud-esm-cannot-spy-fs-builtins.md +0 -30
- package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +0 -22
- package/.github/workflows/ci.yml +0 -84
- package/.github/workflows/publish.yml +0 -56
- package/.github/workflows/unreleased-commits-nag.yml +0 -256
- package/eslint.config.js +0 -67
- package/pnpm-workspace.yaml +0 -2
- package/scripts/presign-transport-e2e.mjs +0 -250
- package/scripts/vault-rebaseline.sh +0 -323
- package/scripts/vault-rescue.sh +0 -332
- package/src/active-company.test.ts +0 -188
- package/src/active-company.ts +0 -168
- package/src/agent-codex-instructions.test.ts +0 -332
- package/src/agent-codex-instructions.ts +0 -309
- package/src/auth.ts +0 -146
- package/src/backup-prune.test.ts +0 -98
- package/src/backup-prune.ts +0 -182
- package/src/bin/backup-prune-runner.ts +0 -33
- package/src/bin/rescue-runner.ts +0 -25
- package/src/bin/sync-runner-company.ts +0 -695
- package/src/bin/sync-runner-events.test.ts +0 -143
- package/src/bin/sync-runner-events.ts +0 -55
- package/src/bin/sync-runner-planning.test.ts +0 -311
- package/src/bin/sync-runner-planning.ts +0 -258
- package/src/bin/sync-runner-rollup.test.ts +0 -37
- package/src/bin/sync-runner-rollup.ts +0 -97
- package/src/bin/sync-runner-telemetry.ts +0 -15
- package/src/bin/sync-runner-watch-loop.ts +0 -1235
- package/src/bin/sync-runner-watch-routes.test.ts +0 -71
- package/src/bin/sync-runner-watch-routes.ts +0 -184
- package/src/bin/sync-runner.test.ts +0 -8767
- package/src/bin/sync-runner.ts +0 -2190
- package/src/cli/accept.ts +0 -124
- package/src/cli/conflict.ts +0 -119
- package/src/cli/doctor.test.ts +0 -581
- package/src/cli/doctor.ts +0 -642
- package/src/cli/index.ts +0 -49
- package/src/cli/invite.test.ts +0 -250
- package/src/cli/invite.ts +0 -214
- package/src/cli/promote.ts +0 -157
- package/src/cli/reindex-knowledge.test.ts +0 -307
- package/src/cli/reindex-knowledge.ts +0 -450
- package/src/cli/reindex.test.ts +0 -957
- package/src/cli/reindex.ts +0 -979
- package/src/cli/rescue-classify-ordering.test.ts +0 -548
- package/src/cli/rescue-clone-diagnostics.test.ts +0 -120
- package/src/cli/rescue-core.ts +0 -3011
- package/src/cli/rescue-drift-reconcile.test.ts +0 -179
- package/src/cli/rescue-drop-dir-symlink.test.ts +0 -224
- package/src/cli/rescue-exec-bit-preserve.test.ts +0 -187
- package/src/cli/rescue-hq-root-guard.test.ts +0 -232
- package/src/cli/rescue-journal-reconcile.test.ts +0 -215
- package/src/cli/rescue-mtime-preserve.test.ts +0 -203
- package/src/cli/rescue-settings-reconcile.test.ts +0 -637
- package/src/cli/rescue-snapshot.test.ts +0 -57
- package/src/cli/rescue-snapshot.ts +0 -51
- package/src/cli/rescue.reindex.test.ts +0 -63
- package/src/cli/rescue.test.ts +0 -131
- package/src/cli/rescue.ts +0 -182
- package/src/cli/share.test.ts +0 -7843
- package/src/cli/share.ts +0 -3663
- package/src/cli/sync-scope.test.ts +0 -652
- package/src/cli/sync.test.ts +0 -5207
- package/src/cli/sync.ts +0 -3470
- package/src/cli/tombstones.ts +0 -106
- package/src/cli/watch-event-push-conflict.test.ts +0 -234
- package/src/client-info.test.ts +0 -214
- package/src/client-info.ts +0 -121
- package/src/cognito-auth.test.ts +0 -712
- package/src/cognito-auth.ts +0 -1422
- package/src/company-resolver.test.ts +0 -618
- package/src/company-resolver.ts +0 -521
- package/src/context.test.ts +0 -583
- package/src/context.ts +0 -378
- package/src/daemon-worker.ts +0 -26
- package/src/daemon.ts +0 -99
- package/src/entity-resolver.test.ts +0 -315
- package/src/entity-resolver.ts +0 -180
- package/src/ignore.test.ts +0 -466
- package/src/ignore.ts +0 -469
- package/src/index.ts +0 -439
- package/src/journal.test.ts +0 -968
- package/src/journal.ts +0 -765
- package/src/lib/cloud-authoritative.test.ts +0 -45
- package/src/lib/cloud-authoritative.ts +0 -59
- package/src/lib/conflict-file.ts +0 -86
- package/src/lib/conflict-index.ts +0 -289
- package/src/lib/conflict.test.ts +0 -348
- package/src/lib/describe-error.test.ts +0 -100
- package/src/lib/describe-error.ts +0 -58
- package/src/lib/exit-codes.ts +0 -24
- package/src/lib/machine-id.test.ts +0 -231
- package/src/lib/machine-id.ts +0 -175
- package/src/lib/net-errors.test.ts +0 -65
- package/src/lib/net-errors.ts +0 -86
- package/src/lib/readlink-safe.test.ts +0 -43
- package/src/lib/readlink-safe.ts +0 -29
- package/src/local-path-codec.test.ts +0 -138
- package/src/local-path-codec.ts +0 -161
- package/src/machine-auth.test.ts +0 -1323
- package/src/manifest-reconcile.test.ts +0 -1123
- package/src/manifest-reconcile.ts +0 -518
- package/src/object-io.test.ts +0 -1221
- package/src/object-io.ts +0 -1306
- package/src/operation-lock.test.ts +0 -484
- package/src/operation-lock.ts +0 -680
- package/src/outcome-telemetry.test.ts +0 -498
- package/src/outcome-telemetry.ts +0 -639
- package/src/personal-vault-exclusions.test.ts +0 -308
- package/src/personal-vault-exclusions.ts +0 -354
- package/src/personal-vault.test.ts +0 -756
- package/src/personal-vault.ts +0 -496
- package/src/prefix-coalesce.test.ts +0 -240
- package/src/prefix-coalesce.ts +0 -273
- package/src/public-surface.test.ts +0 -117
- package/src/qmd-reindex.test.ts +0 -877
- package/src/qmd-reindex.ts +0 -842
- package/src/read-only-state-dir.test.ts +0 -188
- package/src/remote-pull.test.ts +0 -1130
- package/src/remote-pull.ts +0 -618
- package/src/s3.symlink-materialize.test.ts +0 -492
- package/src/s3.test.ts +0 -1789
- package/src/s3.ts +0 -1532
- package/src/schemas/signal-types.test.ts +0 -82
- package/src/schemas/signal-types.ts +0 -38
- package/src/schemas/source-channels.test.ts +0 -82
- package/src/schemas/source-channels.ts +0 -53
- package/src/scope-shrink.test.ts +0 -633
- package/src/scope-shrink.ts +0 -481
- package/src/signals/get.test.ts +0 -310
- package/src/signals/get.ts +0 -75
- package/src/signals/internals.ts +0 -195
- package/src/signals/list.test.ts +0 -420
- package/src/signals/list.ts +0 -79
- package/src/signals/parse.ts +0 -8
- package/src/signals/types.ts +0 -91
- package/src/skill-telemetry.test.ts +0 -1825
- package/src/skill-telemetry.ts +0 -1439
- package/src/sources/get.test.ts +0 -293
- package/src/sources/get.ts +0 -66
- package/src/sources/internals.ts +0 -198
- package/src/sources/list.test.ts +0 -402
- package/src/sources/list.ts +0 -84
- package/src/sources/parse.ts +0 -43
- package/src/sources/types.ts +0 -84
- package/src/sync/event-sync.test.ts +0 -594
- package/src/sync/event-sync.ts +0 -545
- package/src/sync/feature-flags.test.ts +0 -378
- package/src/sync/feature-flags.ts +0 -62
- package/src/sync/index.ts +0 -76
- package/src/sync/lease-client.test.ts +0 -128
- package/src/sync/lease-client.ts +0 -207
- package/src/sync/logger.test.ts +0 -242
- package/src/sync/logger.ts +0 -79
- package/src/sync/metrics.test.ts +0 -462
- package/src/sync/metrics.ts +0 -213
- package/src/sync/pull-scope.ts +0 -265
- package/src/sync/push-event.test.ts +0 -266
- package/src/sync/push-event.ts +0 -224
- package/src/sync/push-receiver.test.ts +0 -566
- package/src/sync/push-receiver.ts +0 -1048
- package/src/sync/push-transport.ts +0 -231
- package/src/sync/realtime-rollout.test.ts +0 -86
- package/src/sync/realtime-rollout.ts +0 -262
- package/src/sync/state-store.test.ts +0 -194
- package/src/sync/state-store.ts +0 -727
- package/src/sync-core.ts +0 -58
- package/src/sync-progress.test.ts +0 -94
- package/src/sync-progress.ts +0 -140
- package/src/telemetry-events.test.ts +0 -88
- package/src/telemetry-events.ts +0 -205
- package/src/telemetry.test.ts +0 -1280
- package/src/telemetry.ts +0 -1109
- package/src/types.ts +0 -314
- package/src/vault-client.test.ts +0 -1380
- package/src/vault-client.ts +0 -1694
- package/src/version.ts +0 -24
- package/src/watch-roots.test.ts +0 -278
- package/src/watch-roots.ts +0 -162
- package/src/watcher-event-gate.test.ts +0 -212
- package/src/watcher.test.ts +0 -1079
- package/src/watcher.ts +0 -1741
- package/test/e2e/sync/cross-tenant-isolation.test.ts +0 -630
- package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +0 -124
- package/test/e2e/sync/transient-company-leg.test.ts +0 -384
- package/test/e2e/sync/windows-unreadable-link-leg.test.ts +0 -191
- package/test/e2e/watcher-real-chokidar.test.ts +0 -165
- package/test/e2e/watcher-recursive-backend.test.ts +0 -181
- package/test/e2e/watcher-scoped-coverage.test.ts +0 -381
- package/test/invite-flow.integration.test.ts +0 -244
- package/test/joiner-manifest-reconcile.integration.test.ts +0 -322
- package/test/share-sync.integration.test.ts +0 -213
- package/tsconfig.json +0 -19
- package/vitest.config.ts +0 -22
package/src/cli/sync.ts
DELETED
|
@@ -1,3470 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `hq sync` command — pull everything allowed from entity vault (VLT-5 US-002).
|
|
3
|
-
*
|
|
4
|
-
* Pulls all files the caller's STS session policy permits.
|
|
5
|
-
* Never auto-overwrites local changes — prompts on conflict.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import * as fs from "fs";
|
|
9
|
-
import * as path from "path";
|
|
10
|
-
import type {
|
|
11
|
-
EntityContext,
|
|
12
|
-
VaultServiceConfig,
|
|
13
|
-
SyncJournal,
|
|
14
|
-
} from "../types.js";
|
|
15
|
-
import { VaultAuthError, VaultClient, type SyncMode } from "../vault-client.js";
|
|
16
|
-
import { readlinkOrNull } from "../lib/readlink-safe.js";
|
|
17
|
-
import {
|
|
18
|
-
emitCloudTelemetry,
|
|
19
|
-
type TelemetryClaims,
|
|
20
|
-
} from "../telemetry-events.js";
|
|
21
|
-
import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
|
|
22
|
-
import { createSyncProgressRecorder } from "../sync-progress.js";
|
|
23
|
-
import {
|
|
24
|
-
canonicalVaultKeySpelling,
|
|
25
|
-
localPathForVaultKey,
|
|
26
|
-
vaultKeyForLocalPath,
|
|
27
|
-
} from "../local-path-codec.js";
|
|
28
|
-
import { SKILLS_KEY_PREFIX } from "./doctor.js";
|
|
29
|
-
import {
|
|
30
|
-
downloadFile,
|
|
31
|
-
listRemoteFiles,
|
|
32
|
-
headRemoteFile,
|
|
33
|
-
primeObjectTransport,
|
|
34
|
-
toPosixKey,
|
|
35
|
-
classifyVaultKey,
|
|
36
|
-
DanglingSymlinkParentError,
|
|
37
|
-
} from "../s3.js";
|
|
38
|
-
import type { DownloadModeWarning, RemoteFile } from "../s3.js";
|
|
39
|
-
import {
|
|
40
|
-
readJournal,
|
|
41
|
-
writeJournal,
|
|
42
|
-
hashFile,
|
|
43
|
-
hashSymlinkTarget,
|
|
44
|
-
updateEntry,
|
|
45
|
-
removeEntry,
|
|
46
|
-
getEntry,
|
|
47
|
-
normalizeEtag,
|
|
48
|
-
migrateToV2,
|
|
49
|
-
gcTombstones,
|
|
50
|
-
isTombstone,
|
|
51
|
-
tombstoneEntry,
|
|
52
|
-
lastPullRecord,
|
|
53
|
-
appendPullRecord,
|
|
54
|
-
generatePullId,
|
|
55
|
-
PERSONAL_VAULT_JOURNAL_SLUG,
|
|
56
|
-
migratePersonalVaultJournal,
|
|
57
|
-
} from "../journal.js";
|
|
58
|
-
import {
|
|
59
|
-
isGeneratedCoreMirrorKey,
|
|
60
|
-
PERSONAL_VAULT_MANIFEST_KEY,
|
|
61
|
-
} from "../personal-vault.js";
|
|
62
|
-
import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
|
|
63
|
-
import {
|
|
64
|
-
buildScopeShrinkPlan,
|
|
65
|
-
applyScopeShrink,
|
|
66
|
-
ScopeShrinkBlockedError,
|
|
67
|
-
ScopeShrinkLargePruneError,
|
|
68
|
-
type ScopeShrinkAdviceContext,
|
|
69
|
-
} from "../scope-shrink.js";
|
|
70
|
-
import {
|
|
71
|
-
coalescePrefixes,
|
|
72
|
-
isCoveredByAny,
|
|
73
|
-
type ScopePrefixInput,
|
|
74
|
-
} from "../prefix-coalesce.js";
|
|
75
|
-
import { createIgnoreFilter } from "../ignore.js";
|
|
76
|
-
import {
|
|
77
|
-
hasRemoteChanged,
|
|
78
|
-
isAccessDenied,
|
|
79
|
-
resolveActiveCompany,
|
|
80
|
-
resolveTransferConcurrency,
|
|
81
|
-
} from "../sync-core.js";
|
|
82
|
-
import { isEphemeralPath, isForbiddenCompanyVaultKey, isMalformedVaultKey } from "./share.js";
|
|
83
|
-
import { resolveConflict } from "./conflict.js";
|
|
84
|
-
import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
|
|
85
|
-
import {
|
|
86
|
-
buildConflictId,
|
|
87
|
-
buildConflictPath,
|
|
88
|
-
readShortMachineId,
|
|
89
|
-
} from "../lib/conflict-file.js";
|
|
90
|
-
import { appendConflictEntry } from "../lib/conflict-index.js";
|
|
91
|
-
import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
|
|
92
|
-
import { reindex } from "./reindex.js";
|
|
93
|
-
import { withOperationLock } from "../operation-lock.js";
|
|
94
|
-
import {
|
|
95
|
-
fetchCompanyTombstones,
|
|
96
|
-
type CompanyTombstone,
|
|
97
|
-
} from "./tombstones.js";
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Per-file events emitted by `sync()` as it progresses.
|
|
101
|
-
*
|
|
102
|
-
* When `SyncOptions.onEvent` is set, these events are delivered to the caller
|
|
103
|
-
* in place of the default human-readable `console.log` / `console.error`
|
|
104
|
-
* output. This is the seam that lets `hq-sync-runner` stream ndjson to the
|
|
105
|
-
* AppBar menubar without the engine knowing anything about ndjson (ADR-0001).
|
|
106
|
-
*
|
|
107
|
-
* The human CLI (`hq sync`) leaves `onEvent` undefined and falls through to
|
|
108
|
-
* `defaultConsoleLogger` below, which preserves the existing tty output.
|
|
109
|
-
*
|
|
110
|
-
* A single `plan` event is emitted once at the start of every run, before
|
|
111
|
-
* any `progress`/`conflict`/`error` events. It carries the totals derived
|
|
112
|
-
* from a Stage-1 classification pass so consumers can render an accurate
|
|
113
|
-
* progress denominator before transfers begin (the menubar's "Preparing
|
|
114
|
-
* sync…" pre-pass becomes obsolete once the runner forwards this).
|
|
115
|
-
*/
|
|
116
|
-
export type SyncProgressEvent =
|
|
117
|
-
| {
|
|
118
|
-
type: "plan";
|
|
119
|
-
/** Files this run intends to download (pull-only; 0 from share). */
|
|
120
|
-
filesToDownload: number;
|
|
121
|
-
bytesToDownload: number;
|
|
122
|
-
/** Files this run intends to upload (push-only; 0 from sync). */
|
|
123
|
-
filesToUpload: number;
|
|
124
|
-
bytesToUpload: number;
|
|
125
|
-
/** Files classified as no-op (ignored, unchanged, local-only on pull). */
|
|
126
|
-
filesToSkip: number;
|
|
127
|
-
/**
|
|
128
|
-
* Files known up-front to be conflicts. Pull-side fills this from the
|
|
129
|
-
* 3-way merge against the journal; push-side leaves it 0 because
|
|
130
|
-
* conflict detection requires a remote HEAD that runs in Stage 2.
|
|
131
|
-
*/
|
|
132
|
-
filesToConflict: number;
|
|
133
|
-
/**
|
|
134
|
-
* Remote keys this run intends to delete from S3 (push-only with
|
|
135
|
-
* `propagateDeletes`; 0 from sync). A key is scheduled for deletion when
|
|
136
|
-
* its journal entry exists, the local file is gone, and the key falls
|
|
137
|
-
* within the share scope. The bucket has versioning enabled so the
|
|
138
|
-
* delete is soft (a delete-marker is written; prior versions remain
|
|
139
|
-
* recoverable).
|
|
140
|
-
*/
|
|
141
|
-
filesToDelete: number;
|
|
142
|
-
}
|
|
143
|
-
| {
|
|
144
|
-
type: "progress";
|
|
145
|
-
path: string;
|
|
146
|
-
bytes: number;
|
|
147
|
-
message?: string;
|
|
148
|
-
/** True when this event reports a remote DeleteObject (no upload). */
|
|
149
|
-
deleted?: boolean;
|
|
150
|
-
/**
|
|
151
|
-
* Transfer direction for this file, stamped by the runner's per-company
|
|
152
|
-
* tagger from the in-flight phase: `"up"` for a push (local→S3 upload),
|
|
153
|
-
* `"down"` for a pull (S3→local download). Optional because the inner
|
|
154
|
-
* `share()`/`sync()` emitters don't know the run-level phase — only the
|
|
155
|
-
* runner's `tagAndEmit` does, and it sets this when re-emitting to stdout.
|
|
156
|
-
* Consumers (the menubar activity log) use it to label each file
|
|
157
|
-
* uploaded vs downloaded.
|
|
158
|
-
*/
|
|
159
|
-
direction?: "up" | "down";
|
|
160
|
-
/**
|
|
161
|
-
* Email of the file's author, read from the S3 object's `created-by`
|
|
162
|
-
* user-metadata. Only set on the download/pull path (a downloaded file
|
|
163
|
-
* was authored by whoever uploaded it); push-side progress has no author
|
|
164
|
-
* because the uploader is the local user. `null`/absent when the object
|
|
165
|
-
* carries no `created-by`. The menubar activity log shows this so the
|
|
166
|
-
* user sees who authored each file they received.
|
|
167
|
-
*/
|
|
168
|
-
author?: string | null;
|
|
169
|
-
}
|
|
170
|
-
| { type: "error"; path: string; message: string }
|
|
171
|
-
| {
|
|
172
|
-
type: "conflict";
|
|
173
|
-
path: string;
|
|
174
|
-
direction: "pull" | "push";
|
|
175
|
-
resolution: ConflictResolution;
|
|
176
|
-
}
|
|
177
|
-
| {
|
|
178
|
-
/**
|
|
179
|
-
* Emitted when a planner-flagged pull conflict turned out to be a false
|
|
180
|
-
* positive: the remote bytes fetched for the conflict mirror hashed
|
|
181
|
-
* byte-for-byte identical to the local file, so there is nothing to
|
|
182
|
-
* resolve. The planner can only see journal-relative deltas (local hash
|
|
183
|
-
* != journal hash AND remote etag != journal etag), so a stale journal
|
|
184
|
-
* baseline makes identical content look like a both-sides change. Known
|
|
185
|
-
* stale-baseline triggers: a shared-journal cross-root collision (the
|
|
186
|
-
* personal vault + `companies/personal` sharing one journal), an
|
|
187
|
-
* mtime-rounding fast-path miss, KMS/multipart etag churn on a no-op
|
|
188
|
-
* re-upload, a second machine advancing S3 + its own journal, or a
|
|
189
|
-
* manual revert. The executor re-stamps the journal baseline (so neither
|
|
190
|
-
* side looks changed next run) and skips — no conflict counted, no
|
|
191
|
-
* `.conflict-*` mirror written. Counted as a skip; surfaced here purely
|
|
192
|
-
* for observability so a UI can explain why N "conflicts" silently
|
|
193
|
-
* resolved themselves.
|
|
194
|
-
*/
|
|
195
|
-
type: "reconciled";
|
|
196
|
-
path: string;
|
|
197
|
-
direction: "pull" | "push";
|
|
198
|
-
}
|
|
199
|
-
| {
|
|
200
|
-
type: "new-files";
|
|
201
|
-
files: Array<{ path: string; bytes: number; addedBy: string | null }>;
|
|
202
|
-
}
|
|
203
|
-
| {
|
|
204
|
-
/**
|
|
205
|
-
* Emitted by the `currency-gated` delete policy when a delete candidate
|
|
206
|
-
* is refused. Two reasons, discriminated by `reason`:
|
|
207
|
-
*
|
|
208
|
-
* - `"stale-etag"`: the local file is missing but the remote object's
|
|
209
|
-
* current ETag no longer matches the journal's last-recorded etag.
|
|
210
|
-
* Some other device (or another out-of-band write) modified the
|
|
211
|
-
* remote object since this machine last synced it. The pull leg of
|
|
212
|
-
* `sync now` will re-pull naturally via the same `hasRemoteChanged`
|
|
213
|
-
* path that powers conflict detection. Journal entry is left intact
|
|
214
|
-
* so the pull can use it as the baseline for the 3-way merge.
|
|
215
|
-
* `journalEtag` and `remoteEtag` are real ETag strings.
|
|
216
|
-
*
|
|
217
|
-
* - `"legacy-no-etag"`: the journal entry predates remoteEtag tracking
|
|
218
|
-
* (no `remoteEtag` recorded). We can't prove currency without an
|
|
219
|
-
* etag, so the delete is refused in the safe direction. A future
|
|
220
|
-
* sync that picks up an etag for this entry can re-evaluate.
|
|
221
|
-
* `journalEtag` and `remoteEtag` are sentinel strings
|
|
222
|
-
* (`<legacy-no-etag>` / `<unknown>`) — do not render as ETags.
|
|
223
|
-
* Consumers should branch on `reason`, not on the etag values.
|
|
224
|
-
*
|
|
225
|
-
* - `"bulk-asymmetry"`: the bulk-asymmetry circuit-breaker tripped
|
|
226
|
-
* (see `delete-refused-bulk-asymmetry`). Each candidate that would
|
|
227
|
-
* have been deleted also surfaces here for path-level visibility.
|
|
228
|
-
* `journalEtag` and `remoteEtag` are sentinel strings
|
|
229
|
-
* (`<bulk-asymmetry>` / `<not-checked>`) since no HEAD was issued.
|
|
230
|
-
*/
|
|
231
|
-
type: "delete-refused-stale-etag";
|
|
232
|
-
path: string;
|
|
233
|
-
journalEtag: string;
|
|
234
|
-
remoteEtag: string;
|
|
235
|
-
reason:
|
|
236
|
-
| "stale-etag"
|
|
237
|
-
| "legacy-no-etag"
|
|
238
|
-
| "bulk-asymmetry"
|
|
239
|
-
| "divergent-local"
|
|
240
|
-
| "missing-delete-intent"
|
|
241
|
-
| "intent-changed"
|
|
242
|
-
| "recreated-locally";
|
|
243
|
-
}
|
|
244
|
-
| {
|
|
245
|
-
/**
|
|
246
|
-
* Emitted at most ONCE per `share()` call when the bulk-asymmetry
|
|
247
|
-
* circuit-breaker tripped: a large fraction of the in-scope journal
|
|
248
|
-
* entries are missing locally, suggesting the local mirror is corrupt
|
|
249
|
-
* (moved hqRoot, partial restore, fresh clone over inherited state,
|
|
250
|
-
* unmounted volume, accidental `rm -rf`) rather than a deliberate
|
|
251
|
-
* delete. The engine refuses to convert the missing entries into
|
|
252
|
-
* remote `DeleteObject` calls; every refused candidate also surfaces
|
|
253
|
-
* via `delete-refused-stale-etag` with `reason: "bulk-asymmetry"`.
|
|
254
|
-
*
|
|
255
|
-
* Trip condition: `candidates >= 10 AND candidates / inScope >= 0.10`.
|
|
256
|
-
* Bypass: set `HQ_SYNC_DELETE_BULK_OVERRIDE=1` (truthy: `1|true|yes`,
|
|
257
|
-
* case-insensitive) OR pass `propagateDeletePolicy: "all"` (the
|
|
258
|
-
* existing emergency-reconcile policy which already opts out of
|
|
259
|
-
* safety gates).
|
|
260
|
-
*
|
|
261
|
-
* `candidates` is the number of journal entries whose local file is
|
|
262
|
-
* missing AND would have been delete-candidates absent the guard;
|
|
263
|
-
* `inScope` is the total journal entries under the share's scope
|
|
264
|
-
* roots; `ratio = candidates / inScope`; `samplePaths` carries up to
|
|
265
|
-
* 10 refused keys for diagnostic display.
|
|
266
|
-
*/
|
|
267
|
-
type: "delete-refused-bulk-asymmetry";
|
|
268
|
-
candidates: number;
|
|
269
|
-
inScope: number;
|
|
270
|
-
ratio: number;
|
|
271
|
-
samplePaths: string[];
|
|
272
|
-
}
|
|
273
|
-
| {
|
|
274
|
-
/**
|
|
275
|
-
* Emitted at most ONCE per `share()` call (push leg of a sync run) when
|
|
276
|
-
* `personalMode === true` and the personal-vault default-exclusion list
|
|
277
|
-
* blocked one or more files that would otherwise have uploaded. Gives
|
|
278
|
-
* the UI a single summary signal — "N files quietly excluded by default
|
|
279
|
-
* policy" — without firing one event per excluded file (which would
|
|
280
|
-
* dominate the event stream on first-sync of a dirty tree).
|
|
281
|
-
*
|
|
282
|
-
* `count` is the total number of paths the exclusion filter rejected
|
|
283
|
-
* (deduplicated across the walk). `samplePaths` carries up to 10
|
|
284
|
-
* forward-slash-separated relative paths for diagnostic display. `byId`
|
|
285
|
-
* is a per-exclusion-rule breakdown so the UI can render which class
|
|
286
|
-
* of exclusion did the work (secret / machine-local / scratch / …).
|
|
287
|
-
*
|
|
288
|
-
* Not emitted when `count === 0` — silent on a clean tree.
|
|
289
|
-
*/
|
|
290
|
-
type: "personal-vault-out-of-policy";
|
|
291
|
-
count: number;
|
|
292
|
-
samplePaths: string[];
|
|
293
|
-
byId: Record<string, number>;
|
|
294
|
-
}
|
|
295
|
-
| {
|
|
296
|
-
/**
|
|
297
|
-
* Emitted at most ONCE per `share()` call (push leg) when the run is
|
|
298
|
-
* scoped to a member/guest's ACL prefixes (`prefixSet`) and one or more
|
|
299
|
-
* candidate paths fell OUTSIDE the granted prefixes. Those paths are
|
|
300
|
-
* skipped from the upload (and delete) plan instead of being PUT — the
|
|
301
|
-
* vended child credential is scoped to the granted prefixes, so pushing
|
|
302
|
-
* them would draw the server's correct 403 `SCOPE_EXCEEDS_PARENT` and
|
|
303
|
-
* (pre-fix) abort the WHOLE company. This is the push-side analogue of
|
|
304
|
-
* the pull-side `skip-out-of-scope` action.
|
|
305
|
-
*
|
|
306
|
-
* `count` is the number of distinct company-relative paths/prefixes the
|
|
307
|
-
* scope filter excluded; `samplePaths` carries up to 10 for diagnostic
|
|
308
|
-
* display. Informational, NOT an error — the company still syncs its
|
|
309
|
-
* in-scope subset and the run exits 0.
|
|
310
|
-
*
|
|
311
|
-
* Not emitted when `count === 0` — silent on a fully in-scope tree.
|
|
312
|
-
*/
|
|
313
|
-
type: "scope-excluded";
|
|
314
|
-
count: number;
|
|
315
|
-
samplePaths: string[];
|
|
316
|
-
}
|
|
317
|
-
| {
|
|
318
|
-
/**
|
|
319
|
-
* Emitted at most ONCE per PULL leg when the leg ran under a
|
|
320
|
-
* membership-scoped `syncMode` (`"shared"` or `"custom"` — never `"all"`)
|
|
321
|
-
* AND one or more remote keys were withheld BY THAT MEMBERSHIP SCOPE
|
|
322
|
-
* (`skip-out-of-scope` with `reason: "membership-scope"`). This turns the
|
|
323
|
-
* silent shared-vs-all gap into a
|
|
324
|
-
* VISIBLE, actionable surface. The reporter of feedback_d2082110 lost
|
|
325
|
-
* files across devices precisely because four memberships defaulted to
|
|
326
|
-
* `shared` rather than `all`, so not all vault content materialized on the
|
|
327
|
-
* second device — and nothing told them; they had to notice the gap and
|
|
328
|
-
* flip each membership to `all` by hand. This event names the gap and the
|
|
329
|
-
* lever (raise the membership's access level to `all`) so FULL
|
|
330
|
-
* materialization is a deliberate, visible choice rather than a silent
|
|
331
|
-
* omission.
|
|
332
|
-
*
|
|
333
|
-
* `count` is the number of remote keys the MEMBERSHIP SCOPE withheld on
|
|
334
|
-
* this leg; `samplePaths` carries up to 10 company-relative keys for
|
|
335
|
-
* display; `syncMode` is the active scoped mode. Push-only keys
|
|
336
|
-
* (`sessions/`, US-006) are deliberately EXCLUDED from both — they are
|
|
337
|
-
* withheld in every mode including `all`, so the lever this event names
|
|
338
|
-
* would not materialize them. `count` is therefore ≤
|
|
339
|
-
* `SyncResult.filesOutOfScope`, which keeps its both-causes meaning.
|
|
340
|
-
* Distinct from the push-side `scope-excluded` (which reports what a
|
|
341
|
-
* grantee could not PUSH). Not emitted in `all` mode or when nothing was
|
|
342
|
-
* withheld by the membership scope — no gap, no noise.
|
|
343
|
-
*/
|
|
344
|
-
type: "scope-materialization-gap";
|
|
345
|
-
count: number;
|
|
346
|
-
samplePaths: string[];
|
|
347
|
-
syncMode: SyncMode;
|
|
348
|
-
}
|
|
349
|
-
| {
|
|
350
|
-
/**
|
|
351
|
-
* Emitted at most ONCE per `share()` push leg when the base ignore
|
|
352
|
-
* filter dropped one or more NOTEWORTHY paths: content that does not
|
|
353
|
-
* match expected build/VCS/cache noise classes. Surfaces an over-broad
|
|
354
|
-
* exclusion that would otherwise drop content silently (DEV-1791).
|
|
355
|
-
*
|
|
356
|
-
* `count` is the number of distinct noteworthy paths; `totalExcluded`
|
|
357
|
-
* is all base-ignore rejections including expected noise; `samplePaths`
|
|
358
|
-
* carries up to 10 noteworthy paths. Not emitted when `count === 0`.
|
|
359
|
-
*/
|
|
360
|
-
type: "ignore-excluded";
|
|
361
|
-
count: number;
|
|
362
|
-
totalExcluded: number;
|
|
363
|
-
samplePaths: string[];
|
|
364
|
-
}
|
|
365
|
-
| {
|
|
366
|
-
/**
|
|
367
|
-
* Emitted by the PUSH leg (`share()`) once per key whose upload was
|
|
368
|
-
* suppressed because an authoritative FILE_TOMBSTONE marks it deleted and
|
|
369
|
-
* the local copy is still the deleted baseline (journal hash unchanged).
|
|
370
|
-
* Without this, a behind peer that still holds the file would re-upload it,
|
|
371
|
-
* and the pull planner's timestamp-only re-create heuristic
|
|
372
|
-
* (`isRemoteRecreateAfterTombstone`) would treat the re-upload as a genuine
|
|
373
|
-
* re-create and resurrect the key for everyone. The deleter's own stale
|
|
374
|
-
* local copy is cleaned by the pull leg's `tombstone-delete`; this event
|
|
375
|
-
* records that the push refused to resurrect it. `deletedAt` is the
|
|
376
|
-
* tombstone's delete time.
|
|
377
|
-
*/
|
|
378
|
-
type: "upload-suppressed-tombstone";
|
|
379
|
-
path: string;
|
|
380
|
-
deletedAt: string;
|
|
381
|
-
}
|
|
382
|
-
| {
|
|
383
|
-
/**
|
|
384
|
-
* Emitted by the PUSH leg (`share()`) once per local file skipped because
|
|
385
|
-
* it exceeds the max sync size (`isWithinSizeLimit`, 50 MB default). This
|
|
386
|
-
* is a permanent, benign, user-policy outcome — the file is too big to
|
|
387
|
-
* sync and will be over the cap on every pass — so it is NOT an error and
|
|
388
|
-
* the run exits 0. It was previously emitted as `type: "error"`, which
|
|
389
|
-
* pushed it into the runner's `errors[]` and made every watch pass return
|
|
390
|
-
* exit 2; the menubar supervisor then reported that as
|
|
391
|
-
* "auto-sync watcher exited unexpectedly (code=Some(2))" on every tick —
|
|
392
|
-
* the HQ-SYNC-4 flood across user machines. Surfaced here purely for visibility
|
|
393
|
-
* (`bytes` is the offending file's size) so a UI/CLI can tell the user
|
|
394
|
-
* which file was too big without treating it as a failure.
|
|
395
|
-
*/
|
|
396
|
-
type: "skip-size-limit";
|
|
397
|
-
path: string;
|
|
398
|
-
bytes: number;
|
|
399
|
-
}
|
|
400
|
-
| {
|
|
401
|
-
/**
|
|
402
|
-
* Emitted once per key skipped because it is permanently unstorable.
|
|
403
|
-
*
|
|
404
|
-
* PULL leg: a remote key the server will never presign (for example, a
|
|
405
|
-
* `companies/…`-prefixed key in a company-scoped vault or a key with
|
|
406
|
-
* control characters) — bucket poisoning from an outdated direct-S3
|
|
407
|
-
* client. PUSH leg: a local key the upload validator would reject, so
|
|
408
|
-
* the upload can only ever throw — a doubled-tree `companies/…` key, or
|
|
409
|
-
* a control-character path such as a macOS Finder `Icon\r` file. Like
|
|
410
|
-
* `skip-size-limit`, this is deliberately NOT `type: "error"` — one bad
|
|
411
|
-
* object must never error a whole company sync (erroring gated
|
|
412
|
-
* heartbeat toolset refreshes fleet-wide in the incident). Surfaced for
|
|
413
|
-
* visibility + cloud telemetry; per policy
|
|
414
|
-
* hq-alert-baseline-calibration the healthy baseline is ZERO such keys,
|
|
415
|
-
* so any occurrence is signal.
|
|
416
|
-
*/
|
|
417
|
-
type: "skip-invalid-scoped-key";
|
|
418
|
-
path: string;
|
|
419
|
-
/** Server-compatible invalid-key code, such as INVALID_KEY_CONTROL_CHARS. */
|
|
420
|
-
errorCode?: string;
|
|
421
|
-
}
|
|
422
|
-
| {
|
|
423
|
-
/**
|
|
424
|
-
* Emitted by the PULL leg once per remote key skipped because its
|
|
425
|
-
* spelling percent-decodes to the same logical path as an
|
|
426
|
-
* already-journaled key with a DIFFERENT spelling — the mixed-version
|
|
427
|
-
* amplifier (a pre-codec peer minting `%3A`/`%253A` twins of a
|
|
428
|
-
* canonical `:` key). Downloading the twin would materialize a junk
|
|
429
|
-
* local dir and fork the journal into a second key family for one
|
|
430
|
-
* logical path, so the established spelling wins and the twin is
|
|
431
|
-
* skipped. Like `skip-size-limit`, this is deliberately NOT
|
|
432
|
-
* `type: "error"` (policy hq-sync-deliberate-skip-not-fatal-error-exit2:
|
|
433
|
-
* a recurring benign per-file skip emitted as an error lands in the
|
|
434
|
-
* runner's `errors[]`, forces exit 2 on every pass, and the menubar
|
|
435
|
-
* Sentry-alerts each time). The US-003 vault doctor is the cleanup
|
|
436
|
-
* path that collapses the junk family itself.
|
|
437
|
-
*/
|
|
438
|
-
type: "skip-junk-key-spelling";
|
|
439
|
-
path: string;
|
|
440
|
-
/** The already-journaled spelling that owns this logical path. */
|
|
441
|
-
journaledKey: string;
|
|
442
|
-
}
|
|
443
|
-
| {
|
|
444
|
-
/**
|
|
445
|
-
* Emitted when local content is deliberately not transferred and would
|
|
446
|
-
* otherwise vanish from sync output without a trace (feedback_258e4a86 /
|
|
447
|
-
* feedback_a51cb63d — an hour lost because a "Pushed 0 file(s)" success
|
|
448
|
-
* named nothing that was skipped). Push batches by reason; pull emits an
|
|
449
|
-
* unreadable link by its remote key. `reason` distinguishes the cause:
|
|
450
|
-
*
|
|
451
|
-
* - `"unreachable-path"`: a path the caller EXPLICITLY named exists on
|
|
452
|
-
* disk but the resolver could not place it under the company folder,
|
|
453
|
-
* or could not find it under any base. This is actionable signal — the
|
|
454
|
-
* named target was NOT shipped — so the CLI may treat it as an error.
|
|
455
|
-
* - `"linked-subtree"`: a directory symlink whose target lives OUTSIDE
|
|
456
|
-
* the company folder was recorded as a link but its contents were not
|
|
457
|
-
* descended/uploaded. They sync via their own repo, not the vault;
|
|
458
|
-
* this is informational, not an error.
|
|
459
|
-
* - `"unreadable-link"`: the OS identified a local symbolic link but
|
|
460
|
-
* did not expose a readable target. The link is skipped rather than
|
|
461
|
-
* dereferenced, and the company leg remains complete.
|
|
462
|
-
*
|
|
463
|
-
* `count` is the number of distinct paths for that reason; `samplePaths`
|
|
464
|
-
* carries up to 10 for display. Not emitted when `count === 0`.
|
|
465
|
-
*/
|
|
466
|
-
type: "not-shipped";
|
|
467
|
-
reason: "unreachable-path" | "unreadable-link" | "linked-subtree";
|
|
468
|
-
count: number;
|
|
469
|
-
samplePaths: string[];
|
|
470
|
-
};
|
|
471
|
-
|
|
472
|
-
export interface SyncOptions {
|
|
473
|
-
/** Company slug or UID (defaults to active company from config) */
|
|
474
|
-
company?: string;
|
|
475
|
-
/** Non-interactive conflict strategy */
|
|
476
|
-
onConflict?: ConflictStrategy;
|
|
477
|
-
/** Vault service config */
|
|
478
|
-
vaultConfig: VaultServiceConfig;
|
|
479
|
-
/** HQ root directory */
|
|
480
|
-
hqRoot: string;
|
|
481
|
-
/**
|
|
482
|
-
* Per-file event callback. When present, suppresses the default
|
|
483
|
-
* `console.log`/`console.error` human output — the caller is expected to
|
|
484
|
-
* render events themselves (e.g. emit ndjson to stdout). When absent, the
|
|
485
|
-
* default human logger is used. See `SyncProgressEvent`.
|
|
486
|
-
*/
|
|
487
|
-
onEvent?: (event: SyncProgressEvent) => void;
|
|
488
|
-
/**
|
|
489
|
-
* When true, the caller is syncing against the caller's person-entity
|
|
490
|
-
* bucket. Pulled keys whose path starts with `companies/` are local
|
|
491
|
-
* (non-cloud) companies that sync to the personal vault by default —
|
|
492
|
-
* they are allowed through EXCEPT for slugs in `teamSyncedSlugs` (which
|
|
493
|
-
* are cloud-backed orphans, see below). `companies/manifest.yaml` is
|
|
494
|
-
* always allowed (routing source-of-truth).
|
|
495
|
-
*/
|
|
496
|
-
personalMode?: boolean;
|
|
497
|
-
/**
|
|
498
|
-
* Slugs of companies the operator has an active team-bucket Membership
|
|
499
|
-
* for. Only consulted when `personalMode === true`: keys under
|
|
500
|
-
* `companies/{slug}/...` for any slug in this set are dropped as orphans
|
|
501
|
-
* from a pre-promotion personal-bucket fallback (the company became
|
|
502
|
-
* cloud-true). The push-side decommission cycle removes these from the
|
|
503
|
-
* bucket; this filter prevents them from re-downloading into the same
|
|
504
|
-
* disk paths the team-bucket pull manages.
|
|
505
|
-
*/
|
|
506
|
-
teamSyncedSlugs?: ReadonlySet<string>;
|
|
507
|
-
/**
|
|
508
|
-
* Override for the per-slug journal file name. Defaults to `ctx.slug`.
|
|
509
|
-
* sync-runner passes `journalSlug: "personal"` for the personal slot so
|
|
510
|
-
* TS runner and Rust first-push share idempotency state.
|
|
511
|
-
*/
|
|
512
|
-
journalSlug?: string;
|
|
513
|
-
/**
|
|
514
|
-
* Effective sync mode for this leg (US-005 wiring). Defaults to `"all"`
|
|
515
|
-
* when absent, preserving the legacy full-bucket pull. The runner resolves
|
|
516
|
-
* this from the membership's sync-config (`getMembershipSyncConfig`).
|
|
517
|
-
*
|
|
518
|
-
* SECURITY NOTE: this is a footprint/UX filter, NOT an authorization
|
|
519
|
-
* boundary. The security boundary is the server (STS credential scope +
|
|
520
|
-
* ACL). An owner's STS is wide (role-bypass), so this client-side scope is
|
|
521
|
-
* what makes selective download durable for owners — but it never grants
|
|
522
|
-
* access beyond what STS already permits.
|
|
523
|
-
*/
|
|
524
|
-
syncMode?: SyncMode;
|
|
525
|
-
/**
|
|
526
|
-
* Coalesced, COMPANY-RELATIVE prefixes the current pull is scoped to when
|
|
527
|
-
* `syncMode` is `"shared"` or `"custom"` (same namespace as `RemoteFile.key`
|
|
528
|
-
* and the per-slug journal keys — e.g. `"knowledge/"`, `"projects/x/"`).
|
|
529
|
-
* Ignored when `syncMode` is `"all"`. The runner derives this from the
|
|
530
|
-
* caller's explicit grants (`shared`) or `customPaths` (`custom`) and is
|
|
531
|
-
* responsible for normalizing into the company-relative namespace.
|
|
532
|
-
*
|
|
533
|
-
* A `shared` leg with an empty/undefined `prefixSet` means "nothing is
|
|
534
|
-
* shared with me" → download nothing. The runner MUST fall back to `"all"`
|
|
535
|
-
* (not empty `"shared"`) on any grant-resolution error, so a transient
|
|
536
|
-
* failure can never silently prune the local tree.
|
|
537
|
-
*/
|
|
538
|
-
prefixSet?: ScopePrefixInput[];
|
|
539
|
-
/**
|
|
540
|
-
* Company-relative prefixes SUBTRACTED from the effective pull scope in EVERY
|
|
541
|
-
* mode — the push-only set (US-006). Carries `sessions/`: session transcripts
|
|
542
|
-
* are pushed into the vault but never auto-pulled onto grantee laptops, even
|
|
543
|
-
* under `syncMode: "all"` (which otherwise pulls the whole bucket). A remote
|
|
544
|
-
* key downloads iff it is covered by the inclusion scope (`prefixSet`, or all
|
|
545
|
-
* of `all` mode) AND NOT covered by any `excludePrefixes` entry; excluded
|
|
546
|
-
* keys are classified `skip-out-of-scope`.
|
|
547
|
-
*
|
|
548
|
-
* CRITICAL: this set is applied ONLY to the download filter. It is
|
|
549
|
-
* deliberately NOT fed into the scope-shrink pass (which keeps using the
|
|
550
|
-
* inclusion `prefixSet` + pins), so a locally-authored (`direction:"up"`) or
|
|
551
|
-
* pinned on-demand-materialized session is NEVER orphaned/pruned — it is
|
|
552
|
-
* merely not re-pulled. The runner derives this from
|
|
553
|
-
* `resolvePullScope().excludePrefixes`.
|
|
554
|
-
*/
|
|
555
|
-
excludePrefixes?: ScopePrefixInput[];
|
|
556
|
-
/**
|
|
557
|
-
* When the effective scope shrinks relative to the last pull and the shrink
|
|
558
|
-
* would orphan locally-modified ("dirty") files, `sync()` aborts with a
|
|
559
|
-
* `ScopeShrinkBlockedError` by default. Set `true` to proceed anyway:
|
|
560
|
-
* dirty files are LEFT ON DISK and only their journal entries are
|
|
561
|
-
* tombstoned. Mirrors `hq sync narrow --force`.
|
|
562
|
-
*/
|
|
563
|
-
forceScopeShrink?: boolean;
|
|
564
|
-
/**
|
|
565
|
-
* How `sync()` handles a scope shrink (US-005 / DEV-1768):
|
|
566
|
-
*
|
|
567
|
-
* - `"block"` (default) — a human is present (foreground `hq sync`). Dirty
|
|
568
|
-
* out-of-scope orphans, or a clean prune over the safety cap, raise a
|
|
569
|
-
* structured error whose advice is followable from a terminal. Clean
|
|
570
|
-
* orphans within the cap are QUARANTINED (moved, not deleted).
|
|
571
|
-
* - `"auto-recover"` — the background menubar runner, which can take no
|
|
572
|
-
* interactive flag. NEVER throws on a shrink: dirty orphans are kept on
|
|
573
|
-
* disk + un-tracked, clean orphans are quarantined, and the bulk-prune
|
|
574
|
-
* cap is bypassed (quarantine is non-destructive). This is what clears an
|
|
575
|
-
* already-wedged journal on the next sync, idempotently and without data
|
|
576
|
-
* loss — the recovery seam for the all→shared seed bug.
|
|
577
|
-
*
|
|
578
|
-
* Both policies are non-destructive for CLEAN files (quarantine, never
|
|
579
|
-
* silent delete) — the deliberate `hq sync narrow --apply` ritual is the only
|
|
580
|
-
* path that hard-deletes, and it confirms first.
|
|
581
|
-
*/
|
|
582
|
-
scopeShrinkPolicy?: "block" | "auto-recover";
|
|
583
|
-
/**
|
|
584
|
-
* The caller's own Cognito `sub`, used by the scope-shrink authorship guard
|
|
585
|
-
* so a scope shrink never prunes content the caller authored. Injected by the
|
|
586
|
-
* entry point — the runner sources it from its decoded idToken claims (the
|
|
587
|
-
* same sub stamped onto uploads as `created-by-sub`). The engine never reads
|
|
588
|
-
* it from disk, so it stays pure/hermetic; undefined degrades safely.
|
|
589
|
-
*/
|
|
590
|
-
callerSub?: string;
|
|
591
|
-
/**
|
|
592
|
-
* Optional decoded Cognito claims used only for action telemetry attribution
|
|
593
|
-
* (agentUid when `custom:entityType=agent`). Sync behavior never branches on
|
|
594
|
-
* this field.
|
|
595
|
-
*/
|
|
596
|
-
telemetryClaims?: TelemetryClaims | null;
|
|
597
|
-
/**
|
|
598
|
-
* Skip the post-sync `reindex()` refresh (skill wrappers + personal overlay
|
|
599
|
-
* mirrors + workers registry). By default, when a sync changes on-disk
|
|
600
|
-
* sources (downloads, tombstones, or scope-orphan removals), `sync()`
|
|
601
|
-
* re-runs reindex so the generated `.claude/skills/<ns>:<skill>` wrappers
|
|
602
|
-
* stay in sync. An orchestrator syncing many companies in one pass can set
|
|
603
|
-
* this and run `reindex()` once itself instead of per-company.
|
|
604
|
-
*/
|
|
605
|
-
skipReindex?: boolean;
|
|
606
|
-
/**
|
|
607
|
-
* Internal runner seam: true only when the caller already holds the
|
|
608
|
-
* per-root operation lock for this sync pass.
|
|
609
|
-
*/
|
|
610
|
-
operationLockAlreadyHeld?: boolean;
|
|
611
|
-
}
|
|
612
|
-
|
|
613
|
-
export interface SyncResult {
|
|
614
|
-
filesDownloaded: number;
|
|
615
|
-
bytesDownloaded: number;
|
|
616
|
-
filesSkipped: number;
|
|
617
|
-
conflicts: number;
|
|
618
|
-
/**
|
|
619
|
-
* Paths (remote keys) that were detected as conflicts during this run.
|
|
620
|
-
* Always populated when `conflicts > 0` so callers can surface them in UI
|
|
621
|
-
* or logs without re-streaming the per-file events.
|
|
622
|
-
*/
|
|
623
|
-
conflictPaths: string[];
|
|
624
|
-
aborted: boolean;
|
|
625
|
-
/**
|
|
626
|
-
* Files classified as "new" during pull — i.e. the remote file had no
|
|
627
|
-
* local counterpart at classification time. Additive field; empty array
|
|
628
|
-
* when no new files were detected or on push-only syncs.
|
|
629
|
-
*/
|
|
630
|
-
newFiles: Array<{ path: string; bytes: number }>;
|
|
631
|
-
/** Convenience count: `newFiles.length`. */
|
|
632
|
-
newFilesCount: number;
|
|
633
|
-
/**
|
|
634
|
-
* Count of remote keys refused at planning time because they matched
|
|
635
|
-
* `EPHEMERAL_PATH_PATTERN` (conflict-mirror files that must never round-
|
|
636
|
-
* trip through the bucket). Mirrors `ShareResult.filesExcludedByPolicy`
|
|
637
|
-
* so push and pull report the same shape. Pre-fix this count was always
|
|
638
|
-
* 0 on the pull side and legacy `.conflict-*` litter rode every sync —
|
|
639
|
-
* see Bug #2 in workspace/reports/hq-cloud-5.33.0-deep-test.md.
|
|
640
|
-
*/
|
|
641
|
-
filesExcludedByPolicy: number;
|
|
642
|
-
/**
|
|
643
|
-
* Count of journal-known keys applied as local deletes during this pull
|
|
644
|
-
* because the remote LIST no longer contains them — the cross-machine
|
|
645
|
-
* delete-propagation signal that Bug #9 closes. The peer's push leg
|
|
646
|
-
* removed the object from S3 (`hq sync` push side verified-to-work in
|
|
647
|
-
* the deep-test addendum), but pre-fix the pull side never enumerated
|
|
648
|
-
* "what's missing-from-remote-that-was-there-before", so the file
|
|
649
|
-
* lingered locally forever. Always 0 when no journal-known keys have
|
|
650
|
-
* disappeared from the remote.
|
|
651
|
-
*/
|
|
652
|
-
filesTombstoned: number;
|
|
653
|
-
/**
|
|
654
|
-
* Count of remote keys NOT downloaded this run because they fall outside
|
|
655
|
-
* the effective `syncMode` scope (US-005). Always 0 in `all` mode. Distinct
|
|
656
|
-
* from `filesSkipped` (which measures "unchanged on this run") so consumers
|
|
657
|
-
* can render a "N outside your sync scope" line. The matching local cleanup
|
|
658
|
-
* of previously-downloaded-now-out-of-scope files is reported via
|
|
659
|
-
* `scopeOrphansRemoved`.
|
|
660
|
-
*/
|
|
661
|
-
filesOutOfScope: number;
|
|
662
|
-
/**
|
|
663
|
-
* Clean local orphans deleted this run because a scope shrink moved them
|
|
664
|
-
* outside the effective scope (US-005). 0 when scope did not shrink.
|
|
665
|
-
*/
|
|
666
|
-
scopeOrphansRemoved: number;
|
|
667
|
-
/**
|
|
668
|
-
* Dirty (locally-modified) orphans that a scope shrink would have pruned.
|
|
669
|
-
* When `forceScopeShrink` is false these are surfaced via a thrown
|
|
670
|
-
* `ScopeShrinkBlockedError` and the leg never reaches this result; when
|
|
671
|
-
* true they are left on disk and tombstoned, and counted here.
|
|
672
|
-
*/
|
|
673
|
-
scopeOrphansBlocked: number;
|
|
674
|
-
/**
|
|
675
|
-
* Remote keys whose local on-disk counterpart changed during this pull.
|
|
676
|
-
* Includes downloads, tombstone deletes, and clean scope-orphan removals.
|
|
677
|
-
*/
|
|
678
|
-
changedPaths?: string[];
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
type SyncEventEmitter = (event: SyncProgressEvent) => void;
|
|
682
|
-
|
|
683
|
-
type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
|
|
684
|
-
|
|
685
|
-
interface PullRunContext {
|
|
686
|
-
options: SyncOptions;
|
|
687
|
-
companyRef: string;
|
|
688
|
-
vaultConfig: VaultServiceConfig;
|
|
689
|
-
hqRoot: string;
|
|
690
|
-
emit: SyncEventEmitter;
|
|
691
|
-
ctx: EntityContext;
|
|
692
|
-
companyRoot: string;
|
|
693
|
-
shouldSync: (filePath: string, isDir?: boolean) => boolean;
|
|
694
|
-
journalSlug: string;
|
|
695
|
-
startedAt: string;
|
|
696
|
-
journal: SyncJournal;
|
|
697
|
-
remoteFiles: RemoteFile[];
|
|
698
|
-
syncMode: SyncMode;
|
|
699
|
-
currentPrefixSet: string[];
|
|
700
|
-
/**
|
|
701
|
-
* Coalesced push-only exclusion prefixes (US-006: `sessions/`), subtracted
|
|
702
|
-
* from the download filter in EVERY mode. Kept SEPARATE from
|
|
703
|
-
* `currentPrefixSet` on purpose: the scope-shrink pass reads only
|
|
704
|
-
* `currentPrefixSet` (+ pins), so excluded-but-authored/pinned session files
|
|
705
|
-
* are never orphaned.
|
|
706
|
-
*/
|
|
707
|
-
currentExcludeSet: string[];
|
|
708
|
-
fileTombstones: ReadonlyMap<string, CompanyTombstone>;
|
|
709
|
-
/** Successful downloads since the last durable journal checkpoint. */
|
|
710
|
-
downloadsSinceJournalCheckpoint: number;
|
|
711
|
-
}
|
|
712
|
-
|
|
713
|
-
interface PullCounters {
|
|
714
|
-
filesDownloaded: number;
|
|
715
|
-
bytesDownloaded: number;
|
|
716
|
-
filesSkipped: number;
|
|
717
|
-
conflicts: number;
|
|
718
|
-
filesTombstoned: number;
|
|
719
|
-
filesOutOfScope: number;
|
|
720
|
-
conflictPaths: string[];
|
|
721
|
-
changedPaths: string[];
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
interface ScopeShrinkPlanState {
|
|
725
|
-
lastRecord: ReturnType<typeof lastPullRecord>;
|
|
726
|
-
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
727
|
-
autoRecover: boolean;
|
|
728
|
-
adviceContext: ScopeShrinkAdviceContext;
|
|
729
|
-
effectiveForce: boolean;
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
interface ScopeShrinkRun {
|
|
733
|
-
shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
|
|
734
|
-
shrinkResult: ReturnType<typeof applyScopeShrink>;
|
|
735
|
-
scopeOrphansRemoved: number;
|
|
736
|
-
changedPaths: string[];
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
/**
|
|
740
|
-
* Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
|
|
741
|
-
* scope shrink that would delete more than this many CLEAN local files in one
|
|
742
|
-
* pull is refused with `ScopeShrinkLargePruneError`. Default 100; `0` (or a
|
|
743
|
-
* non-positive / unparseable value) disables the cap (unlimited). Override via
|
|
744
|
-
* `HQ_SYNC_MAX_AUTO_PRUNE`.
|
|
745
|
-
*/
|
|
746
|
-
export function resolveAutoPruneCap(): number {
|
|
747
|
-
const raw = process.env.HQ_SYNC_MAX_AUTO_PRUNE;
|
|
748
|
-
if (raw === undefined || raw === "") return 100;
|
|
749
|
-
const parsed = Number.parseInt(raw, 10);
|
|
750
|
-
// NaN or negative → treat as "unlimited" (0) rather than silently capping.
|
|
751
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
|
|
752
|
-
}
|
|
753
|
-
|
|
754
|
-
/** Max time to wait on the best-effort new-files notification POST. */
|
|
755
|
-
const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
|
|
756
|
-
/** Bound crash recovery to at most this many completed pull downloads. */
|
|
757
|
-
const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 8;
|
|
758
|
-
|
|
759
|
-
/**
|
|
760
|
-
* Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
|
|
761
|
-
* an oversized batch wholesale, so the client MUST split a large report into
|
|
762
|
-
* chunks at or under this size — otherwise a first sync with more than this many
|
|
763
|
-
* new files reports none of them, and the same oversized batch re-triggers every
|
|
764
|
-
* sync cycle (wasted work + dropped notifications). Keep in lockstep with the
|
|
765
|
-
* server-side limit.
|
|
766
|
-
*/
|
|
767
|
-
const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
|
|
768
|
-
|
|
769
|
-
/**
|
|
770
|
-
* Best-effort report of the files that were new to this drive during the sync,
|
|
771
|
-
* so the HQ Sync app can show a persistent cross-session "new files" history.
|
|
772
|
-
*
|
|
773
|
-
* POSTs to `${apiUrl}/v1/notify/file-added`, which writes per-recipient
|
|
774
|
-
* FILE_EVENT rows for the calling user (the one the files are new for). Fully
|
|
775
|
-
* non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
|
|
776
|
-
* is the synced file itself; this is only a notification mirror. Bounded by a
|
|
777
|
-
* 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
|
|
778
|
-
* when there are no new files.
|
|
779
|
-
*
|
|
780
|
-
* Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
|
|
781
|
-
* files (the server's per-report cap). Each chunk is POSTed independently and
|
|
782
|
-
* best-effort, so one failing/oversized batch can never block the others or the
|
|
783
|
-
* sync. Exported only so the chunking can be unit-tested directly.
|
|
784
|
-
*/
|
|
785
|
-
export async function reportNewFilesToNotify(
|
|
786
|
-
vaultConfig: VaultServiceConfig,
|
|
787
|
-
companyUid: string,
|
|
788
|
-
companySlug: string,
|
|
789
|
-
files: Array<{ path: string; bytes: number; addedBy: string | null }>,
|
|
790
|
-
telemetryClaims?: TelemetryClaims | null,
|
|
791
|
-
): Promise<void> {
|
|
792
|
-
if (files.length === 0) return;
|
|
793
|
-
|
|
794
|
-
let token: string;
|
|
795
|
-
try {
|
|
796
|
-
token =
|
|
797
|
-
typeof vaultConfig.authToken === "function"
|
|
798
|
-
? await vaultConfig.authToken()
|
|
799
|
-
: vaultConfig.authToken;
|
|
800
|
-
} catch (err) {
|
|
801
|
-
logNotifyFailure(err);
|
|
802
|
-
return;
|
|
803
|
-
}
|
|
804
|
-
const base = vaultConfig.apiUrl.replace(/\/+$/, "");
|
|
805
|
-
const telemetryClient = new VaultClient(vaultConfig);
|
|
806
|
-
const batchCount = Math.ceil(files.length / NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
807
|
-
|
|
808
|
-
for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
|
|
809
|
-
const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
|
|
810
|
-
const batchIndex = Math.floor(i / NOTIFY_FILE_ADDED_MAX_BATCH) + 1;
|
|
811
|
-
const controller = new AbortController();
|
|
812
|
-
const timer = setTimeout(
|
|
813
|
-
() => controller.abort(),
|
|
814
|
-
NOTIFY_FILE_ADDED_TIMEOUT_MS,
|
|
815
|
-
);
|
|
816
|
-
try {
|
|
817
|
-
const response = await fetch(`${base}/v1/notify/file-added`, {
|
|
818
|
-
method: "POST",
|
|
819
|
-
headers: {
|
|
820
|
-
Authorization: `Bearer ${token}`,
|
|
821
|
-
"Content-Type": "application/json",
|
|
822
|
-
},
|
|
823
|
-
body: JSON.stringify({
|
|
824
|
-
companyUid,
|
|
825
|
-
companySlug,
|
|
826
|
-
files: batch.map((f) => ({
|
|
827
|
-
path: f.path,
|
|
828
|
-
bytes: f.bytes,
|
|
829
|
-
...(f.addedBy ? { addedBy: f.addedBy } : {}),
|
|
830
|
-
})),
|
|
831
|
-
}),
|
|
832
|
-
signal: controller.signal,
|
|
833
|
-
});
|
|
834
|
-
if (!response.ok) {
|
|
835
|
-
void emitCloudTelemetry(telemetryClient, {
|
|
836
|
-
eventName: "new_files_notification_reported",
|
|
837
|
-
source: "hq-sync",
|
|
838
|
-
companyUid,
|
|
839
|
-
properties: {
|
|
840
|
-
status: "failure",
|
|
841
|
-
fileCount: batch.length,
|
|
842
|
-
batchIndex,
|
|
843
|
-
batchCount,
|
|
844
|
-
statusCode: response.status,
|
|
845
|
-
},
|
|
846
|
-
}, { claims: telemetryClaims });
|
|
847
|
-
}
|
|
848
|
-
} catch (err) {
|
|
849
|
-
// Best-effort per chunk: never let notification reporting affect the sync
|
|
850
|
-
// result, and a failed chunk must not abort the remaining chunks.
|
|
851
|
-
logNotifyFailure(err);
|
|
852
|
-
void emitCloudTelemetry(telemetryClient, {
|
|
853
|
-
eventName: "new_files_notification_reported",
|
|
854
|
-
source: "hq-sync",
|
|
855
|
-
companyUid,
|
|
856
|
-
properties: {
|
|
857
|
-
status: "failure",
|
|
858
|
-
fileCount: batch.length,
|
|
859
|
-
batchIndex,
|
|
860
|
-
batchCount,
|
|
861
|
-
errorClass: safeErrorName(err),
|
|
862
|
-
},
|
|
863
|
-
}, { claims: telemetryClaims });
|
|
864
|
-
} finally {
|
|
865
|
-
clearTimeout(timer);
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
|
|
870
|
-
function safeErrorName(err: unknown): string {
|
|
871
|
-
return err instanceof Error && err.name ? err.name : "error";
|
|
872
|
-
}
|
|
873
|
-
|
|
874
|
-
/** Log a non-fatal notify failure without ever throwing out of the logger. */
|
|
875
|
-
function logNotifyFailure(err: unknown): void {
|
|
876
|
-
try {
|
|
877
|
-
console.error(
|
|
878
|
-
`[hq-sync] new-files notify report failed (non-fatal): ${
|
|
879
|
-
err instanceof Error ? err.message : String(err)
|
|
880
|
-
}`,
|
|
881
|
-
);
|
|
882
|
-
} catch {
|
|
883
|
-
// swallow — logging must never break sync
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
|
|
887
|
-
/** Log a non-fatal post-sync maintenance failure without changing sync state. */
|
|
888
|
-
function logPostSyncMaintenanceFailure(
|
|
889
|
-
task: string,
|
|
890
|
-
hqRoot: string,
|
|
891
|
-
err: unknown,
|
|
892
|
-
): void {
|
|
893
|
-
try {
|
|
894
|
-
console.error(
|
|
895
|
-
`[hq-sync] ${task} failed (non-fatal; hqRoot=${hqRoot}): ${
|
|
896
|
-
err instanceof Error ? err.message : String(err)
|
|
897
|
-
}`,
|
|
898
|
-
);
|
|
899
|
-
} catch {
|
|
900
|
-
// swallow — logging must never break sync
|
|
901
|
-
}
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
/**
|
|
905
|
-
* Sync (pull) all allowed files from the entity vault.
|
|
906
|
-
*/
|
|
907
|
-
export async function sync(options: SyncOptions): Promise<SyncResult> {
|
|
908
|
-
if (options.operationLockAlreadyHeld) {
|
|
909
|
-
return syncWithOperationLockHeld(options);
|
|
910
|
-
}
|
|
911
|
-
return withOperationLock(options.hqRoot, "sync", () =>
|
|
912
|
-
syncWithOperationLockHeld(options),
|
|
913
|
-
);
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
async function syncWithOperationLockHeld(
|
|
917
|
-
options: SyncOptions,
|
|
918
|
-
): Promise<SyncResult> {
|
|
919
|
-
const run = await buildPullContext(options);
|
|
920
|
-
const plan = planPull(run);
|
|
921
|
-
|
|
922
|
-
emitPullPlan(run.emit, plan);
|
|
923
|
-
reportInvalidRemoteKeys(run, plan);
|
|
924
|
-
|
|
925
|
-
const scopePlan = planScopeShrink(run);
|
|
926
|
-
const scopeRun = executeScopeShrink(run, scopePlan);
|
|
927
|
-
const counters = createPullCounters();
|
|
928
|
-
|
|
929
|
-
const transferConcurrency = resolveTransferConcurrency();
|
|
930
|
-
const conflictRun = await executeConflictExecutor(
|
|
931
|
-
run,
|
|
932
|
-
plan,
|
|
933
|
-
scopeRun,
|
|
934
|
-
counters,
|
|
935
|
-
);
|
|
936
|
-
if (conflictRun.abortResult) {
|
|
937
|
-
return conflictRun.abortResult;
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
await executeDownloadExecutor(
|
|
941
|
-
run,
|
|
942
|
-
conflictRun.downloadItems,
|
|
943
|
-
transferConcurrency,
|
|
944
|
-
counters,
|
|
945
|
-
);
|
|
946
|
-
|
|
947
|
-
await emitAndReportNewFiles(run, plan);
|
|
948
|
-
await verifyPlannedJournalTombstones(run, plan);
|
|
949
|
-
executeJournalTombstoneDeletes(run, plan, counters);
|
|
950
|
-
|
|
951
|
-
emitScopeMaterializationGap(run, plan);
|
|
952
|
-
|
|
953
|
-
return finalizePullRun(run, plan, scopeRun, counters);
|
|
954
|
-
}
|
|
955
|
-
|
|
956
|
-
async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
|
|
957
|
-
const { company, vaultConfig, hqRoot } = options;
|
|
958
|
-
const baseEmit = options.onEvent ?? defaultConsoleLogger;
|
|
959
|
-
|
|
960
|
-
const companyRef = company ?? resolveActiveCompany(hqRoot);
|
|
961
|
-
if (!companyRef) {
|
|
962
|
-
throw new Error(
|
|
963
|
-
"No company specified and no active company found. " +
|
|
964
|
-
"Use --company <slug> or set up .hq/config.json.",
|
|
965
|
-
);
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
const ctx = await resolveEntityContext(companyRef, vaultConfig);
|
|
969
|
-
const companyRoot = options.personalMode === true
|
|
970
|
-
? hqRoot
|
|
971
|
-
: path.join(hqRoot, "companies", ctx.slug);
|
|
972
|
-
const shouldSync = createIgnoreFilter(hqRoot);
|
|
973
|
-
const journalSlug = options.journalSlug ?? ctx.slug;
|
|
974
|
-
const startedAt = new Date().toISOString();
|
|
975
|
-
|
|
976
|
-
// Mirror every progress event into the shared cross-process snapshot so the
|
|
977
|
-
// menubar shows live progress for THIS sync regardless of who launched it
|
|
978
|
-
// (auto-sync / Sync Now / CLI). Record the friendly company slug (e.g.
|
|
979
|
-
// "indigo") — or null for the personal vault so the menubar shows "Personal"
|
|
980
|
-
// — never the raw prs_/cmp_ UID. Best-effort; never affects the sync.
|
|
981
|
-
const recordProgress = createSyncProgressRecorder({
|
|
982
|
-
company: options.personalMode === true ? null : ctx.slug,
|
|
983
|
-
phase: "pull",
|
|
984
|
-
});
|
|
985
|
-
const emit: SyncEventEmitter = (event) => {
|
|
986
|
-
baseEmit(event);
|
|
987
|
-
recordProgress(event);
|
|
988
|
-
};
|
|
989
|
-
|
|
990
|
-
if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
|
|
991
|
-
const journal = migrateToV2(readJournal(journalSlug));
|
|
992
|
-
gcTombstones(journal, Date.now());
|
|
993
|
-
|
|
994
|
-
const syncMode: SyncMode = options.syncMode ?? "all";
|
|
995
|
-
const currentPrefixSet: string[] =
|
|
996
|
-
syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
|
|
997
|
-
// Push-only exclusions (US-006). Coalesced independently of `currentPrefixSet`
|
|
998
|
-
// and applied to EVERY mode's download filter — including `all` — but kept out
|
|
999
|
-
// of the scope-shrink inputs (see `planScopeShrink`). Empty when the caller
|
|
1000
|
-
// passes nothing (legacy behavior: no exclusions).
|
|
1001
|
-
const currentExcludeSet: string[] = coalescePrefixes(
|
|
1002
|
-
options.excludePrefixes ?? [],
|
|
1003
|
-
);
|
|
1004
|
-
|
|
1005
|
-
const remoteFiles = await listRemoteFiles(ctx);
|
|
1006
|
-
const fileTombstones =
|
|
1007
|
-
options.personalMode === true
|
|
1008
|
-
? new Map<string, CompanyTombstone>()
|
|
1009
|
-
: await fetchCompanyTombstones(vaultConfig, ctx.uid);
|
|
1010
|
-
|
|
1011
|
-
return {
|
|
1012
|
-
options,
|
|
1013
|
-
companyRef,
|
|
1014
|
-
vaultConfig,
|
|
1015
|
-
hqRoot,
|
|
1016
|
-
emit,
|
|
1017
|
-
ctx,
|
|
1018
|
-
companyRoot,
|
|
1019
|
-
shouldSync,
|
|
1020
|
-
journalSlug,
|
|
1021
|
-
startedAt,
|
|
1022
|
-
journal,
|
|
1023
|
-
remoteFiles,
|
|
1024
|
-
syncMode,
|
|
1025
|
-
currentPrefixSet,
|
|
1026
|
-
currentExcludeSet,
|
|
1027
|
-
fileTombstones,
|
|
1028
|
-
downloadsSinceJournalCheckpoint: 0,
|
|
1029
|
-
};
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
function planPull(run: PullRunContext): PullPlan {
|
|
1033
|
-
return computePullPlan(
|
|
1034
|
-
run.remoteFiles,
|
|
1035
|
-
run.journal,
|
|
1036
|
-
run.companyRoot,
|
|
1037
|
-
run.shouldSync,
|
|
1038
|
-
run.options.personalMode === true,
|
|
1039
|
-
run.options.teamSyncedSlugs ?? null,
|
|
1040
|
-
run.currentPrefixSet,
|
|
1041
|
-
run.fileTombstones,
|
|
1042
|
-
run.currentExcludeSet,
|
|
1043
|
-
);
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
|
|
1047
|
-
emit({
|
|
1048
|
-
type: "plan",
|
|
1049
|
-
filesToDownload: plan.filesToDownload,
|
|
1050
|
-
bytesToDownload: plan.bytesToDownload,
|
|
1051
|
-
filesToUpload: 0,
|
|
1052
|
-
bytesToUpload: 0,
|
|
1053
|
-
filesToSkip: plan.filesToSkip,
|
|
1054
|
-
filesToConflict: plan.filesToConflict,
|
|
1055
|
-
filesToDelete: plan.filesToTombstoneDelete,
|
|
1056
|
-
});
|
|
1057
|
-
}
|
|
1058
|
-
|
|
1059
|
-
/**
|
|
1060
|
-
* Surface permanently-unpresignable remote keys the planner skipped (see the
|
|
1061
|
-
* `invalidRemoteKeys` doc on PullPlan). One warning event per key on the
|
|
1062
|
-
* existing sync event channel (deliberately NOT `type: "error"` — see the
|
|
1063
|
-
* `skip-invalid-scoped-key` event doc), plus one best-effort cloud telemetry
|
|
1064
|
-
* event on the channel sync already uses (emitCloudTelemetry). Per policy
|
|
1065
|
-
* hq-alert-baseline-calibration the documented healthy baseline is ZERO
|
|
1066
|
-
* invalid-key occurrences fleet-wide, so consumers can alert on any non-zero
|
|
1067
|
-
* count with no threshold tuning.
|
|
1068
|
-
*/
|
|
1069
|
-
function reportInvalidRemoteKeys(run: PullRunContext, plan: PullPlan): void {
|
|
1070
|
-
if (plan.invalidRemoteKeys.length === 0) return;
|
|
1071
|
-
const countsByErrorCode = new Map<string, number>();
|
|
1072
|
-
for (const { path, errorCode } of plan.invalidRemoteKeys) {
|
|
1073
|
-
run.emit({ type: "skip-invalid-scoped-key", path, errorCode });
|
|
1074
|
-
countsByErrorCode.set(errorCode, (countsByErrorCode.get(errorCode) ?? 0) + 1);
|
|
1075
|
-
}
|
|
1076
|
-
for (const [errorCode, count] of countsByErrorCode) {
|
|
1077
|
-
void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
|
|
1078
|
-
eventName: "sync_invalid_scoped_key_skipped",
|
|
1079
|
-
source: "hq-sync",
|
|
1080
|
-
companyUid: run.ctx.uid,
|
|
1081
|
-
properties: { count, errorCode, leg: "pull" },
|
|
1082
|
-
}, { claims: run.options.telemetryClaims });
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
/**
|
|
1087
|
-
* A legacy object without hq-mode can otherwise downgrade a hook to the
|
|
1088
|
-
* receiver umask with no durable signal. downloadFile already prints a local
|
|
1089
|
-
* warning for every caller; the sync path also records a path-free telemetry
|
|
1090
|
-
* event so operators can find vaults that still need the one-time backfill.
|
|
1091
|
-
*/
|
|
1092
|
-
function reportModeWarnings(
|
|
1093
|
-
run: PullRunContext,
|
|
1094
|
-
warnings: DownloadModeWarning[] | undefined,
|
|
1095
|
-
): void {
|
|
1096
|
-
for (const warning of warnings ?? []) {
|
|
1097
|
-
void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
|
|
1098
|
-
eventName: "sync_mode_guardrail_warning",
|
|
1099
|
-
source: "hq-sync",
|
|
1100
|
-
companyUid: run.ctx.uid,
|
|
1101
|
-
properties: {
|
|
1102
|
-
leg: "pull",
|
|
1103
|
-
reason: warning.reason,
|
|
1104
|
-
fallback: warning.fallback ?? null,
|
|
1105
|
-
},
|
|
1106
|
-
}, { claims: run.options.telemetryClaims });
|
|
1107
|
-
}
|
|
1108
|
-
}
|
|
1109
|
-
|
|
1110
|
-
function createPullCounters(): PullCounters {
|
|
1111
|
-
return {
|
|
1112
|
-
filesDownloaded: 0,
|
|
1113
|
-
bytesDownloaded: 0,
|
|
1114
|
-
filesSkipped: 0,
|
|
1115
|
-
conflicts: 0,
|
|
1116
|
-
filesTombstoned: 0,
|
|
1117
|
-
filesOutOfScope: 0,
|
|
1118
|
-
conflictPaths: [],
|
|
1119
|
-
changedPaths: [],
|
|
1120
|
-
};
|
|
1121
|
-
}
|
|
1122
|
-
|
|
1123
|
-
function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
|
|
1124
|
-
const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
|
|
1125
|
-
const lastPrefixSet =
|
|
1126
|
-
lastRecord && lastRecord.prefixSet.length > 0
|
|
1127
|
-
? lastRecord.prefixSet
|
|
1128
|
-
: [""];
|
|
1129
|
-
const shrinkPlan = buildScopeShrinkPlan({
|
|
1130
|
-
journal: run.journal,
|
|
1131
|
-
hqRoot: run.companyRoot,
|
|
1132
|
-
lastPrefixSet,
|
|
1133
|
-
// INCLUSION scope only (+ pins, already folded into currentPrefixSet by the
|
|
1134
|
-
// resolver). The push-only exclude set (US-006: `sessions/`) is DELIBERATELY
|
|
1135
|
-
// NOT subtracted here: a session file that was authored locally
|
|
1136
|
-
// (`direction:"up"`) or pinned + materialized on demand must never be
|
|
1137
|
-
// orphaned just because sessions are excluded from the download filter.
|
|
1138
|
-
// Feeding `currentExcludeSet` in here would prune exactly those files.
|
|
1139
|
-
currentPrefixSet: run.currentPrefixSet,
|
|
1140
|
-
callerSub: run.options.callerSub,
|
|
1141
|
-
protectUnknownAuthors: true,
|
|
1142
|
-
});
|
|
1143
|
-
const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
|
|
1144
|
-
const autoRecover = scopeShrinkPolicy === "auto-recover";
|
|
1145
|
-
const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
|
|
1146
|
-
const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
|
|
1147
|
-
|
|
1148
|
-
return {
|
|
1149
|
-
lastRecord,
|
|
1150
|
-
shrinkPlan,
|
|
1151
|
-
autoRecover,
|
|
1152
|
-
adviceContext,
|
|
1153
|
-
effectiveForce,
|
|
1154
|
-
};
|
|
1155
|
-
}
|
|
1156
|
-
|
|
1157
|
-
function executeScopeShrink(
|
|
1158
|
-
run: PullRunContext,
|
|
1159
|
-
scopePlan: ScopeShrinkPlanState,
|
|
1160
|
-
): ScopeShrinkRun {
|
|
1161
|
-
const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
|
|
1162
|
-
|
|
1163
|
-
if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
|
|
1164
|
-
throw new ScopeShrinkBlockedError(
|
|
1165
|
-
run.ctx.uid,
|
|
1166
|
-
lastRecord?.syncMode ?? "unknown",
|
|
1167
|
-
run.syncMode,
|
|
1168
|
-
shrinkPlan.dirty,
|
|
1169
|
-
shrinkPlan.clean,
|
|
1170
|
-
adviceContext,
|
|
1171
|
-
);
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
const autoPruneCap = resolveAutoPruneCap();
|
|
1175
|
-
if (
|
|
1176
|
-
!effectiveForce &&
|
|
1177
|
-
autoPruneCap > 0 &&
|
|
1178
|
-
shrinkPlan.clean.length > autoPruneCap
|
|
1179
|
-
) {
|
|
1180
|
-
throw new ScopeShrinkLargePruneError(
|
|
1181
|
-
run.ctx.uid,
|
|
1182
|
-
run.syncMode,
|
|
1183
|
-
shrinkPlan.clean.length,
|
|
1184
|
-
autoPruneCap,
|
|
1185
|
-
adviceContext,
|
|
1186
|
-
);
|
|
1187
|
-
}
|
|
1188
|
-
|
|
1189
|
-
const scopeQuarantineRoot = path.join(
|
|
1190
|
-
run.hqRoot,
|
|
1191
|
-
".hq",
|
|
1192
|
-
"scope-quarantine",
|
|
1193
|
-
run.journalSlug,
|
|
1194
|
-
);
|
|
1195
|
-
const shrinkResult = applyScopeShrink({
|
|
1196
|
-
journal: run.journal,
|
|
1197
|
-
plan: shrinkPlan,
|
|
1198
|
-
hqRoot: run.companyRoot,
|
|
1199
|
-
forceScopeShrink: effectiveForce,
|
|
1200
|
-
reason: "scope_shrink",
|
|
1201
|
-
cleanDisposition: "quarantine",
|
|
1202
|
-
quarantineRoot: scopeQuarantineRoot,
|
|
1203
|
-
});
|
|
1204
|
-
|
|
1205
|
-
for (const relPath of shrinkResult.quarantinedPaths) {
|
|
1206
|
-
run.emit({
|
|
1207
|
-
type: "progress",
|
|
1208
|
-
path: relPath,
|
|
1209
|
-
bytes: 0,
|
|
1210
|
-
deleted: true,
|
|
1211
|
-
message: `scope-narrowed: moved out-of-scope copy to ${scopeQuarantineRoot}`,
|
|
1212
|
-
});
|
|
1213
|
-
}
|
|
1214
|
-
for (const relPath of shrinkResult.removedPaths) {
|
|
1215
|
-
run.emit({
|
|
1216
|
-
type: "progress",
|
|
1217
|
-
path: relPath,
|
|
1218
|
-
bytes: 0,
|
|
1219
|
-
deleted: true,
|
|
1220
|
-
message: "scope-narrowed (removed local copy outside sync scope)",
|
|
1221
|
-
});
|
|
1222
|
-
}
|
|
1223
|
-
for (const relPath of shrinkResult.dirtyKeptPaths) {
|
|
1224
|
-
run.emit({
|
|
1225
|
-
type: "progress",
|
|
1226
|
-
path: relPath,
|
|
1227
|
-
bytes: 0,
|
|
1228
|
-
message:
|
|
1229
|
-
"scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
|
|
1230
|
-
});
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
return {
|
|
1234
|
-
shrinkPlan,
|
|
1235
|
-
shrinkResult,
|
|
1236
|
-
scopeOrphansRemoved:
|
|
1237
|
-
shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
|
|
1238
|
-
changedPaths: [
|
|
1239
|
-
...shrinkResult.quarantinedPaths,
|
|
1240
|
-
...shrinkResult.removedPaths,
|
|
1241
|
-
],
|
|
1242
|
-
};
|
|
1243
|
-
}
|
|
1244
|
-
|
|
1245
|
-
async function refreshRunContextIfExpiring(
|
|
1246
|
-
run: PullRunContext,
|
|
1247
|
-
): Promise<void> {
|
|
1248
|
-
if (isExpiringSoon(run.ctx.expiresAt)) {
|
|
1249
|
-
run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
|
-
|
|
1253
|
-
async function executeConflictExecutor(
|
|
1254
|
-
run: PullRunContext,
|
|
1255
|
-
plan: PullPlan,
|
|
1256
|
-
scopeRun: ScopeShrinkRun,
|
|
1257
|
-
counters: PullCounters,
|
|
1258
|
-
): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
|
|
1259
|
-
const downloadItems: PullDownloadItem[] = [];
|
|
1260
|
-
|
|
1261
|
-
for (const item of plan.items) {
|
|
1262
|
-
if (
|
|
1263
|
-
item.action === "skip-ignored" ||
|
|
1264
|
-
item.action === "skip-personal-mode" ||
|
|
1265
|
-
item.action === "skip-unchanged" ||
|
|
1266
|
-
item.action === "skip-local-only" ||
|
|
1267
|
-
item.action === "skip-stale-overlay-marker" ||
|
|
1268
|
-
item.action === "skip-overlay-marker-with-children"
|
|
1269
|
-
) {
|
|
1270
|
-
counters.filesSkipped++;
|
|
1271
|
-
continue;
|
|
1272
|
-
}
|
|
1273
|
-
if (item.action === "skip-unreadable-link") {
|
|
1274
|
-
// An unreadable local link must stay non-fatal: the runner treats a
|
|
1275
|
-
// generic error event as a failed company leg. Surface the exact remote
|
|
1276
|
-
// key through the recoverable not-shipped channel instead.
|
|
1277
|
-
counters.filesSkipped++;
|
|
1278
|
-
run.emit({
|
|
1279
|
-
type: "not-shipped",
|
|
1280
|
-
reason: "unreadable-link",
|
|
1281
|
-
count: 1,
|
|
1282
|
-
samplePaths: [item.remoteFile.key],
|
|
1283
|
-
});
|
|
1284
|
-
continue;
|
|
1285
|
-
}
|
|
1286
|
-
if (item.action === "skip-excluded-policy") {
|
|
1287
|
-
continue;
|
|
1288
|
-
}
|
|
1289
|
-
if (item.action === "skip-out-of-scope") {
|
|
1290
|
-
counters.filesOutOfScope++;
|
|
1291
|
-
continue;
|
|
1292
|
-
}
|
|
1293
|
-
if (item.action === "skip-junk-key-spelling") {
|
|
1294
|
-
// Deliberate, recoverable skip — counted as skipped and surfaced via a
|
|
1295
|
-
// dedicated warning event, never `type: "error"` (see the event doc /
|
|
1296
|
-
// policy hq-sync-deliberate-skip-not-fatal-error-exit2).
|
|
1297
|
-
counters.filesSkipped++;
|
|
1298
|
-
run.emit({
|
|
1299
|
-
type: "skip-junk-key-spelling",
|
|
1300
|
-
path: item.remoteFile.key,
|
|
1301
|
-
journaledKey: item.journaledKey,
|
|
1302
|
-
});
|
|
1303
|
-
continue;
|
|
1304
|
-
}
|
|
1305
|
-
if (item.action === "tombstone-delete") {
|
|
1306
|
-
executeFileTombstoneDelete(run, item, counters);
|
|
1307
|
-
continue;
|
|
1308
|
-
}
|
|
1309
|
-
if (item.action === "download") {
|
|
1310
|
-
downloadItems.push(item);
|
|
1311
|
-
continue;
|
|
1312
|
-
}
|
|
1313
|
-
|
|
1314
|
-
const abortResult = await executeConflictItem(
|
|
1315
|
-
run,
|
|
1316
|
-
plan,
|
|
1317
|
-
scopeRun,
|
|
1318
|
-
counters,
|
|
1319
|
-
downloadItems,
|
|
1320
|
-
item,
|
|
1321
|
-
);
|
|
1322
|
-
if (abortResult) {
|
|
1323
|
-
return { downloadItems, abortResult };
|
|
1324
|
-
}
|
|
1325
|
-
}
|
|
1326
|
-
|
|
1327
|
-
return { downloadItems, abortResult: null };
|
|
1328
|
-
}
|
|
1329
|
-
|
|
1330
|
-
function executeFileTombstoneDelete(
|
|
1331
|
-
run: PullRunContext,
|
|
1332
|
-
item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
|
|
1333
|
-
counters: PullCounters,
|
|
1334
|
-
): void {
|
|
1335
|
-
const tombstoneKey = item.remoteFile.key;
|
|
1336
|
-
const tombstonePath = item.localPath;
|
|
1337
|
-
if (!isDownloadWritePathStillContained(run.companyRoot, tombstoneKey, tombstonePath)) {
|
|
1338
|
-
run.emit({
|
|
1339
|
-
type: "error",
|
|
1340
|
-
path: tombstoneKey,
|
|
1341
|
-
message: "tombstone-suppress unlink skipped: local parent escaped the sync root",
|
|
1342
|
-
});
|
|
1343
|
-
return;
|
|
1344
|
-
}
|
|
1345
|
-
try {
|
|
1346
|
-
const lstat = fs.lstatSync(tombstonePath);
|
|
1347
|
-
if (!matchesLocalSnapshot(tombstonePath, item.localSnapshot)) {
|
|
1348
|
-
reportLocalSnapshotConflict(run, counters, tombstoneKey);
|
|
1349
|
-
return;
|
|
1350
|
-
}
|
|
1351
|
-
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
1352
|
-
fs.unlinkSync(tombstonePath);
|
|
1353
|
-
}
|
|
1354
|
-
} catch (err: unknown) {
|
|
1355
|
-
const code =
|
|
1356
|
-
err && typeof err === "object" && "code" in err
|
|
1357
|
-
? (err as { code?: string }).code
|
|
1358
|
-
: undefined;
|
|
1359
|
-
if (code !== "ENOENT") {
|
|
1360
|
-
run.emit({
|
|
1361
|
-
type: "error",
|
|
1362
|
-
path: tombstoneKey,
|
|
1363
|
-
message: `tombstone-suppress unlink failed: ${
|
|
1364
|
-
err instanceof Error ? err.message : String(err)
|
|
1365
|
-
}`,
|
|
1366
|
-
});
|
|
1367
|
-
return;
|
|
1368
|
-
}
|
|
1369
|
-
}
|
|
1370
|
-
removeEntry(run.journal, tombstoneKey);
|
|
1371
|
-
counters.filesTombstoned++;
|
|
1372
|
-
counters.changedPaths.push(tombstoneKey);
|
|
1373
|
-
run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
async function executeConflictItem(
|
|
1377
|
-
run: PullRunContext,
|
|
1378
|
-
plan: PullPlan,
|
|
1379
|
-
scopeRun: ScopeShrinkRun,
|
|
1380
|
-
counters: PullCounters,
|
|
1381
|
-
downloadItems: PullDownloadItem[],
|
|
1382
|
-
item: Extract<PullPlanItem, { action: "conflict" }>,
|
|
1383
|
-
): Promise<SyncResult | null> {
|
|
1384
|
-
const { remoteFile, localPath } = item;
|
|
1385
|
-
|
|
1386
|
-
// Cloud-authoritative paths (server-regenerated: company-brief.md, board.json,
|
|
1387
|
-
// ontology/ signals/ sources/) are PULL-WINS: the cloud copy is the source of
|
|
1388
|
-
// truth, so a divergence here is not a real conflict. Resolve it as a normal
|
|
1389
|
-
// overwrite-from-cloud — queue a plain download (which stamps the journal with
|
|
1390
|
-
// the REMOTE hash/etag) and skip the conflict path entirely. This removes both
|
|
1391
|
-
// the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
|
|
1392
|
-
// divergent local content) that the keep path produced for these files.
|
|
1393
|
-
if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
|
|
1394
|
-
downloadItems.push({
|
|
1395
|
-
action: "download",
|
|
1396
|
-
remoteFile,
|
|
1397
|
-
localPath,
|
|
1398
|
-
isNew: false,
|
|
1399
|
-
localSnapshot: item.localSnapshot,
|
|
1400
|
-
});
|
|
1401
|
-
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1402
|
-
return null;
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
await refreshRunContextIfExpiring(run);
|
|
1406
|
-
|
|
1407
|
-
const detectedAt = new Date().toISOString();
|
|
1408
|
-
const machineId = readShortMachineId(run.hqRoot);
|
|
1409
|
-
const originalRelative = vaultKeyForLocalPath(run.hqRoot, localPath);
|
|
1410
|
-
const conflictRelative = buildConflictPath(
|
|
1411
|
-
originalRelative,
|
|
1412
|
-
detectedAt,
|
|
1413
|
-
machineId,
|
|
1414
|
-
);
|
|
1415
|
-
const conflictAbs = localPathForVaultKey(run.hqRoot, conflictRelative);
|
|
1416
|
-
const conflictKey = vaultKeyForLocalPath(run.companyRoot, conflictAbs);
|
|
1417
|
-
|
|
1418
|
-
if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
|
|
1419
|
-
counters.filesSkipped++;
|
|
1420
|
-
run.emit({
|
|
1421
|
-
type: "error",
|
|
1422
|
-
path: remoteFile.key,
|
|
1423
|
-
message: "conflict mirror skipped: local parent escaped the sync root",
|
|
1424
|
-
});
|
|
1425
|
-
return null;
|
|
1426
|
-
}
|
|
1427
|
-
|
|
1428
|
-
let remoteFetched = false;
|
|
1429
|
-
let converged = false;
|
|
1430
|
-
try {
|
|
1431
|
-
const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
|
|
1432
|
-
remoteFetched = true;
|
|
1433
|
-
if (fs.lstatSync(conflictAbs).isSymbolicLink()) {
|
|
1434
|
-
const target = readlinkOrNull(conflictAbs);
|
|
1435
|
-
if (target === null) {
|
|
1436
|
-
counters.filesSkipped++;
|
|
1437
|
-
run.emit({
|
|
1438
|
-
type: "not-shipped",
|
|
1439
|
-
reason: "unreadable-link",
|
|
1440
|
-
count: 1,
|
|
1441
|
-
samplePaths: [remoteFile.key],
|
|
1442
|
-
});
|
|
1443
|
-
return null;
|
|
1444
|
-
} else {
|
|
1445
|
-
converged = hashSymlinkTarget(target) === item.localHash;
|
|
1446
|
-
}
|
|
1447
|
-
} else {
|
|
1448
|
-
converged = (downloaded.contentHash ?? hashFile(conflictAbs)) === item.localHash;
|
|
1449
|
-
}
|
|
1450
|
-
} catch (probeErr) {
|
|
1451
|
-
if (probeErr instanceof VaultAuthError) throw probeErr;
|
|
1452
|
-
run.emit({
|
|
1453
|
-
type: "error",
|
|
1454
|
-
path: remoteFile.key,
|
|
1455
|
-
message: `conflict convergence probe failed: ${
|
|
1456
|
-
probeErr instanceof Error ? probeErr.message : String(probeErr)
|
|
1457
|
-
}`,
|
|
1458
|
-
});
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
if (converged) {
|
|
1462
|
-
if (remoteFetched) {
|
|
1463
|
-
try {
|
|
1464
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
1465
|
-
} catch {
|
|
1466
|
-
/* best-effort cleanup; a stray identical mirror is harmless */
|
|
1467
|
-
}
|
|
1468
|
-
}
|
|
1469
|
-
updateEntry(
|
|
1470
|
-
run.journal,
|
|
1471
|
-
remoteFile.key,
|
|
1472
|
-
item.localHash,
|
|
1473
|
-
item.localSize,
|
|
1474
|
-
"down",
|
|
1475
|
-
localPath,
|
|
1476
|
-
{
|
|
1477
|
-
remoteEtag: remoteFile.etag,
|
|
1478
|
-
mtimeMs: item.localMtime.getTime(),
|
|
1479
|
-
kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
|
|
1480
|
-
},
|
|
1481
|
-
);
|
|
1482
|
-
run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
|
|
1483
|
-
counters.filesSkipped++;
|
|
1484
|
-
return null;
|
|
1485
|
-
}
|
|
1486
|
-
|
|
1487
|
-
counters.conflicts++;
|
|
1488
|
-
counters.conflictPaths.push(remoteFile.key);
|
|
1489
|
-
|
|
1490
|
-
const resolution = await resolveConflict(
|
|
1491
|
-
{
|
|
1492
|
-
path: remoteFile.key,
|
|
1493
|
-
localHash: item.localHash,
|
|
1494
|
-
remoteModified: remoteFile.lastModified,
|
|
1495
|
-
localModified: item.localMtime,
|
|
1496
|
-
direction: "pull",
|
|
1497
|
-
},
|
|
1498
|
-
run.options.onConflict,
|
|
1499
|
-
);
|
|
1500
|
-
|
|
1501
|
-
run.emit({
|
|
1502
|
-
type: "conflict",
|
|
1503
|
-
path: remoteFile.key,
|
|
1504
|
-
direction: "pull",
|
|
1505
|
-
resolution,
|
|
1506
|
-
});
|
|
1507
|
-
|
|
1508
|
-
if (resolution !== "abort" && resolution !== "overwrite") {
|
|
1509
|
-
if (remoteFetched) {
|
|
1510
|
-
try {
|
|
1511
|
-
appendConflictEntry(run.hqRoot, {
|
|
1512
|
-
id: buildConflictId(originalRelative, detectedAt),
|
|
1513
|
-
originalPath: originalRelative,
|
|
1514
|
-
conflictPath: conflictRelative,
|
|
1515
|
-
detectedAt,
|
|
1516
|
-
side: "pull",
|
|
1517
|
-
machineId,
|
|
1518
|
-
localHash: item.localHash,
|
|
1519
|
-
remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
|
|
1520
|
-
});
|
|
1521
|
-
} catch (mirrorErr) {
|
|
1522
|
-
run.emit({
|
|
1523
|
-
type: "error",
|
|
1524
|
-
path: remoteFile.key,
|
|
1525
|
-
message: `conflict mirror index write failed: ${
|
|
1526
|
-
mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
|
|
1527
|
-
}`,
|
|
1528
|
-
});
|
|
1529
|
-
}
|
|
1530
|
-
}
|
|
1531
|
-
} else if (remoteFetched) {
|
|
1532
|
-
try {
|
|
1533
|
-
fs.rmSync(conflictAbs, { force: true });
|
|
1534
|
-
} catch {
|
|
1535
|
-
/* best-effort; a leftover mirror is cosmetic, not corrupting */
|
|
1536
|
-
}
|
|
1537
|
-
}
|
|
1538
|
-
|
|
1539
|
-
if (resolution === "abort") {
|
|
1540
|
-
run.emit({ type: "new-files", files: [] });
|
|
1541
|
-
writeJournal(run.journalSlug, run.journal);
|
|
1542
|
-
return {
|
|
1543
|
-
filesDownloaded: counters.filesDownloaded,
|
|
1544
|
-
bytesDownloaded: counters.bytesDownloaded,
|
|
1545
|
-
filesSkipped: counters.filesSkipped,
|
|
1546
|
-
conflicts: counters.conflicts,
|
|
1547
|
-
conflictPaths: counters.conflictPaths,
|
|
1548
|
-
aborted: true,
|
|
1549
|
-
newFiles: plan.newFiles,
|
|
1550
|
-
newFilesCount: plan.newFilesCount,
|
|
1551
|
-
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1552
|
-
filesTombstoned: 0,
|
|
1553
|
-
filesOutOfScope: counters.filesOutOfScope,
|
|
1554
|
-
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1555
|
-
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1556
|
-
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
1557
|
-
};
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
if (resolution === "keep" || resolution === "skip") {
|
|
1561
|
-
counters.filesSkipped++;
|
|
1562
|
-
updateEntry(
|
|
1563
|
-
run.journal,
|
|
1564
|
-
remoteFile.key,
|
|
1565
|
-
item.localHash,
|
|
1566
|
-
item.localSize,
|
|
1567
|
-
"down",
|
|
1568
|
-
localPath,
|
|
1569
|
-
{
|
|
1570
|
-
remoteEtag: remoteFile.etag,
|
|
1571
|
-
mtimeMs: item.localMtime.getTime(),
|
|
1572
|
-
kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
|
|
1573
|
-
},
|
|
1574
|
-
);
|
|
1575
|
-
// Journal-honesty: we recorded the remote etag so this conflict can't
|
|
1576
|
-
// re-fire (#137), but the KEPT local copy diverges from that remote — it
|
|
1577
|
-
// never matched it. Flag the entry so the currency-gated delete planner
|
|
1578
|
-
// refuses to propagate a delete for it (its currency would falsely match on
|
|
1579
|
-
// HEAD, and the delete would destroy the divergent remote version). Any
|
|
1580
|
-
// genuine future download clears the flag by replacing the entry.
|
|
1581
|
-
const keptEntry = getEntry(run.journal, remoteFile.key);
|
|
1582
|
-
if (keptEntry) keptEntry.localDiverges = true;
|
|
1583
|
-
return null;
|
|
1584
|
-
}
|
|
1585
|
-
|
|
1586
|
-
downloadItems.push({
|
|
1587
|
-
action: "download",
|
|
1588
|
-
remoteFile,
|
|
1589
|
-
localPath,
|
|
1590
|
-
isNew: false,
|
|
1591
|
-
localSnapshot: item.localSnapshot,
|
|
1592
|
-
});
|
|
1593
|
-
return null;
|
|
1594
|
-
}
|
|
1595
|
-
|
|
1596
|
-
async function executeDownloadExecutor(
|
|
1597
|
-
run: PullRunContext,
|
|
1598
|
-
downloadItems: PullDownloadItem[],
|
|
1599
|
-
transferConcurrency: number,
|
|
1600
|
-
counters: PullCounters,
|
|
1601
|
-
): Promise<void> {
|
|
1602
|
-
if (downloadItems.length === 0) return;
|
|
1603
|
-
|
|
1604
|
-
await primeObjectTransport(
|
|
1605
|
-
run.ctx,
|
|
1606
|
-
"get",
|
|
1607
|
-
downloadItems.map((d) => d.remoteFile.key),
|
|
1608
|
-
);
|
|
1609
|
-
|
|
1610
|
-
const queue = [...downloadItems];
|
|
1611
|
-
const inFlight: Set<Promise<unknown>> = new Set();
|
|
1612
|
-
const workerErrors: Error[] = [];
|
|
1613
|
-
|
|
1614
|
-
while (queue.length > 0 || inFlight.size > 0) {
|
|
1615
|
-
while (inFlight.size < transferConcurrency && queue.length > 0) {
|
|
1616
|
-
const downloadItem = queue.shift()!;
|
|
1617
|
-
const p: Promise<void> = downloadOne(run, downloadItem, counters)
|
|
1618
|
-
.catch((err: unknown) => {
|
|
1619
|
-
workerErrors.push(err instanceof Error ? err : new Error(String(err)));
|
|
1620
|
-
})
|
|
1621
|
-
.finally(() => {
|
|
1622
|
-
inFlight.delete(p);
|
|
1623
|
-
});
|
|
1624
|
-
inFlight.add(p);
|
|
1625
|
-
}
|
|
1626
|
-
if (inFlight.size > 0) {
|
|
1627
|
-
await Promise.race(Array.from(inFlight));
|
|
1628
|
-
}
|
|
1629
|
-
}
|
|
1630
|
-
|
|
1631
|
-
if (workerErrors.length > 0) {
|
|
1632
|
-
writeJournal(run.journalSlug, run.journal);
|
|
1633
|
-
const first = workerErrors[0]!;
|
|
1634
|
-
if (workerErrors.length > 1) {
|
|
1635
|
-
first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
|
|
1636
|
-
}
|
|
1637
|
-
throw first;
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
async function downloadOne(
|
|
1642
|
-
run: PullRunContext,
|
|
1643
|
-
downloadItem: PullDownloadItem,
|
|
1644
|
-
counters: PullCounters,
|
|
1645
|
-
): Promise<void> {
|
|
1646
|
-
const { remoteFile, localPath } = downloadItem;
|
|
1647
|
-
|
|
1648
|
-
await refreshRunContextIfExpiring(run);
|
|
1649
|
-
|
|
1650
|
-
if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
|
|
1651
|
-
counters.filesSkipped++;
|
|
1652
|
-
run.emit({
|
|
1653
|
-
type: "error",
|
|
1654
|
-
path: remoteFile.key,
|
|
1655
|
-
message: "download skipped: local parent escaped the sync root",
|
|
1656
|
-
});
|
|
1657
|
-
return;
|
|
1658
|
-
}
|
|
1659
|
-
|
|
1660
|
-
try {
|
|
1661
|
-
const { metadata, contentHash, contentSize, modeWarnings } = await downloadFile(
|
|
1662
|
-
run.ctx,
|
|
1663
|
-
remoteFile.key,
|
|
1664
|
-
localPath,
|
|
1665
|
-
{
|
|
1666
|
-
beforeReplace: () => {
|
|
1667
|
-
if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
|
|
1668
|
-
throw new LocalSnapshotChangedError(remoteFile.key);
|
|
1669
|
-
}
|
|
1670
|
-
},
|
|
1671
|
-
},
|
|
1672
|
-
);
|
|
1673
|
-
reportModeWarnings(run, modeWarnings);
|
|
1674
|
-
const author = metadata?.["created-by"] ?? null;
|
|
1675
|
-
const createdBySub = metadata?.["created-by-sub"];
|
|
1676
|
-
|
|
1677
|
-
const localLstat = fs.lstatSync(localPath);
|
|
1678
|
-
const isLocalSymlink = localLstat.isSymbolicLink();
|
|
1679
|
-
let hash: string;
|
|
1680
|
-
if (isLocalSymlink) {
|
|
1681
|
-
const target = readlinkOrNull(localPath);
|
|
1682
|
-
if (target === null) {
|
|
1683
|
-
counters.filesSkipped++;
|
|
1684
|
-
run.emit({
|
|
1685
|
-
type: "not-shipped",
|
|
1686
|
-
reason: "unreadable-link",
|
|
1687
|
-
count: 1,
|
|
1688
|
-
samplePaths: [remoteFile.key],
|
|
1689
|
-
});
|
|
1690
|
-
return;
|
|
1691
|
-
}
|
|
1692
|
-
hash = hashSymlinkTarget(target);
|
|
1693
|
-
} else {
|
|
1694
|
-
hash = contentHash ?? hashFile(localPath);
|
|
1695
|
-
}
|
|
1696
|
-
const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
|
|
1697
|
-
|
|
1698
|
-
updateEntry(
|
|
1699
|
-
run.journal,
|
|
1700
|
-
remoteFile.key,
|
|
1701
|
-
hash,
|
|
1702
|
-
size,
|
|
1703
|
-
"down",
|
|
1704
|
-
localPath,
|
|
1705
|
-
{
|
|
1706
|
-
remoteEtag: remoteFile.etag,
|
|
1707
|
-
mtimeMs: localLstat.mtimeMs,
|
|
1708
|
-
...(createdBySub !== undefined ? { createdBySub } : {}),
|
|
1709
|
-
kind: isLocalSymlink ? "symlink" : "file",
|
|
1710
|
-
},
|
|
1711
|
-
);
|
|
1712
|
-
|
|
1713
|
-
const priorEntry = getEntry(run.journal, remoteFile.key);
|
|
1714
|
-
const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
|
|
1715
|
-
run.emit({
|
|
1716
|
-
type: "progress",
|
|
1717
|
-
path: remoteFile.key,
|
|
1718
|
-
bytes: size,
|
|
1719
|
-
...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
|
|
1720
|
-
...(author ? { author } : {}),
|
|
1721
|
-
});
|
|
1722
|
-
|
|
1723
|
-
counters.filesDownloaded++;
|
|
1724
|
-
counters.bytesDownloaded += size;
|
|
1725
|
-
counters.changedPaths.push(remoteFile.key);
|
|
1726
|
-
} catch (err) {
|
|
1727
|
-
if (err instanceof VaultAuthError) throw err;
|
|
1728
|
-
if (err instanceof LocalSnapshotChangedError) {
|
|
1729
|
-
reportLocalSnapshotConflict(run, counters, remoteFile.key);
|
|
1730
|
-
} else if (isAccessDenied(err)) {
|
|
1731
|
-
counters.filesSkipped++;
|
|
1732
|
-
} else if (err instanceof DanglingSymlinkParentError) {
|
|
1733
|
-
// Per-object condition, not a company failure. The pre-flight
|
|
1734
|
-
// containment check can miss this: it runs before the download, and a
|
|
1735
|
-
// sibling worker in the SAME batch may materialize the parent symlink in
|
|
1736
|
-
// between (the vault stores the link and the files beneath it as
|
|
1737
|
-
// separate objects, ordered arbitrarily). Skip loudly — never silently —
|
|
1738
|
-
// so the run stays `complete` instead of `errored` on every cycle.
|
|
1739
|
-
counters.filesSkipped++;
|
|
1740
|
-
run.emit({
|
|
1741
|
-
type: "error",
|
|
1742
|
-
path: remoteFile.key,
|
|
1743
|
-
message: err.message,
|
|
1744
|
-
});
|
|
1745
|
-
} else {
|
|
1746
|
-
run.emit({
|
|
1747
|
-
type: "error",
|
|
1748
|
-
path: remoteFile.key,
|
|
1749
|
-
message: err instanceof Error ? err.message : String(err),
|
|
1750
|
-
});
|
|
1751
|
-
}
|
|
1752
|
-
return;
|
|
1753
|
-
}
|
|
1754
|
-
|
|
1755
|
-
run.downloadsSinceJournalCheckpoint++;
|
|
1756
|
-
if (
|
|
1757
|
-
run.downloadsSinceJournalCheckpoint >=
|
|
1758
|
-
DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE
|
|
1759
|
-
) {
|
|
1760
|
-
// writeJournal is already fsync + rename atomic. Persist completed work
|
|
1761
|
-
// before scheduling more objects so an abrupt process exit only replays a
|
|
1762
|
-
// small tail, not an entire bulk download.
|
|
1763
|
-
writeJournal(run.journalSlug, run.journal);
|
|
1764
|
-
run.downloadsSinceJournalCheckpoint = 0;
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
async function emitAndReportNewFiles(
|
|
1769
|
-
run: PullRunContext,
|
|
1770
|
-
plan: PullPlan,
|
|
1771
|
-
): Promise<void> {
|
|
1772
|
-
const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
|
|
1773
|
-
// Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
|
|
1774
|
-
// created-by HEADs below reuse the cache instead of each minting its own
|
|
1775
|
-
// presign. Without this, a big catch-up pull (hundreds of new files) bursts
|
|
1776
|
-
// the presign endpoint, trips the circuit breaker, and every enrichment HEAD
|
|
1777
|
-
// then fails. Mirrors the tombstone HEAD-verify pre-prime.
|
|
1778
|
-
await primeObjectTransport(
|
|
1779
|
-
run.ctx,
|
|
1780
|
-
"get",
|
|
1781
|
-
plan.newFiles.map((nf) => nf.path),
|
|
1782
|
-
);
|
|
1783
|
-
const HEAD_CONCURRENCY = 5;
|
|
1784
|
-
for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
|
|
1785
|
-
const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
|
|
1786
|
-
const results = await Promise.all(
|
|
1787
|
-
batch.map(async (nf) => {
|
|
1788
|
-
let addedBy: string | null = null;
|
|
1789
|
-
try {
|
|
1790
|
-
const head = await headRemoteFile(run.ctx, nf.path);
|
|
1791
|
-
if (head?.metadata?.["created-by"]) {
|
|
1792
|
-
addedBy = head.metadata["created-by"];
|
|
1793
|
-
}
|
|
1794
|
-
} catch (headErr) {
|
|
1795
|
-
if (headErr instanceof VaultAuthError) throw headErr;
|
|
1796
|
-
try {
|
|
1797
|
-
console.error(
|
|
1798
|
-
`[hq-sync] HeadObject failed for ${nf.path}: ${
|
|
1799
|
-
headErr instanceof Error ? headErr.message : String(headErr)
|
|
1800
|
-
}`,
|
|
1801
|
-
);
|
|
1802
|
-
} catch {
|
|
1803
|
-
// Swallow — logging must never break sync.
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
return { path: nf.path, bytes: nf.bytes, addedBy };
|
|
1807
|
-
}),
|
|
1808
|
-
);
|
|
1809
|
-
enrichedNewFiles.push(...results);
|
|
1810
|
-
}
|
|
1811
|
-
run.emit({ type: "new-files", files: enrichedNewFiles });
|
|
1812
|
-
await reportNewFilesToNotify(
|
|
1813
|
-
run.vaultConfig,
|
|
1814
|
-
run.ctx.uid,
|
|
1815
|
-
run.ctx.slug,
|
|
1816
|
-
enrichedNewFiles,
|
|
1817
|
-
run.options.telemetryClaims,
|
|
1818
|
-
);
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
|
-
async function verifyPlannedJournalTombstones(
|
|
1822
|
-
run: PullRunContext,
|
|
1823
|
-
plan: PullPlan,
|
|
1824
|
-
): Promise<void> {
|
|
1825
|
-
if (plan.tombstones.length === 0) return;
|
|
1826
|
-
|
|
1827
|
-
// Scope-invalid journal keys (company mode, `companies/…` prefix) skip the
|
|
1828
|
-
// HEAD verify: the presign transport's server validator rejects such keys
|
|
1829
|
-
// outright (INVALID_KEY_COMPANIES_SCOPED), so a HEAD can never succeed and
|
|
1830
|
-
// would defer the tombstone forever — leaving the poisoned journal entry
|
|
1831
|
-
// (and its doubled-tree local file) re-firing on every sync. The key can
|
|
1832
|
-
// never legitimately exist in a company vault, so treating it as verified
|
|
1833
|
-
// is safe: the executor cleans the local copy + journal entry instead of
|
|
1834
|
-
// the push leg ever re-uploading it (tombstone-not-reupload).
|
|
1835
|
-
const autoVerified: PullPlan["tombstones"] = [];
|
|
1836
|
-
const toHeadVerify: PullPlan["tombstones"] = [];
|
|
1837
|
-
for (const tombstone of plan.tombstones) {
|
|
1838
|
-
if (run.options.personalMode !== true && tombstone.key.startsWith("companies/")) {
|
|
1839
|
-
autoVerified.push(tombstone);
|
|
1840
|
-
} else {
|
|
1841
|
-
toHeadVerify.push(tombstone);
|
|
1842
|
-
}
|
|
1843
|
-
}
|
|
1844
|
-
plan.tombstones = toHeadVerify;
|
|
1845
|
-
if (plan.tombstones.length === 0) {
|
|
1846
|
-
plan.tombstones = autoVerified;
|
|
1847
|
-
return;
|
|
1848
|
-
}
|
|
1849
|
-
|
|
1850
|
-
await primeObjectTransport(run.ctx, "get", plan.tombstones.map((t) => t.key));
|
|
1851
|
-
|
|
1852
|
-
const HEAD_VERIFY_CONCURRENCY = 5;
|
|
1853
|
-
const verified: PullPlan["tombstones"] = [];
|
|
1854
|
-
for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
|
|
1855
|
-
const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
|
|
1856
|
-
const results = await Promise.all(
|
|
1857
|
-
batch.map(async (tombstone) => {
|
|
1858
|
-
try {
|
|
1859
|
-
const head = await headRemoteFile(run.ctx, tombstone.key);
|
|
1860
|
-
return head === null ? tombstone : null;
|
|
1861
|
-
} catch (err) {
|
|
1862
|
-
if (err instanceof VaultAuthError) throw err;
|
|
1863
|
-
if (isAccessDenied(err)) return null;
|
|
1864
|
-
run.emit({
|
|
1865
|
-
type: "error",
|
|
1866
|
-
path: tombstone.key,
|
|
1867
|
-
message: `tombstone HEAD verify failed (deferring): ${
|
|
1868
|
-
err instanceof Error ? err.message : String(err)
|
|
1869
|
-
}`,
|
|
1870
|
-
});
|
|
1871
|
-
return null;
|
|
1872
|
-
}
|
|
1873
|
-
}),
|
|
1874
|
-
);
|
|
1875
|
-
for (const tombstone of results) {
|
|
1876
|
-
if (tombstone !== null) verified.push(tombstone);
|
|
1877
|
-
}
|
|
1878
|
-
}
|
|
1879
|
-
plan.tombstones = [...autoVerified, ...verified];
|
|
1880
|
-
}
|
|
1881
|
-
|
|
1882
|
-
function executeJournalTombstoneDeletes(
|
|
1883
|
-
run: PullRunContext,
|
|
1884
|
-
plan: PullPlan,
|
|
1885
|
-
counters: PullCounters,
|
|
1886
|
-
): void {
|
|
1887
|
-
for (const tombstone of plan.tombstones) {
|
|
1888
|
-
const { key, localPath, localSnapshot } = tombstone;
|
|
1889
|
-
if (!isDownloadWritePathStillContained(run.companyRoot, key, localPath)) {
|
|
1890
|
-
run.emit({
|
|
1891
|
-
type: "error",
|
|
1892
|
-
path: key,
|
|
1893
|
-
message: "tombstone unlink skipped: local parent escaped the sync root",
|
|
1894
|
-
});
|
|
1895
|
-
continue;
|
|
1896
|
-
}
|
|
1897
|
-
let removedSomething = false;
|
|
1898
|
-
try {
|
|
1899
|
-
const lstat = fs.lstatSync(localPath);
|
|
1900
|
-
if (!matchesLocalSnapshot(localPath, localSnapshot)) {
|
|
1901
|
-
reportLocalSnapshotConflict(run, counters, key);
|
|
1902
|
-
continue;
|
|
1903
|
-
}
|
|
1904
|
-
if (lstat.isSymbolicLink() || lstat.isFile()) {
|
|
1905
|
-
fs.unlinkSync(localPath);
|
|
1906
|
-
removedSomething = true;
|
|
1907
|
-
} else if (lstat.isDirectory()) {
|
|
1908
|
-
// A dir at a key is converged by dropping only the journal entry.
|
|
1909
|
-
}
|
|
1910
|
-
} catch (err: unknown) {
|
|
1911
|
-
const code =
|
|
1912
|
-
err && typeof err === "object" && "code" in err
|
|
1913
|
-
? (err as { code?: string }).code
|
|
1914
|
-
: undefined;
|
|
1915
|
-
if (code !== "ENOENT") {
|
|
1916
|
-
run.emit({
|
|
1917
|
-
type: "error",
|
|
1918
|
-
path: key,
|
|
1919
|
-
message: `tombstone unlink failed: ${
|
|
1920
|
-
err instanceof Error ? err.message : String(err)
|
|
1921
|
-
}`,
|
|
1922
|
-
});
|
|
1923
|
-
continue;
|
|
1924
|
-
}
|
|
1925
|
-
}
|
|
1926
|
-
removeEntry(run.journal, key);
|
|
1927
|
-
counters.filesTombstoned++;
|
|
1928
|
-
counters.changedPaths.push(key);
|
|
1929
|
-
run.emit({
|
|
1930
|
-
type: "progress",
|
|
1931
|
-
path: key,
|
|
1932
|
-
bytes: 0,
|
|
1933
|
-
deleted: true,
|
|
1934
|
-
message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
|
|
1935
|
-
});
|
|
1936
|
-
}
|
|
1937
|
-
}
|
|
1938
|
-
|
|
1939
|
-
function finalizePullRun(
|
|
1940
|
-
run: PullRunContext,
|
|
1941
|
-
plan: PullPlan,
|
|
1942
|
-
scopeRun: ScopeShrinkRun,
|
|
1943
|
-
counters: PullCounters,
|
|
1944
|
-
): SyncResult {
|
|
1945
|
-
appendPullRecord(run.journal, {
|
|
1946
|
-
pullId: generatePullId(),
|
|
1947
|
-
companyUid: run.ctx.uid,
|
|
1948
|
-
startedAt: run.startedAt,
|
|
1949
|
-
completedAt: new Date().toISOString(),
|
|
1950
|
-
syncMode: run.syncMode,
|
|
1951
|
-
prefixSet: run.currentPrefixSet,
|
|
1952
|
-
scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
|
|
1953
|
-
orphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1954
|
-
orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1955
|
-
});
|
|
1956
|
-
|
|
1957
|
-
run.journal.lastSync = new Date().toISOString();
|
|
1958
|
-
writeJournal(run.journalSlug, run.journal);
|
|
1959
|
-
|
|
1960
|
-
const changedOnDisk =
|
|
1961
|
-
counters.filesDownloaded > 0 ||
|
|
1962
|
-
counters.filesTombstoned > 0 ||
|
|
1963
|
-
scopeRun.scopeOrphansRemoved > 0;
|
|
1964
|
-
if (!run.options.skipReindex && changedOnDisk) {
|
|
1965
|
-
try {
|
|
1966
|
-
const result = reindex({ repoRoot: run.hqRoot, skipLock: true });
|
|
1967
|
-
void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
|
|
1968
|
-
eventName: "reindex_run",
|
|
1969
|
-
source: "hq-sync",
|
|
1970
|
-
companyUid: run.ctx.uid,
|
|
1971
|
-
properties: {
|
|
1972
|
-
result: result.status === 0 ? "completed" : "failed",
|
|
1973
|
-
statusCode: result.status,
|
|
1974
|
-
lockResult: "skipped",
|
|
1975
|
-
changedCount:
|
|
1976
|
-
counters.changedPaths.length + scopeRun.changedPaths.length,
|
|
1977
|
-
},
|
|
1978
|
-
}, { claims: run.options.telemetryClaims });
|
|
1979
|
-
} catch (err) {
|
|
1980
|
-
logPostSyncMaintenanceFailure("post-sync reindex", run.hqRoot, err);
|
|
1981
|
-
// best-effort: a post-sync refresh failure never fails the sync
|
|
1982
|
-
}
|
|
1983
|
-
}
|
|
1984
|
-
|
|
1985
|
-
return {
|
|
1986
|
-
filesDownloaded: counters.filesDownloaded,
|
|
1987
|
-
bytesDownloaded: counters.bytesDownloaded,
|
|
1988
|
-
filesSkipped: counters.filesSkipped,
|
|
1989
|
-
conflicts: counters.conflicts,
|
|
1990
|
-
conflictPaths: counters.conflictPaths,
|
|
1991
|
-
aborted: false,
|
|
1992
|
-
newFiles: plan.newFiles,
|
|
1993
|
-
newFilesCount: plan.newFilesCount,
|
|
1994
|
-
filesExcludedByPolicy: plan.filesExcludedByPolicy,
|
|
1995
|
-
filesTombstoned: counters.filesTombstoned,
|
|
1996
|
-
filesOutOfScope: counters.filesOutOfScope,
|
|
1997
|
-
scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
|
|
1998
|
-
scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
|
|
1999
|
-
changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
|
|
2000
|
-
};
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
/**
|
|
2004
|
-
* Surface the shared-vs-all materialization gap (feedback_d2082110). When a
|
|
2005
|
-
* pull leg ran under a membership-scoped `syncMode` and skipped one or more
|
|
2006
|
-
* remote keys as out-of-scope, emit a single summary event so the operator
|
|
2007
|
-
* SEES that not all vault content materialized on this device and knows the
|
|
2008
|
-
* lever — raise the membership's access level to `all`. Silent in `all` mode
|
|
2009
|
-
* (nothing is scoped away) and when the membership scope withheld nothing: no
|
|
2010
|
-
* gap, no noise. Pure — count and samples are both read off the plan's
|
|
2011
|
-
* `skip-out-of-scope` items, no I/O.
|
|
2012
|
-
*
|
|
2013
|
-
* Counts ONLY `reason: "membership-scope"` items. `push-only` keys
|
|
2014
|
-
* (`sessions/`, US-006) are also classified `skip-out-of-scope` and also land
|
|
2015
|
-
* on the `filesOutOfScope` axis, but they are withheld in EVERY mode — `all`
|
|
2016
|
-
* included — so raising the access level does NOT materialize them. Counting
|
|
2017
|
-
* them here would inflate the gap, could fill `samplePaths` entirely with
|
|
2018
|
-
* session keys, and could fire the event (and its "set this membership to
|
|
2019
|
-
* `all`" advice) when the membership scope withheld nothing at all — sending
|
|
2020
|
-
* the operator back into exactly the flip-everything-to-`all`-and-still-miss
|
|
2021
|
-
* -files loop this event exists to end. `SyncResult.filesOutOfScope` keeps its
|
|
2022
|
-
* original both-causes meaning; only this advisory is narrowed.
|
|
2023
|
-
*/
|
|
2024
|
-
function emitScopeMaterializationGap(run: PullRunContext, plan: PullPlan): void {
|
|
2025
|
-
if (run.syncMode === "all") return;
|
|
2026
|
-
const withheldByMembership = plan.items.filter(
|
|
2027
|
-
(item) => item.action === "skip-out-of-scope" && item.reason === "membership-scope",
|
|
2028
|
-
);
|
|
2029
|
-
if (withheldByMembership.length === 0) return;
|
|
2030
|
-
run.emit({
|
|
2031
|
-
type: "scope-materialization-gap",
|
|
2032
|
-
count: withheldByMembership.length,
|
|
2033
|
-
samplePaths: withheldByMembership.slice(0, 10).map((item) => item.remoteFile.key),
|
|
2034
|
-
syncMode: run.syncMode,
|
|
2035
|
-
});
|
|
2036
|
-
}
|
|
2037
|
-
|
|
2038
|
-
/**
|
|
2039
|
-
* Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
|
|
2040
|
-
* written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
|
|
2041
|
-
* object must still sync (the tombstone is not permanent suppression). Returns
|
|
2042
|
-
* true to ALLOW the download (re-create), false to honor the tombstone.
|
|
2043
|
-
*
|
|
2044
|
-
* Decisive signal: the remote object's `lastModified` strictly newer than the
|
|
2045
|
-
* tombstone's `deletedAt`. A delete-marker + tombstone are written together at
|
|
2046
|
-
* delete time, so any object that post-dates the tombstone is a new write at
|
|
2047
|
-
* that key. An object at-or-before the tombstone is the deleted version (a
|
|
2048
|
-
* stale re-push or an un-propagated delete-marker) → honor the tombstone.
|
|
2049
|
-
*
|
|
2050
|
-
* Fail-OPEN (treat as re-create, allow download) when the comparison can't be
|
|
2051
|
-
* made — a malformed `deletedAt`, or no remote `lastModified`. Hiding a file the
|
|
2052
|
-
* user can see is worse than re-pulling one they deleted; the latter is
|
|
2053
|
-
* recoverable, the former looks like data loss.
|
|
2054
|
-
*/
|
|
2055
|
-
function isRemoteRecreateAfterTombstone(
|
|
2056
|
-
remote: { lastModified?: Date },
|
|
2057
|
-
tombstone: CompanyTombstone,
|
|
2058
|
-
): boolean {
|
|
2059
|
-
const deletedAtMs = Date.parse(tombstone.deletedAt);
|
|
2060
|
-
if (Number.isNaN(deletedAtMs)) return true; // malformed tombstone → don't suppress
|
|
2061
|
-
const remoteMs =
|
|
2062
|
-
remote.lastModified instanceof Date
|
|
2063
|
-
? remote.lastModified.getTime()
|
|
2064
|
-
: NaN;
|
|
2065
|
-
if (Number.isNaN(remoteMs)) return true; // no remote timestamp → don't suppress
|
|
2066
|
-
return remoteMs > deletedAtMs;
|
|
2067
|
-
}
|
|
2068
|
-
|
|
2069
|
-
function hasTraversalSegment(key: string): boolean {
|
|
2070
|
-
return key.split("/").some((segment) => segment === "..");
|
|
2071
|
-
}
|
|
2072
|
-
|
|
2073
|
-
function isPathWithin(root: string, candidate: string): boolean {
|
|
2074
|
-
const relative = path.relative(root, candidate);
|
|
2075
|
-
return (
|
|
2076
|
-
relative === "" ||
|
|
2077
|
-
(!relative.startsWith("..") && !path.isAbsolute(relative))
|
|
2078
|
-
);
|
|
2079
|
-
}
|
|
2080
|
-
|
|
2081
|
-
function deepestExistingAncestor(start: string): string | null {
|
|
2082
|
-
let current = start;
|
|
2083
|
-
for (;;) {
|
|
2084
|
-
try {
|
|
2085
|
-
fs.lstatSync(current);
|
|
2086
|
-
return current;
|
|
2087
|
-
} catch (err: unknown) {
|
|
2088
|
-
const code =
|
|
2089
|
-
err && typeof err === "object" && "code" in err
|
|
2090
|
-
? (err as { code?: string }).code
|
|
2091
|
-
: undefined;
|
|
2092
|
-
if (code !== "ENOENT" && code !== "ENOTDIR") return null;
|
|
2093
|
-
}
|
|
2094
|
-
|
|
2095
|
-
const parent = path.dirname(current);
|
|
2096
|
-
if (parent === current) return null;
|
|
2097
|
-
current = parent;
|
|
2098
|
-
}
|
|
2099
|
-
}
|
|
2100
|
-
|
|
2101
|
-
function resolveContainedVaultPath(root: string, key: string): string | null {
|
|
2102
|
-
if (isMalformedVaultKey(key) || hasTraversalSegment(key)) return null;
|
|
2103
|
-
|
|
2104
|
-
const resolvedRoot = path.resolve(root);
|
|
2105
|
-
const resolvedLocal = localPathForVaultKey(resolvedRoot, key);
|
|
2106
|
-
if (!isPathWithin(resolvedRoot, resolvedLocal)) return null;
|
|
2107
|
-
|
|
2108
|
-
let realRoot: string;
|
|
2109
|
-
try {
|
|
2110
|
-
realRoot = fs.realpathSync.native(resolvedRoot);
|
|
2111
|
-
} catch {
|
|
2112
|
-
// If the vault root does not exist yet, no below-root symlink component can
|
|
2113
|
-
// already exist to redirect this key. Preserve first-pull behavior.
|
|
2114
|
-
return resolvedLocal;
|
|
2115
|
-
}
|
|
2116
|
-
|
|
2117
|
-
const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
|
|
2118
|
-
if (existingAncestor === null) return null;
|
|
2119
|
-
try {
|
|
2120
|
-
const realAncestor = fs.realpathSync.native(existingAncestor);
|
|
2121
|
-
if (!isPathWithin(realRoot, realAncestor)) return null;
|
|
2122
|
-
} catch {
|
|
2123
|
-
return null;
|
|
2124
|
-
}
|
|
2125
|
-
return resolvedLocal;
|
|
2126
|
-
}
|
|
2127
|
-
|
|
2128
|
-
function isDownloadWritePathStillContained(
|
|
2129
|
-
root: string,
|
|
2130
|
-
key: string,
|
|
2131
|
-
localPath: string,
|
|
2132
|
-
): boolean {
|
|
2133
|
-
const resolved = resolveContainedVaultPath(root, key);
|
|
2134
|
-
return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
function snapshotLocalState(localPath: string): LocalSnapshot | null {
|
|
2138
|
-
try {
|
|
2139
|
-
const lstat = fs.lstatSync(localPath);
|
|
2140
|
-
if (lstat.isSymbolicLink()) {
|
|
2141
|
-
return { kind: "symlink", hash: hashSymlinkTarget(fs.readlinkSync(localPath)) };
|
|
2142
|
-
}
|
|
2143
|
-
if (lstat.isFile()) {
|
|
2144
|
-
return { kind: "file", hash: hashFile(localPath) };
|
|
2145
|
-
}
|
|
2146
|
-
if (lstat.isDirectory()) return { kind: "directory" };
|
|
2147
|
-
return { kind: "other" };
|
|
2148
|
-
} catch (err: unknown) {
|
|
2149
|
-
const code =
|
|
2150
|
-
err && typeof err === "object" && "code" in err
|
|
2151
|
-
? (err as { code?: string }).code
|
|
2152
|
-
: undefined;
|
|
2153
|
-
return code === "ENOENT" ? { kind: "absent" } : null;
|
|
2154
|
-
}
|
|
2155
|
-
}
|
|
2156
|
-
|
|
2157
|
-
function matchesLocalSnapshot(localPath: string, expected: LocalSnapshot): boolean {
|
|
2158
|
-
const actual = snapshotLocalState(localPath);
|
|
2159
|
-
if (actual === null || actual.kind !== expected.kind) return false;
|
|
2160
|
-
return (
|
|
2161
|
-
(actual.kind !== "file" && actual.kind !== "symlink") ||
|
|
2162
|
-
actual.hash === (expected as Extract<LocalSnapshot, { hash: string }>).hash
|
|
2163
|
-
);
|
|
2164
|
-
}
|
|
2165
|
-
|
|
2166
|
-
function hasCurrentLocalDeleteIntent(
|
|
2167
|
-
entry: SyncJournal["files"][string] | undefined,
|
|
2168
|
-
): boolean {
|
|
2169
|
-
const intent = entry?.localDeleteIntent;
|
|
2170
|
-
return !!(
|
|
2171
|
-
intent &&
|
|
2172
|
-
intent.version === 1 &&
|
|
2173
|
-
entry.remoteEtag &&
|
|
2174
|
-
entry.kind &&
|
|
2175
|
-
intent.remoteEtag === entry.remoteEtag &&
|
|
2176
|
-
intent.localHash === entry.hash &&
|
|
2177
|
-
intent.localKind === entry.kind
|
|
2178
|
-
);
|
|
2179
|
-
}
|
|
2180
|
-
|
|
2181
|
-
function reportLocalSnapshotConflict(
|
|
2182
|
-
run: PullRunContext,
|
|
2183
|
-
counters: PullCounters,
|
|
2184
|
-
key: string,
|
|
2185
|
-
): void {
|
|
2186
|
-
counters.conflicts++;
|
|
2187
|
-
counters.conflictPaths.push(key);
|
|
2188
|
-
run.emit({ type: "conflict", path: key, direction: "pull", resolution: "skip" });
|
|
2189
|
-
}
|
|
2190
|
-
|
|
2191
|
-
class LocalSnapshotChangedError extends Error {
|
|
2192
|
-
constructor(key: string) {
|
|
2193
|
-
super(`local state changed while downloading ${key}; replanning required`);
|
|
2194
|
-
}
|
|
2195
|
-
}
|
|
2196
|
-
|
|
2197
|
-
/**
|
|
2198
|
-
* Stage-1 classification for a single remote object. Each remote file falls
|
|
2199
|
-
* into exactly one bucket; the executor in `sync()` switches on `action` to
|
|
2200
|
-
* decide what to do. `localHash` is carried on `conflict` items so the
|
|
2201
|
-
* executor can hand it to `resolveConflict` without re-hashing.
|
|
2202
|
-
*/
|
|
2203
|
-
type LocalSnapshot =
|
|
2204
|
-
| { kind: "absent" | "directory" | "other" }
|
|
2205
|
-
| { kind: "file" | "symlink"; hash: string };
|
|
2206
|
-
|
|
2207
|
-
/**
|
|
2208
|
-
* Why a remote key was classified `skip-out-of-scope`. The two causes look
|
|
2209
|
-
* identical in the count but have OPPOSITE remedies, so they must never be
|
|
2210
|
-
* reported as one: `membership-scope` is fixed by raising the membership's
|
|
2211
|
-
* access level to `all`; `push-only` is not fixed by that at all.
|
|
2212
|
-
*/
|
|
2213
|
-
type OutOfScopeReason = "membership-scope" | "push-only";
|
|
2214
|
-
|
|
2215
|
-
type PullPlanItem =
|
|
2216
|
-
| {
|
|
2217
|
-
action: "download";
|
|
2218
|
-
remoteFile: RemoteFile;
|
|
2219
|
-
localPath: string;
|
|
2220
|
-
isNew: boolean;
|
|
2221
|
-
localSnapshot: LocalSnapshot;
|
|
2222
|
-
}
|
|
2223
|
-
| { action: "skip-ignored"; remoteFile: RemoteFile; localPath: string }
|
|
2224
|
-
| { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
|
|
2225
|
-
| { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
|
|
2226
|
-
| { action: "skip-local-only"; remoteFile: RemoteFile; localPath: string }
|
|
2227
|
-
// A local symlink was classified as a link but Windows (or a filesystem
|
|
2228
|
-
// race) would not expose its target. This stays distinct from generic
|
|
2229
|
-
// skip-local-only so the executor can report the exact path without making
|
|
2230
|
-
// the whole company leg fail.
|
|
2231
|
-
| { action: "skip-unreadable-link"; remoteFile: RemoteFile; localPath: string }
|
|
2232
|
-
// A stale personal-overlay symlink marker in the vault sitting at a key that
|
|
2233
|
-
// is ALSO a real, release-shipped core directory (e.g. an old
|
|
2234
|
-
// `core/knowledge/public/agent-browser` overlay marker after a release
|
|
2235
|
-
// shipped a real directory of the same name). The local release-shipped
|
|
2236
|
-
// directory is authoritative; the marker is inert derived state that must
|
|
2237
|
-
// NEVER overwrite core. Distinct from skip-local-only so the benign,
|
|
2238
|
-
// no-action-needed case is never surfaced with the alarming "rm -rf the
|
|
2239
|
-
// local directory" reconciliation advice.
|
|
2240
|
-
| { action: "skip-stale-overlay-marker"; remoteFile: RemoteFile; localPath: string }
|
|
2241
|
-
// A directory-overlay marker (a single vault object at a key that ALSO has
|
|
2242
|
-
// child objects under it in the same LIST — a symlink record for a directory
|
|
2243
|
-
// mirrored from another tree) colliding with a local REAL directory whose
|
|
2244
|
-
// contents are already materialized. The marker can never be renamed over the
|
|
2245
|
-
// directory; its children carry the real content and materialize via their own
|
|
2246
|
-
// plan items. A benign, no-action skip — distinct from skip-local-only so the
|
|
2247
|
-
// case is never surfaced with the alarming (and, for a materialized directory,
|
|
2248
|
-
// data-destroying) "rm -rf the local directory to pull" advice. feedback_d2082110.
|
|
2249
|
-
| { action: "skip-overlay-marker-with-children"; remoteFile: RemoteFile; localPath: string }
|
|
2250
|
-
// Remote keys refused by ephemeral-mirror policy. The push walker has
|
|
2251
|
-
// refused to upload these since 5.33.0; the pull walker now refuses to
|
|
2252
|
-
// download them so legacy litter in cloud staging drains naturally.
|
|
2253
|
-
| { action: "skip-excluded-policy"; remoteFile: RemoteFile; localPath: string }
|
|
2254
|
-
// Remote keys present in the remote LIST (and accessible per STS) but
|
|
2255
|
-
// deliberately not downloaded. Two DIFFERENT causes share this action and
|
|
2256
|
-
// the `filesOutOfScope` count, so `reason` discriminates them:
|
|
2257
|
-
//
|
|
2258
|
-
// `membership-scope` — outside the effective `syncMode` prefix set
|
|
2259
|
-
// (US-005). RAISING the membership's access level to `all` WOULD
|
|
2260
|
-
// materialize these keys.
|
|
2261
|
-
// `push-only` — under a push-only exclude prefix (`sessions/`,
|
|
2262
|
-
// US-006). These are withheld in EVERY mode, `all` included; raising
|
|
2263
|
-
// the access level does NOT materialize them (`hq files get` does).
|
|
2264
|
-
//
|
|
2265
|
-
// Only `membership-scope` items may be attributed to the shared-vs-all gap
|
|
2266
|
-
// — see `emitScopeMaterializationGap`. feedback_d2082110.
|
|
2267
|
-
| {
|
|
2268
|
-
action: "skip-out-of-scope";
|
|
2269
|
-
remoteFile: RemoteFile;
|
|
2270
|
-
localPath: string;
|
|
2271
|
-
reason: OutOfScopeReason;
|
|
2272
|
-
}
|
|
2273
|
-
| {
|
|
2274
|
-
// Mixed-version amplifier guard — see the `skip-junk-key-spelling`
|
|
2275
|
-
// event doc. Carries the journaled spelling for the warning surface.
|
|
2276
|
-
action: "skip-junk-key-spelling";
|
|
2277
|
-
remoteFile: RemoteFile;
|
|
2278
|
-
localPath: string;
|
|
2279
|
-
journaledKey: string;
|
|
2280
|
-
}
|
|
2281
|
-
// Remote key present in the LIST but carrying a FILE_TOMBSTONE that marks it
|
|
2282
|
-
// intentionally deleted (and the remote object is NOT a newer re-create). The
|
|
2283
|
-
// executor deletes any local copy and drops the journal entry — the
|
|
2284
|
-
// authoritative-delete arm of delete-resync. Unlike `PullPlan.tombstones`
|
|
2285
|
-
// (keys ABSENT from the LIST, HEAD-verified before delete), these are
|
|
2286
|
-
// suppressed purely on the FILE_TOMBSTONE authority and skip HEAD-verify (the
|
|
2287
|
-
// remote object is present by definition).
|
|
2288
|
-
| {
|
|
2289
|
-
action: "tombstone-delete";
|
|
2290
|
-
remoteFile: RemoteFile;
|
|
2291
|
-
localPath: string;
|
|
2292
|
-
localSnapshot: LocalSnapshot;
|
|
2293
|
-
}
|
|
2294
|
-
| {
|
|
2295
|
-
action: "conflict";
|
|
2296
|
-
remoteFile: RemoteFile;
|
|
2297
|
-
localPath: string;
|
|
2298
|
-
localHash: string;
|
|
2299
|
-
// Captured from the planner's lstat (does NOT follow symlinks),
|
|
2300
|
-
// so a dangling symlink doesn't make the conflict executor
|
|
2301
|
-
// statSync-then-ENOENT and abort the whole pull. See the
|
|
2302
|
-
// executor: prior to this field, the prompt builder called
|
|
2303
|
-
// fs.statSync(localPath).mtime to populate the conflict UI's
|
|
2304
|
-
// "local modified at X" line, which crashed for dangling links.
|
|
2305
|
-
localMtime: Date;
|
|
2306
|
-
// Symlink-aware size for the post-resolution journal stamp.
|
|
2307
|
-
// For a symlink, journal convention is 0 (the wire body is
|
|
2308
|
-
// empty + target metadata; matches what the push side stamps).
|
|
2309
|
-
// For a regular file, mirrors the on-disk byte length. The
|
|
2310
|
-
// executor's keep/skip branch reads this instead of calling
|
|
2311
|
-
// fs.statSync(localPath), which would follow a dangling link
|
|
2312
|
-
// and silently fail to stamp the journal — leaving the
|
|
2313
|
-
// conflict to re-fire forever.
|
|
2314
|
-
localSize: number;
|
|
2315
|
-
localSnapshot: LocalSnapshot;
|
|
2316
|
-
};
|
|
2317
|
-
|
|
2318
|
-
interface PullPlan {
|
|
2319
|
-
items: PullPlanItem[];
|
|
2320
|
-
filesToDownload: number;
|
|
2321
|
-
bytesToDownload: number;
|
|
2322
|
-
filesToSkip: number;
|
|
2323
|
-
filesToConflict: number;
|
|
2324
|
-
/** Files classified as new (no local counterpart at classification time). */
|
|
2325
|
-
newFiles: Array<{ path: string; bytes: number }>;
|
|
2326
|
-
newFilesCount: number;
|
|
2327
|
-
/** Count of remote keys refused by ephemeral-mirror policy. */
|
|
2328
|
-
filesExcludedByPolicy: number;
|
|
2329
|
-
/** Count of remote keys skipped because they fall outside the sync scope. */
|
|
2330
|
-
filesOutOfScope: number;
|
|
2331
|
-
/**
|
|
2332
|
-
* Journal-known keys missing from the remote LIST. The executor will
|
|
2333
|
-
* apply each as a local delete (file or symlink) + journal removal,
|
|
2334
|
-
* propagating the peer's push-side delete cross-machine (Bug #9).
|
|
2335
|
-
* Carried on the plan so the executor can iterate without re-walking.
|
|
2336
|
-
*/
|
|
2337
|
-
tombstones: Array<{
|
|
2338
|
-
key: string;
|
|
2339
|
-
localPath: string;
|
|
2340
|
-
localSnapshot: LocalSnapshot;
|
|
2341
|
-
}>;
|
|
2342
|
-
/**
|
|
2343
|
-
* Count of `tombstone-delete` items — remote keys present in the LIST but
|
|
2344
|
-
* suppressed by a FILE_TOMBSTONE (delete-resync). Surfaced on the plan event's
|
|
2345
|
-
* `filesToDelete` axis; the per-item executor applies the local delete.
|
|
2346
|
-
*/
|
|
2347
|
-
filesToTombstoneDelete: number;
|
|
2348
|
-
/**
|
|
2349
|
-
* Remote keys skipped because the server will never presign them. Counted in
|
|
2350
|
-
* `filesExcludedByPolicy`; carried with the server-compatible error code so
|
|
2351
|
-
* the caller can log one warning per key and emit telemetry. Healthy
|
|
2352
|
-
* baseline is an empty array (policy
|
|
2353
|
-
* hq-alert-baseline-calibration: zero expected, any occurrence is signal).
|
|
2354
|
-
*/
|
|
2355
|
-
invalidRemoteKeys: Array<{ path: string; errorCode: string }>;
|
|
2356
|
-
}
|
|
2357
|
-
|
|
2358
|
-
/**
|
|
2359
|
-
* Stage-1 planning pass: classify every remote file into download / skip /
|
|
2360
|
-
* conflict buckets without performing any S3 transfers. Local hashes are
|
|
2361
|
-
* computed here (not in the transfer loop) so the totals returned reflect
|
|
2362
|
-
* the real outcome of the upcoming Stage-2 execution rather than an
|
|
2363
|
-
* upper-bound guess.
|
|
2364
|
-
*
|
|
2365
|
-
* Pure function: no S3 calls, no journal writes, no event emission. The
|
|
2366
|
-
* caller (`sync()`) is responsible for emitting the resulting plan event
|
|
2367
|
-
* before iterating `items`.
|
|
2368
|
-
*/
|
|
2369
|
-
// Pull-leg intentional-delete bulk-asymmetry breaker. Mirrors the push leg's
|
|
2370
|
-
// BULK_ASYMMETRY_* (share.ts): an abnormally large fraction of clean files
|
|
2371
|
-
// vanishing at once is a corrupt mirror / bulk op / unmount (the ridge-incident
|
|
2372
|
-
// class), not intent — so at/above the threshold the pull leg drift-restores
|
|
2373
|
-
// every candidate instead of tombstoning it. Below it, a missing clean file is
|
|
2374
|
-
// an intentional delete.
|
|
2375
|
-
const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
|
|
2376
|
-
const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
|
|
2377
|
-
|
|
2378
|
-
/**
|
|
2379
|
-
* win32 only: true when an existing local symlink's TARGET exists but the
|
|
2380
|
-
* link cannot be traversed as that target's type — the file-flavor-link-at-
|
|
2381
|
-
* a-directory shape minted while the target was absent (or by an older
|
|
2382
|
-
* client). A dangling link is NOT broken-flavored (there is nothing to
|
|
2383
|
-
* repair until the target appears). POSIX links are flavorless: always
|
|
2384
|
-
* false off win32.
|
|
2385
|
-
*/
|
|
2386
|
-
export function win32SymlinkFlavorBroken(
|
|
2387
|
-
localPath: string,
|
|
2388
|
-
win32: boolean = process.platform === "win32",
|
|
2389
|
-
): boolean {
|
|
2390
|
-
if (!win32) return false;
|
|
2391
|
-
let target: string;
|
|
2392
|
-
try {
|
|
2393
|
-
target = fs.readlinkSync(localPath);
|
|
2394
|
-
} catch {
|
|
2395
|
-
return false;
|
|
2396
|
-
}
|
|
2397
|
-
const resolved = path.resolve(path.dirname(localPath), target);
|
|
2398
|
-
let targetIsDir: boolean;
|
|
2399
|
-
try {
|
|
2400
|
-
targetIsDir = fs.statSync(resolved).isDirectory();
|
|
2401
|
-
} catch {
|
|
2402
|
-
return false; // dangling target — nothing to repair yet
|
|
2403
|
-
}
|
|
2404
|
-
try {
|
|
2405
|
-
return fs.statSync(localPath).isDirectory() !== targetIsDir;
|
|
2406
|
-
} catch {
|
|
2407
|
-
return true; // target exists but the link cannot traverse: wrong flavor
|
|
2408
|
-
}
|
|
2409
|
-
}
|
|
2410
|
-
|
|
2411
|
-
function computePullPlan(
|
|
2412
|
-
remoteFiles: RemoteFile[],
|
|
2413
|
-
journal: SyncJournal,
|
|
2414
|
-
companyRoot: string,
|
|
2415
|
-
shouldSync: (filePath: string, isDir?: boolean) => boolean,
|
|
2416
|
-
personalMode: boolean,
|
|
2417
|
-
teamSyncedSlugs: ReadonlySet<string> | null,
|
|
2418
|
-
// Coalesced, company-relative prefixes the pull is scoped to (US-005).
|
|
2419
|
-
// `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
|
|
2420
|
-
// the scope filter below becomes a no-op and legacy behavior is preserved.
|
|
2421
|
-
prefixSet: readonly ScopePrefixInput[],
|
|
2422
|
-
// FILE_TOMBSTONE records (POSIX-keyed) for the company — the durable
|
|
2423
|
-
// "this key was intentionally deleted" signal the planner consults before
|
|
2424
|
-
// re-downloading a key, so a deleted folder does not resync back in
|
|
2425
|
-
// (delete-resync). An empty map (the default / degraded-fetch case)
|
|
2426
|
-
// preserves the legacy behavior bit-for-bit. (Named `fileTombstones` to avoid
|
|
2427
|
-
// shadowing the local `tombstones` string[] — the journal-vs-LIST delete set.)
|
|
2428
|
-
fileTombstones: ReadonlyMap<string, CompanyTombstone> = new Map(),
|
|
2429
|
-
// Push-only exclusion prefixes (US-006: `sessions/`). Keys covered by any of
|
|
2430
|
-
// these are classified `skip-out-of-scope` in EVERY mode — including `all` —
|
|
2431
|
-
// AFTER passing the inclusion check. Empty (the default) preserves legacy
|
|
2432
|
-
// behavior bit-for-bit. Applied ONLY here (the download filter), never to
|
|
2433
|
-
// scope-shrink, so an authored/pinned excluded key is not re-pulled yet also
|
|
2434
|
-
// never orphaned.
|
|
2435
|
-
excludePrefixes: readonly ScopePrefixInput[] = [],
|
|
2436
|
-
): PullPlan {
|
|
2437
|
-
const items: PullPlanItem[] = [];
|
|
2438
|
-
// Remote keys the server will never presign — collected so the caller can
|
|
2439
|
-
// warn + emit telemetry without allowing one poison object to fail a pull.
|
|
2440
|
-
const invalidRemoteKeys: Array<{ path: string; errorCode: string }> = [];
|
|
2441
|
-
// Clean/current/non-divergent files missing locally — collected here, then
|
|
2442
|
-
// resolved post-loop by the bulk-asymmetry decision (tombstone vs. restore).
|
|
2443
|
-
const intentionalDeleteCandidates: Array<{
|
|
2444
|
-
remoteFile: RemoteFile;
|
|
2445
|
-
localPath: string;
|
|
2446
|
-
}> = [];
|
|
2447
|
-
|
|
2448
|
-
// Junk-spelling guard (mixed-version amplifier): index every journaled key
|
|
2449
|
-
// by its canonical (percent-decode fixpoint) form so a remote twin minted
|
|
2450
|
-
// under a different spelling is recognized before it materializes. When
|
|
2451
|
-
// several journaled spellings already share one canonical form (a
|
|
2452
|
-
// pre-guard fork), the exactly-canonical spelling is preferred as the
|
|
2453
|
-
// representative; entries journaled under their own exact spelling are
|
|
2454
|
-
// never gated by this map (see the guard below), so pre-existing families
|
|
2455
|
-
// keep syncing untouched until the vault doctor (US-003) collapses them.
|
|
2456
|
-
const journaledKeyByCanonical = new Map<string, string>();
|
|
2457
|
-
for (const journaledKey of Object.keys(journal.files)) {
|
|
2458
|
-
if (!journaledKey.startsWith(SKILLS_KEY_PREFIX)) continue;
|
|
2459
|
-
const canonical = canonicalVaultKeySpelling(journaledKey);
|
|
2460
|
-
const existing = journaledKeyByCanonical.get(canonical);
|
|
2461
|
-
if (existing === undefined || journaledKey === canonical) {
|
|
2462
|
-
journaledKeyByCanonical.set(canonical, journaledKey);
|
|
2463
|
-
}
|
|
2464
|
-
}
|
|
2465
|
-
|
|
2466
|
-
// Remote keys that ALSO appear as an ancestor of another remote key — i.e.
|
|
2467
|
-
// keys that carry child objects under `${key}/…` in this same LIST. The
|
|
2468
|
-
// vault stores a directory OVERLAY (a symlink into another tree, e.g. a
|
|
2469
|
-
// company skill mirrored from a shared repo) as a symlink RECORD at the
|
|
2470
|
-
// directory's own key AND stores the directory's contents as separate child
|
|
2471
|
-
// objects beneath it. A REAL directory, by contrast, only ever syncs as
|
|
2472
|
-
// those child objects — never as a single object AT its own key. So a single
|
|
2473
|
-
// remote object sitting at a key that also has children is unambiguously a
|
|
2474
|
-
// directory-overlay marker, not a regular file. Precomputed once (pure path
|
|
2475
|
-
// derivation, no I/O — the planner is synchronous) so the dir-vs-object
|
|
2476
|
-
// collision branch can tell an inert overlay marker apart from a genuine
|
|
2477
|
-
// file-vs-directory structural collision. feedback_d2082110.
|
|
2478
|
-
const remoteKeysWithChildren = new Set<string>();
|
|
2479
|
-
for (const rf of remoteFiles) {
|
|
2480
|
-
const key = toPosixKey(rf.key);
|
|
2481
|
-
let slash = key.indexOf("/");
|
|
2482
|
-
while (slash !== -1) {
|
|
2483
|
-
remoteKeysWithChildren.add(key.slice(0, slash));
|
|
2484
|
-
slash = key.indexOf("/", slash + 1);
|
|
2485
|
-
}
|
|
2486
|
-
}
|
|
2487
|
-
|
|
2488
|
-
for (const remoteFile of remoteFiles) {
|
|
2489
|
-
const invalidKey = classifyVaultKey(
|
|
2490
|
-
remoteFile.key,
|
|
2491
|
-
personalMode ? "personal" : "company",
|
|
2492
|
-
);
|
|
2493
|
-
if (invalidKey !== null) {
|
|
2494
|
-
invalidRemoteKeys.push({
|
|
2495
|
-
path: remoteFile.key,
|
|
2496
|
-
errorCode: invalidKey.code,
|
|
2497
|
-
});
|
|
2498
|
-
items.push({
|
|
2499
|
-
action: "skip-excluded-policy",
|
|
2500
|
-
remoteFile,
|
|
2501
|
-
localPath: companyRoot,
|
|
2502
|
-
});
|
|
2503
|
-
continue;
|
|
2504
|
-
}
|
|
2505
|
-
|
|
2506
|
-
const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
|
|
2507
|
-
if (localPath === null) {
|
|
2508
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
|
|
2509
|
-
continue;
|
|
2510
|
-
}
|
|
2511
|
-
|
|
2512
|
-
// Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
|
|
2513
|
-
// in the 5.33.0 deep-test: the push side has refused to upload conflict
|
|
2514
|
-
// mirrors since 5.33.0, but the pull side downloaded them freely from
|
|
2515
|
-
// staging, so legacy `.conflict-*` litter rode every sync into clean
|
|
2516
|
-
// trees. Refuse them here; the V2 verification test (direct S3 inject
|
|
2517
|
-
// bypassing the push filter entirely) is the regression contract.
|
|
2518
|
-
if (isEphemeralPath(remoteFile.key)) {
|
|
2519
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2520
|
-
continue;
|
|
2521
|
-
}
|
|
2522
|
-
|
|
2523
|
-
// Company-vault double-scoped-key filter. A company vault is already
|
|
2524
|
-
// anchored at its company root, so a remote key under `companies/...` is a
|
|
2525
|
-
// corrupt doubly-scoped object the vault-service refuses to presign
|
|
2526
|
-
// (INVALID_KEY_COMPANIES_SCOPED). Without this, the puller enqueues it, the
|
|
2527
|
-
// per-file GET fails, and the whole company wedges at "errored" (runner
|
|
2528
|
-
// exit 2) on every run (frogbear, 2026-06-16). Refuse at plan time,
|
|
2529
|
-
// symmetric with the malformed/ephemeral filters above. Personal vaults
|
|
2530
|
-
// handle `companies/...` keys in the dedicated branch below.
|
|
2531
|
-
if (isForbiddenCompanyVaultKey(remoteFile.key, personalMode)) {
|
|
2532
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2533
|
-
continue;
|
|
2534
|
-
}
|
|
2535
|
-
|
|
2536
|
-
// Personal-vault policy must be symmetric across both transfer legs.
|
|
2537
|
-
// The push walker refuses derived/machine-local/sensitive paths, but old
|
|
2538
|
-
// objects can remain in the bucket from clients that predate an exclusion.
|
|
2539
|
-
// Refuse those objects here too so a pull cannot rematerialize them (or,
|
|
2540
|
-
// for generated skill wrappers, fight reindex over a directory link).
|
|
2541
|
-
if (personalMode && isPersonalVaultExcluded(remoteFile.key)) {
|
|
2542
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2543
|
-
continue;
|
|
2544
|
-
}
|
|
2545
|
-
|
|
2546
|
-
if (
|
|
2547
|
-
personalMode &&
|
|
2548
|
-
remoteFile.key.startsWith("companies/") &&
|
|
2549
|
-
// EXEMPTION: companies/manifest.yaml is the routing source-of-truth
|
|
2550
|
-
// carved INTO the personal vault on the push side
|
|
2551
|
-
// (computePersonalVaultPaths). It must round-trip on the pull leg too —
|
|
2552
|
-
// skipping it here leaves it forever unjournaled, which re-fires a
|
|
2553
|
-
// transient push-side conflict every sync (no journal baseline). Let it
|
|
2554
|
-
// fall through to download + journal like any personal file.
|
|
2555
|
-
remoteFile.key !== PERSONAL_VAULT_MANIFEST_KEY
|
|
2556
|
-
) {
|
|
2557
|
-
// Local (non-cloud) companies sync to the personal vault: keys under
|
|
2558
|
-
// `companies/{slug}/...` are legitimate content a peer machine pushed
|
|
2559
|
-
// for a company that is not cloud-backed. Allow them through EXCEPT
|
|
2560
|
-
// for slugs the operator has an active team-bucket Membership for:
|
|
2561
|
-
// those are orphan remnants from before the company became cloud-true,
|
|
2562
|
-
// and downloading them would clash with the team-bucket pull at the
|
|
2563
|
-
// same disk path.
|
|
2564
|
-
//
|
|
2565
|
-
// Symmetric to the push-side `decommissionPrefixes` logic in
|
|
2566
|
-
// share.ts — both target the same orphan class, both honor the
|
|
2567
|
-
// same `teamSyncedSlugs` set derived from the operator's live
|
|
2568
|
-
// membership at runtime.
|
|
2569
|
-
const slug = remoteFile.key.split("/")[1] ?? "";
|
|
2570
|
-
const isTeamSyncedOrphan =
|
|
2571
|
-
teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
|
|
2572
|
-
if (isTeamSyncedOrphan) {
|
|
2573
|
-
items.push({ action: "skip-personal-mode", remoteFile, localPath });
|
|
2574
|
-
continue;
|
|
2575
|
-
}
|
|
2576
|
-
}
|
|
2577
|
-
|
|
2578
|
-
// Scope filter (US-005). Keys outside the effective `syncMode` prefix set
|
|
2579
|
-
// are not downloaded. `prefixSet` is `[""]` in `all` mode, which
|
|
2580
|
-
// `isCoveredByAny` treats as covering everything — so this is a no-op for
|
|
2581
|
-
// `all` and preserves the legacy full-bucket pull bit-for-bit. The
|
|
2582
|
-
// previously-downloaded counterparts of these keys (if scope just shrank)
|
|
2583
|
-
// are pruned separately by the scope-shrink pass in `sync()`.
|
|
2584
|
-
// A key can fail the inclusion check AND sit under a push-only exclude
|
|
2585
|
-
// prefix. Attribute it to `push-only` in that case: widening the
|
|
2586
|
-
// membership to `all` would move it into `prefixSet` but the exclusion
|
|
2587
|
-
// below would still withhold it, so it is NOT part of the shared-vs-all
|
|
2588
|
-
// gap and must not be advertised as fixable by raising the access level.
|
|
2589
|
-
if (!isCoveredByAny(remoteFile.key, prefixSet)) {
|
|
2590
|
-
items.push({
|
|
2591
|
-
action: "skip-out-of-scope",
|
|
2592
|
-
remoteFile,
|
|
2593
|
-
localPath,
|
|
2594
|
-
reason: isCoveredByAny(remoteFile.key, excludePrefixes)
|
|
2595
|
-
? "push-only"
|
|
2596
|
-
: "membership-scope",
|
|
2597
|
-
});
|
|
2598
|
-
continue;
|
|
2599
|
-
}
|
|
2600
|
-
|
|
2601
|
-
// Push-only exclusion filter (US-006). A key that PASSED the inclusion check
|
|
2602
|
-
// above is STILL not downloaded if it falls under a push-only exclude prefix
|
|
2603
|
-
// (`sessions/`). This is what makes sessions push-only in EVERY mode — it
|
|
2604
|
-
// subtracts from `all` (whose `prefixSet` is `[""]`, covering everything)
|
|
2605
|
-
// and from an owner's role-wide scope alike. Empty `excludePrefixes` (the
|
|
2606
|
-
// default) makes this a no-op. Deliberately AFTER inclusion and reported on
|
|
2607
|
-
// the same `filesOutOfScope` axis. The matching local file (if a session was
|
|
2608
|
-
// previously pulled, authored locally, or pinned+materialized) is NOT pruned
|
|
2609
|
-
// here or by scope-shrink — see `planScopeShrink`, which never sees this set.
|
|
2610
|
-
if (isCoveredByAny(remoteFile.key, excludePrefixes)) {
|
|
2611
|
-
items.push({
|
|
2612
|
-
action: "skip-out-of-scope",
|
|
2613
|
-
remoteFile,
|
|
2614
|
-
localPath,
|
|
2615
|
-
reason: "push-only",
|
|
2616
|
-
});
|
|
2617
|
-
continue;
|
|
2618
|
-
}
|
|
2619
|
-
|
|
2620
|
-
// LIST gives us no kind signal for the remote object — we don't
|
|
2621
|
-
// know whether this key is a regular file or a symlink record
|
|
2622
|
-
// until we either HEAD it (expensive — N extra calls per pull) or
|
|
2623
|
-
// download it. So probe the ignore filter with BOTH isDir hints
|
|
2624
|
-
// and include the entry if either probe passes. Without this,
|
|
2625
|
-
// a symlink record at a dir-only-allowlisted path (e.g. an
|
|
2626
|
-
// .hqinclude of `companies/*/knowledge/`) would be classified
|
|
2627
|
-
// skip-ignored and the symlinks freshly uploaded by the
|
|
2628
|
-
// push-side fix would never propagate. Mirrors the dual-hint
|
|
2629
|
-
// probe in walkDir/collectFiles. Pure path lookup, no I/O.
|
|
2630
|
-
if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) {
|
|
2631
|
-
items.push({ action: "skip-ignored", remoteFile, localPath });
|
|
2632
|
-
continue;
|
|
2633
|
-
}
|
|
2634
|
-
|
|
2635
|
-
const journalEntry = getEntry(journal, remoteFile.key);
|
|
2636
|
-
|
|
2637
|
-
// Reindex and package wiring own symlinks under these `core/<type>`
|
|
2638
|
-
// surfaces; their canonical sources are elsewhere. A prior pull records
|
|
2639
|
-
// the remote object as a symlink, then reindex removes the generated link.
|
|
2640
|
-
// Without this classification the next pull sees only an absent local path
|
|
2641
|
-
// and downloads the same marker forever. Retain regular core files: the
|
|
2642
|
-
// journal kind is the discriminator, because LIST exposes no object type.
|
|
2643
|
-
// An unchanged ETag is also required: a remote writer can replace a legacy
|
|
2644
|
-
// symlink with canonical file content at the same key, and that replacement
|
|
2645
|
-
// must still be pulled.
|
|
2646
|
-
if (
|
|
2647
|
-
personalMode &&
|
|
2648
|
-
journalEntry?.kind === "symlink" &&
|
|
2649
|
-
isGeneratedCoreMirrorKey(remoteFile.key) &&
|
|
2650
|
-
journalEntry.remoteEtag !== undefined &&
|
|
2651
|
-
!hasRemoteChanged(remoteFile, journalEntry)
|
|
2652
|
-
) {
|
|
2653
|
-
items.push({ action: "skip-excluded-policy", remoteFile, localPath });
|
|
2654
|
-
continue;
|
|
2655
|
-
}
|
|
2656
|
-
|
|
2657
|
-
// Junk-spelling guard — SKILL-FARM KEYS ONLY. The amplifier is specific
|
|
2658
|
-
// to `.claude/skills/<ns>:<skill>` wrapper spellings; outside that prefix,
|
|
2659
|
-
// keys whose percent-decode fixpoints collide (`notes/a b.md` vs a genuine
|
|
2660
|
-
// `notes/a%20b.md`) are DISTINCT files that must both pull.
|
|
2661
|
-
// Within the farm: a remote key that is NOT journaled under its own
|
|
2662
|
-
// spelling but whose percent-decode fixpoint matches an already-journaled
|
|
2663
|
-
// key with a different spelling is a mixed-version twin (`%3A`/`%253A`
|
|
2664
|
-
// generations of a canonical `:` key). Materializing it would mint junk
|
|
2665
|
-
// local dirs and fork the journal into a second key family for the same
|
|
2666
|
-
// logical path — skip it (loudly, never fatally; see the
|
|
2667
|
-
// `skip-junk-key-spelling` event doc) and leave the journaled spelling
|
|
2668
|
-
// authoritative for this machine.
|
|
2669
|
-
if (
|
|
2670
|
-
journalEntry === undefined &&
|
|
2671
|
-
remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
|
|
2672
|
-
// Gate NON-canonical remote spellings only. A remote key that already
|
|
2673
|
-
// IS the canonical spelling must always be downloadable — after the
|
|
2674
|
-
// doctor collapses the vault, a machine whose journal still holds only
|
|
2675
|
-
// the junk spelling would otherwise skip the canonical object forever
|
|
2676
|
-
// (and the delete-resync twin protection keeps that stale junk entry),
|
|
2677
|
-
// deadlocking it out of every future update.
|
|
2678
|
-
canonicalVaultKeySpelling(remoteFile.key) !== remoteFile.key
|
|
2679
|
-
) {
|
|
2680
|
-
const journaledTwin = journaledKeyByCanonical.get(
|
|
2681
|
-
canonicalVaultKeySpelling(remoteFile.key),
|
|
2682
|
-
);
|
|
2683
|
-
if (journaledTwin !== undefined && journaledTwin !== remoteFile.key) {
|
|
2684
|
-
items.push({
|
|
2685
|
-
action: "skip-junk-key-spelling",
|
|
2686
|
-
remoteFile,
|
|
2687
|
-
localPath,
|
|
2688
|
-
journaledKey: journaledTwin,
|
|
2689
|
-
});
|
|
2690
|
-
continue;
|
|
2691
|
-
}
|
|
2692
|
-
}
|
|
2693
|
-
|
|
2694
|
-
// ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
|
|
2695
|
-
// A remote object present in the LIST may be an intentionally-deleted key
|
|
2696
|
-
// (a peer re-pushed it, or its delete-marker hasn't propagated to this
|
|
2697
|
-
// caller's view). `tombstoneSuppresses` is true when a FILE_TOMBSTONE marks
|
|
2698
|
-
// this key deleted AND the remote object is NOT a newer re-create — i.e. the
|
|
2699
|
-
// object is the stale/deleted version and must NOT be downloaded. A genuine
|
|
2700
|
-
// re-create (object newer than the tombstone) leaves this false so the
|
|
2701
|
-
// normal download/merge path runs (the tombstone is not permanent
|
|
2702
|
-
// suppression). An empty tombstone map (the default / degraded-fetch case)
|
|
2703
|
-
// makes this always false — legacy behavior preserved bit-for-bit.
|
|
2704
|
-
const tombstone = fileTombstones.get(toPosixKey(remoteFile.key));
|
|
2705
|
-
const tombstoneSuppresses =
|
|
2706
|
-
tombstone !== undefined &&
|
|
2707
|
-
!isRemoteRecreateAfterTombstone(remoteFile, tombstone);
|
|
2708
|
-
// LOCAL journal tombstone — this machine's own intentional-delete record,
|
|
2709
|
-
// independent of the (best-effort, intermittently-degraded) server
|
|
2710
|
-
// FILE_TOMBSTONE fetch above. When a clean local delete is classified, the
|
|
2711
|
-
// push leg stamps `removedAt` on the journal entry; honoring it here makes
|
|
2712
|
-
// the delete-vs-drift-restore outcome DETERMINISTIC, closing the
|
|
2713
|
-
// intermittent respawn where a degraded tombstone fetch let the pull leg
|
|
2714
|
-
// re-materialize an intentionally-deleted file. Reuses the same re-create
|
|
2715
|
-
// guard (a remote object newer than the local delete is a genuine re-create
|
|
2716
|
-
// and still downloads). Consulted ONLY on the `!localExists` re-download
|
|
2717
|
-
// path below — never to delete a file that is present locally.
|
|
2718
|
-
const localTombstoneSuppresses =
|
|
2719
|
-
journalEntry !== undefined &&
|
|
2720
|
-
isTombstone(journalEntry) &&
|
|
2721
|
-
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
2722
|
-
journalEntry.removedAt !== undefined &&
|
|
2723
|
-
!isRemoteRecreateAfterTombstone(remoteFile, {
|
|
2724
|
-
deletedAt: journalEntry.removedAt,
|
|
2725
|
-
});
|
|
2726
|
-
|
|
2727
|
-
// lstat (not existsSync/statSync) handles three cases the legacy
|
|
2728
|
-
// checks got wrong for symlinks:
|
|
2729
|
-
// 1. A valid symlink at localPath: existsSync returns true and the
|
|
2730
|
-
// planner falls through to hashFile, which follows the link and
|
|
2731
|
-
// hashes the target file's bytes — never matching the journal's
|
|
2732
|
-
// sha256(target string), so every pull would mis-classify a
|
|
2733
|
-
// pristine symlink as locally changed.
|
|
2734
|
-
// 2. A dangling symlink: existsSync returns false (existsSync
|
|
2735
|
-
// follows links) so the planner treats the entry as missing and
|
|
2736
|
-
// re-downloads, silently clobbering the user's intentional
|
|
2737
|
-
// dangling link on every run.
|
|
2738
|
-
// 3. A regular file: indistinguishable from current behaviour.
|
|
2739
|
-
let localLstat: fs.Stats | null = null;
|
|
2740
|
-
let plannedLocalSnapshot: LocalSnapshot = { kind: "absent" };
|
|
2741
|
-
let localPathBlockedByFileAncestor = false;
|
|
2742
|
-
try {
|
|
2743
|
-
localLstat = fs.lstatSync(localPath);
|
|
2744
|
-
} catch (err: unknown) {
|
|
2745
|
-
const code =
|
|
2746
|
-
err && typeof err === "object" && "code" in err
|
|
2747
|
-
? (err as { code?: string }).code
|
|
2748
|
-
: undefined;
|
|
2749
|
-
// ENOENT → truly absent → treated as "new file from cloud".
|
|
2750
|
-
// ENOTDIR → an ancestor of localPath is a regular file (cloud has a
|
|
2751
|
-
// dir at that key, local has a file). Pre-fix this threw
|
|
2752
|
-
// and aborted the whole company sync (Bug #10 in the
|
|
2753
|
-
// 5.33.0 deep-test verification — \`v4-dir-vs-file\` repro
|
|
2754
|
-
// wedged the personal company at \"errored\" status,
|
|
2755
|
-
// skipping every later file). Recover by classifying as
|
|
2756
|
-
// a structural collision the operator must resolve
|
|
2757
|
-
// manually, and let the rest of the company process.
|
|
2758
|
-
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
2759
|
-
throw err;
|
|
2760
|
-
}
|
|
2761
|
-
if (code === "ENOTDIR") {
|
|
2762
|
-
localPathBlockedByFileAncestor = true;
|
|
2763
|
-
}
|
|
2764
|
-
}
|
|
2765
|
-
const localExists = localLstat !== null;
|
|
2766
|
-
|
|
2767
|
-
if (localPathBlockedByFileAncestor) {
|
|
2768
|
-
// Symmetric counterpart to the (local-dir, cloud-file) warning below.
|
|
2769
|
-
// Emit the same "manual reconciliation required" surface so the
|
|
2770
|
-
// operator sees one consistent message for both topologies; record as
|
|
2771
|
-
// skip-local-only so the file is not silently dropped from the
|
|
2772
|
-
// count and the executor never tries to lstat/write it.
|
|
2773
|
-
console.error(
|
|
2774
|
-
` Warning: an ancestor of ${remoteFile.key} exists locally as a regular file; ` +
|
|
2775
|
-
`cloud has a deeper key under that path. Skipping; manual ` +
|
|
2776
|
-
`reconciliation required (rm the conflicting local file to pull).`,
|
|
2777
|
-
);
|
|
2778
|
-
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
2779
|
-
continue;
|
|
2780
|
-
}
|
|
2781
|
-
|
|
2782
|
-
if (localExists) {
|
|
2783
|
-
const isLocalSymlink = localLstat!.isSymbolicLink();
|
|
2784
|
-
// Kind-mismatch guard: a remote LIST entry is always a single
|
|
2785
|
-
// S3 object (regular file or symlink record), but the local
|
|
2786
|
-
// path may be a real directory if the user previously had a
|
|
2787
|
-
// dir at this key. Hashing a directory throws EISDIR and
|
|
2788
|
-
// aborts the whole pull. We don't auto-replace because that
|
|
2789
|
-
// would `rm -rf` the user's directory; surface as a non-fatal
|
|
2790
|
-
// skip and warn so the operator can reconcile manually
|
|
2791
|
-
// (delete the local dir, or restructure the remote).
|
|
2792
|
-
if (localLstat!.isDirectory() && !isLocalSymlink) {
|
|
2793
|
-
// Stale personal-overlay marker vs. release-shipped core directory:
|
|
2794
|
-
// a single vault object at a `core/<overlay-type>/.../<entry>` key
|
|
2795
|
-
// whose local counterpart is a real directory is an orphaned overlay
|
|
2796
|
-
// symlink marker (reindex mirror artifact) left behind after a release
|
|
2797
|
-
// shipped a real core dir of the same name. The local release-shipped
|
|
2798
|
-
// directory is authoritative and the marker can never be materialized
|
|
2799
|
-
// over it, so keep local and ignore the marker QUIETLY — the generic
|
|
2800
|
-
// "rm -rf the local directory" advice below is actively dangerous here
|
|
2801
|
-
// (it would destroy release-shipped core content to pull inert derived
|
|
2802
|
-
// state), and it recurred on every sync. feedback_ce9aeede.
|
|
2803
|
-
if (isGeneratedCoreMirrorKey(remoteFile.key)) {
|
|
2804
|
-
console.error(
|
|
2805
|
-
` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
|
|
2806
|
-
`shadowing a release-shipped core directory; keeping the local ` +
|
|
2807
|
-
`directory and ignoring the marker (no action needed).`,
|
|
2808
|
-
);
|
|
2809
|
-
items.push({
|
|
2810
|
-
action: "skip-stale-overlay-marker",
|
|
2811
|
-
remoteFile,
|
|
2812
|
-
localPath,
|
|
2813
|
-
});
|
|
2814
|
-
continue;
|
|
2815
|
-
}
|
|
2816
|
-
// Directory-overlay marker vs. a locally-materialized directory:
|
|
2817
|
-
// the remote object at this key carries CHILD objects under it in the
|
|
2818
|
-
// same LIST (`remoteKeysWithChildren`), so it is a symlink/overlay
|
|
2819
|
-
// record for a directory — e.g. the DPP vault's `skills/cody-copywriter`
|
|
2820
|
-
// overlay whose 13 child objects are already materialized locally. The
|
|
2821
|
-
// marker can never be renamed over the real directory, and its children
|
|
2822
|
-
// carry the actual content and materialize via their own plan items, so
|
|
2823
|
-
// the marker is a benign, no-action skip. The generic "rm -rf the local
|
|
2824
|
-
// directory to pull" advice below would tell the operator to DELETE the
|
|
2825
|
-
// materialized (and possibly locally-authored) contents to install an
|
|
2826
|
-
// inert link, and it recurred on EVERY sync cycle. feedback_d2082110.
|
|
2827
|
-
if (remoteKeysWithChildren.has(toPosixKey(remoteFile.key))) {
|
|
2828
|
-
console.error(
|
|
2829
|
-
` Note: ${remoteFile.key} is a directory-overlay marker whose ` +
|
|
2830
|
-
`contents already exist locally as a directory; keeping the ` +
|
|
2831
|
-
`local directory and ignoring the marker (no action needed).`,
|
|
2832
|
-
);
|
|
2833
|
-
items.push({
|
|
2834
|
-
action: "skip-overlay-marker-with-children",
|
|
2835
|
-
remoteFile,
|
|
2836
|
-
localPath,
|
|
2837
|
-
});
|
|
2838
|
-
continue;
|
|
2839
|
-
}
|
|
2840
|
-
console.error(
|
|
2841
|
-
` Warning: ${remoteFile.key} exists locally as a directory; ` +
|
|
2842
|
-
`cloud has a single object at this key. Skipping; manual ` +
|
|
2843
|
-
`reconciliation required (rm -rf the local directory to pull).`,
|
|
2844
|
-
);
|
|
2845
|
-
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
2846
|
-
continue;
|
|
2847
|
-
}
|
|
2848
|
-
// Fast-path (5.36.0): when the journal entry has an mtimeMs and the
|
|
2849
|
-
// local lstat matches both size and mtimeMs, the file hasn't been
|
|
2850
|
-
// touched locally since the last sync, so we can reuse the journal's
|
|
2851
|
-
// recorded hash without re-reading the file's bytes. Same shape as
|
|
2852
|
-
// the share-side fast-path; only kicks in for regular files (symlink
|
|
2853
|
-
// hashing through hashSymlinkTarget is already cheap — it hashes a
|
|
2854
|
-
// short target string, not file bytes).
|
|
2855
|
-
let localHash: string;
|
|
2856
|
-
if (
|
|
2857
|
-
!isLocalSymlink &&
|
|
2858
|
-
journalEntry &&
|
|
2859
|
-
journalEntry.mtimeMs !== undefined &&
|
|
2860
|
-
journalEntry.hash &&
|
|
2861
|
-
localLstat!.size === journalEntry.size &&
|
|
2862
|
-
localLstat!.mtimeMs === journalEntry.mtimeMs
|
|
2863
|
-
) {
|
|
2864
|
-
localHash = journalEntry.hash;
|
|
2865
|
-
} else {
|
|
2866
|
-
if (isLocalSymlink) {
|
|
2867
|
-
const target = readlinkOrNull(localPath);
|
|
2868
|
-
if (target === null) {
|
|
2869
|
-
// We cannot prove whether the local link diverged, so never
|
|
2870
|
-
// overwrite or tombstone it from this plan. A later pass can retry
|
|
2871
|
-
// once the OS exposes a readable target.
|
|
2872
|
-
items.push({ action: "skip-unreadable-link", remoteFile, localPath });
|
|
2873
|
-
continue;
|
|
2874
|
-
}
|
|
2875
|
-
localHash = hashSymlinkTarget(target);
|
|
2876
|
-
} else {
|
|
2877
|
-
localHash = hashFile(localPath);
|
|
2878
|
-
}
|
|
2879
|
-
}
|
|
2880
|
-
const localChanged = !!journalEntry && journalEntry.hash !== localHash;
|
|
2881
|
-
plannedLocalSnapshot = {
|
|
2882
|
-
kind: isLocalSymlink ? "symlink" : "file",
|
|
2883
|
-
hash: localHash,
|
|
2884
|
-
};
|
|
2885
|
-
const remoteChanged =
|
|
2886
|
-
!!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
|
|
2887
|
-
|
|
2888
|
-
// ── Authoritative delete reaches a machine that still holds the file ──
|
|
2889
|
-
// A FILE_TOMBSTONE marks this key deleted and the remote object is not a
|
|
2890
|
-
// newer re-create, but this machine still has a local copy. Honor the
|
|
2891
|
-
// delete — EXCEPT never destroy unsynced local work: if the local copy
|
|
2892
|
-
// diverges from the synced baseline (a post-tombstone edit, or an
|
|
2893
|
-
// untracked local file with no journal entry), surface a CONFLICT instead
|
|
2894
|
-
// of silently deleting (the safety guard the brief requires + the 3-way
|
|
2895
|
-
// merge invariant). A local copy that still matches the deleted baseline
|
|
2896
|
-
// is the stale version → delete it + drop the journal entry so it stays
|
|
2897
|
-
// gone on this machine.
|
|
2898
|
-
if (tombstoneSuppresses) {
|
|
2899
|
-
if (localChanged || !journalEntry) {
|
|
2900
|
-
items.push({
|
|
2901
|
-
action: "conflict",
|
|
2902
|
-
remoteFile,
|
|
2903
|
-
localPath,
|
|
2904
|
-
localHash,
|
|
2905
|
-
localMtime: localLstat!.mtime,
|
|
2906
|
-
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
2907
|
-
localSnapshot: plannedLocalSnapshot,
|
|
2908
|
-
});
|
|
2909
|
-
} else {
|
|
2910
|
-
items.push({
|
|
2911
|
-
action: "tombstone-delete",
|
|
2912
|
-
remoteFile,
|
|
2913
|
-
localPath,
|
|
2914
|
-
localSnapshot: plannedLocalSnapshot,
|
|
2915
|
-
});
|
|
2916
|
-
}
|
|
2917
|
-
continue;
|
|
2918
|
-
}
|
|
2919
|
-
|
|
2920
|
-
// ── Untracked-local guard (unjournaled local file) ──────────────
|
|
2921
|
-
// Both `localChanged` and `remoteChanged` are gated on `!!journalEntry`,
|
|
2922
|
-
// so a key with NO journal entry reads "clean on both sides" and falls
|
|
2923
|
-
// all the way through to a plain `download` — silently replacing a local
|
|
2924
|
-
// file that exists on disk. There is no conflict, no `.conflict-` mirror
|
|
2925
|
-
// and no conflict-index record, so the loss is invisible unless someone
|
|
2926
|
-
// happens to know what the file should contain.
|
|
2927
|
-
//
|
|
2928
|
-
// This destroyed in-progress edits to
|
|
2929
|
-
// `companies/*/projects/*/prd.json` three times on 2026-08-01/02, but
|
|
2930
|
-
// nothing about it is prd- or project-specific: it fires for ANY synced
|
|
2931
|
-
// path whose journal entry is absent. Absence is common — a file created
|
|
2932
|
-
// on this machine and on a peer under the same key before either pushed,
|
|
2933
|
-
// a journal shard reset/lost/rebuilt (`readJournal` returns an empty
|
|
2934
|
-
// `files` map for a missing shard, so EVERY divergent local file is in
|
|
2935
|
-
// scope), a run under a different `HQ_STATE_DIR`, or an entry dropped by
|
|
2936
|
-
// scope-shrink or rescue.
|
|
2937
|
-
//
|
|
2938
|
-
// No journal entry means no evidence that local and remote ever agreed,
|
|
2939
|
-
// which is exactly when an overwrite is least safe. Route it through the
|
|
2940
|
-
// conflict path instead: the executor's convergence probe fetches the
|
|
2941
|
-
// remote once and, when the bytes already match (the ordinary
|
|
2942
|
-
// fresh-machine / rebuilt-journal case), reconciles silently — no
|
|
2943
|
-
// conflict, no mirror, journal stamped. Only genuine divergence surfaces,
|
|
2944
|
-
// and there local is PRESERVED as-is with the cloud copy written beside
|
|
2945
|
-
// it under `--on-conflict keep`, or the run halts under `abort`.
|
|
2946
|
-
// Cloud-authoritative paths (board.json, ontology/, signals/, …) are
|
|
2947
|
-
// unaffected — the conflict executor short-circuits those back to a
|
|
2948
|
-
// pull-wins download before any mirror is minted.
|
|
2949
|
-
if (!journalEntry) {
|
|
2950
|
-
items.push({
|
|
2951
|
-
action: "conflict",
|
|
2952
|
-
remoteFile,
|
|
2953
|
-
localPath,
|
|
2954
|
-
localHash,
|
|
2955
|
-
localMtime: localLstat!.mtime,
|
|
2956
|
-
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
2957
|
-
localSnapshot: plannedLocalSnapshot,
|
|
2958
|
-
});
|
|
2959
|
-
continue;
|
|
2960
|
-
}
|
|
2961
|
-
|
|
2962
|
-
// Mirror the original 3-way merge from the inline loop. Tested by
|
|
2963
|
-
// `does NOT flag a pull conflict when only local changed since last
|
|
2964
|
-
// sync` and `detects conflicts with local changes…`.
|
|
2965
|
-
if (localChanged && remoteChanged) {
|
|
2966
|
-
items.push({
|
|
2967
|
-
action: "conflict",
|
|
2968
|
-
remoteFile,
|
|
2969
|
-
localPath,
|
|
2970
|
-
localHash,
|
|
2971
|
-
// localLstat is non-null inside this `if (localExists)`
|
|
2972
|
-
// branch — the planner already lstat'd the path. Carry
|
|
2973
|
-
// its mtime forward so the executor doesn't have to
|
|
2974
|
-
// re-stat (which would follow links and crash on a
|
|
2975
|
-
// dangling symlink). Same goes for size: the conflict
|
|
2976
|
-
// executor's keep/skip branch needs to stamp the
|
|
2977
|
-
// journal with a size, and statSync on a dangling
|
|
2978
|
-
// symlink throws ENOENT (silently swallowed pre-fix),
|
|
2979
|
-
// leaving the journal stale and re-firing the conflict
|
|
2980
|
-
// every sync.
|
|
2981
|
-
localMtime: localLstat!.mtime,
|
|
2982
|
-
localSize: isLocalSymlink ? 0 : localLstat!.size,
|
|
2983
|
-
localSnapshot: plannedLocalSnapshot,
|
|
2984
|
-
});
|
|
2985
|
-
continue;
|
|
2986
|
-
}
|
|
2987
|
-
if (journalEntry && localChanged && !remoteChanged) {
|
|
2988
|
-
items.push({ action: "skip-local-only", remoteFile, localPath });
|
|
2989
|
-
continue;
|
|
2990
|
-
}
|
|
2991
|
-
if (journalEntry && !localChanged && !remoteChanged) {
|
|
2992
|
-
// Flavor-health repair (win32 skill farm): a link journaled while its
|
|
2993
|
-
// target was absent was minted file-flavor; once the target directory
|
|
2994
|
-
// exists the hashes still match (hash = sha256(target)) so this
|
|
2995
|
-
// branch would skip it forever, leaving the wrapper broken. Re-download
|
|
2996
|
-
// instead — the materializer's flavor-aware path recreates it 'dir'.
|
|
2997
|
-
if (
|
|
2998
|
-
isLocalSymlink &&
|
|
2999
|
-
remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
|
|
3000
|
-
win32SymlinkFlavorBroken(localPath)
|
|
3001
|
-
) {
|
|
3002
|
-
items.push({
|
|
3003
|
-
action: "download",
|
|
3004
|
-
remoteFile,
|
|
3005
|
-
localPath,
|
|
3006
|
-
isNew: false,
|
|
3007
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3008
|
-
});
|
|
3009
|
-
continue;
|
|
3010
|
-
}
|
|
3011
|
-
items.push({ action: "skip-unchanged", remoteFile, localPath });
|
|
3012
|
-
continue;
|
|
3013
|
-
}
|
|
3014
|
-
if (journalEntry && !localChanged && remoteChanged) {
|
|
3015
|
-
// ── Anti-rollback guard (silent newer→older overwrite) ───────────
|
|
3016
|
-
// Remote etag moved but local still matches the journal baseline, so
|
|
3017
|
-
// the classic 3-way merge calls this a clean `download` and overwrites
|
|
3018
|
-
// local with remote. That assumes the remote *advanced*. It can also
|
|
3019
|
-
// REGRESS: a second device (or a stale re-push) can replace the S3
|
|
3020
|
-
// object with an OLDER, SMALLER copy. `hasRemoteChanged` is pure-ETag
|
|
3021
|
-
// and cannot tell an advance from a rollback — ListObjectsV2 carries no
|
|
3022
|
-
// content hash, and `lastModified` is the upload time (newer even for
|
|
3023
|
-
// stale content), so neither distinguishes the two. A silent download
|
|
3024
|
-
// then DESTROYS newer local bytes with no conflict surfaced.
|
|
3025
|
-
//
|
|
3026
|
-
// This bit append-only bot chat logs on 2026-06-10, the day
|
|
3027
|
-
// event-driven bidirectional sync (which lets a peer device push to
|
|
3028
|
-
// S3 between this device's pulls) first rolled out: a regressed S3
|
|
3029
|
-
// object truncated freshly-appended lines, undetected by the planner.
|
|
3030
|
-
//
|
|
3031
|
-
// Guard: if the incoming remote object is strictly SMALLER than the
|
|
3032
|
-
// local file we already hold (a size regression on a tracked regular
|
|
3033
|
-
// file), do NOT silently clobber. Route through the conflict path —
|
|
3034
|
-
// the executor's convergence probe fetches the remote once and, on
|
|
3035
|
-
// genuine divergence, PRESERVES local as a `.conflict-*` mirror instead
|
|
3036
|
-
// of overwriting it (and under `--on-conflict abort` halts loudly
|
|
3037
|
-
// rather than losing data). A byte-identical remote can never be
|
|
3038
|
-
// smaller, so this never manufactures a false conflict on a no-op
|
|
3039
|
-
// re-upload. Symlinks are exempt: their on-disk size is not the target
|
|
3040
|
-
// length, so the comparison is meaningless.
|
|
3041
|
-
//
|
|
3042
|
-
// Direction gate: only guard copies this device AUTHORED (last synced
|
|
3043
|
-
// UP). A consumer copy (last synced DOWN) has no local authorship to
|
|
3044
|
-
// protect — the remote is authoritative, so a smaller remote is a
|
|
3045
|
-
// legitimate update (e.g. the gardener rewriting a shorter brief), not a
|
|
3046
|
-
// regression. Without this gate a consumer device floods `.conflict`
|
|
3047
|
-
// mirrors and stalls on stale gardener output forever (the 786-file
|
|
3048
|
-
// pile-up, 2026-06-22). `"up"` and any missing/unknown direction stay
|
|
3049
|
-
// guarded exactly as before — the 2026-06-10 chat-log protection holds.
|
|
3050
|
-
if (
|
|
3051
|
-
journalEntry.direction !== "down" &&
|
|
3052
|
-
!isLocalSymlink &&
|
|
3053
|
-
remoteFile.size < localLstat!.size
|
|
3054
|
-
) {
|
|
3055
|
-
items.push({
|
|
3056
|
-
action: "conflict",
|
|
3057
|
-
remoteFile,
|
|
3058
|
-
localPath,
|
|
3059
|
-
localHash,
|
|
3060
|
-
localMtime: localLstat!.mtime,
|
|
3061
|
-
localSize: localLstat!.size,
|
|
3062
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3063
|
-
});
|
|
3064
|
-
continue;
|
|
3065
|
-
}
|
|
3066
|
-
// Remote genuinely advanced (same-or-larger) → normal catch-up
|
|
3067
|
-
// download below.
|
|
3068
|
-
}
|
|
3069
|
-
// No journal entry, or remote-only changed (non-shrinking) → fall
|
|
3070
|
-
// through to download.
|
|
3071
|
-
}
|
|
3072
|
-
|
|
3073
|
-
// Suppress the re-download of an intentionally-deleted key (delete-resync).
|
|
3074
|
-
// Reaches here only for the `!localExists` case — the `localExists` branch
|
|
3075
|
-
// above already handled (and `continue`d) any suppressing tombstone, so a
|
|
3076
|
-
// tombstone that survives to here means "remote present, local absent, not a
|
|
3077
|
-
// re-create": the classic resurrection ("remote present → I'm behind →
|
|
3078
|
-
// download") the planner must NOT do. Route to `tombstone-delete` so the
|
|
3079
|
-
// executor drops any stale journal entry and the key stays gone, instead of
|
|
3080
|
-
// pulling the deleted object back in. `localTombstoneSuppresses` adds the
|
|
3081
|
-
// same protection driven by this machine's own journal tombstone, so an
|
|
3082
|
-
// intentional delete is not resurrected even when the server FILE_TOMBSTONE
|
|
3083
|
-
// fetch is degraded.
|
|
3084
|
-
if (tombstoneSuppresses || localTombstoneSuppresses) {
|
|
3085
|
-
// A persisting `local-delete` tombstone (stamped by the intentional-delete
|
|
3086
|
-
// producer below) must KEEP its journal entry — the push leg still needs
|
|
3087
|
-
// it to propagate the S3 delete. Skip the re-download without dropping the
|
|
3088
|
-
// entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
|
|
3089
|
-
if (journalEntry?.removedReason === "local-delete") {
|
|
3090
|
-
continue;
|
|
3091
|
-
}
|
|
3092
|
-
items.push({
|
|
3093
|
-
action: "tombstone-delete",
|
|
3094
|
-
remoteFile,
|
|
3095
|
-
localPath,
|
|
3096
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3097
|
-
});
|
|
3098
|
-
continue;
|
|
3099
|
-
}
|
|
3100
|
-
|
|
3101
|
-
// Intentional-delete producer: a clean, CURRENT (etag matches), non-divergent
|
|
3102
|
-
// file that is MISSING locally is the user deleting a genuinely-synced file.
|
|
3103
|
-
// Collect it; the post-loop bulk-asymmetry decision tombstones it (intent) or
|
|
3104
|
-
// restores it (mass loss = accident). A stale-etag, divergent, tombstoned, or
|
|
3105
|
-
// never-journaled missing file falls through to the normal download below.
|
|
3106
|
-
if (
|
|
3107
|
-
!localExists &&
|
|
3108
|
-
journalEntry !== undefined &&
|
|
3109
|
-
!isTombstone(journalEntry) &&
|
|
3110
|
-
hasCurrentLocalDeleteIntent(journalEntry) &&
|
|
3111
|
-
normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag
|
|
3112
|
-
) {
|
|
3113
|
-
intentionalDeleteCandidates.push({ remoteFile, localPath });
|
|
3114
|
-
continue;
|
|
3115
|
-
}
|
|
3116
|
-
|
|
3117
|
-
items.push({
|
|
3118
|
-
action: "download",
|
|
3119
|
-
remoteFile,
|
|
3120
|
-
localPath,
|
|
3121
|
-
isNew: !localExists,
|
|
3122
|
-
localSnapshot: plannedLocalSnapshot,
|
|
3123
|
-
});
|
|
3124
|
-
}
|
|
3125
|
-
|
|
3126
|
-
// ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
|
|
3127
|
-
// Below the breaker → stamp a persisting `local-delete` tombstone (the
|
|
3128
|
-
// suppression above keeps it from being re-pulled; the push leg propagates the
|
|
3129
|
-
// S3 delete on its next pass). At/above → drift-restore EVERY candidate: a
|
|
3130
|
-
// mass disappearance (corrupt mirror / bulk rm / checkout / unmount) is
|
|
3131
|
-
// accidental loss, never intent (the ridge-incident guard).
|
|
3132
|
-
const totalJournalEntries = Object.keys(journal.files).length;
|
|
3133
|
-
const intentionalDeleteIsBulk =
|
|
3134
|
-
intentionalDeleteCandidates.length >= PULL_INTENTIONAL_DELETE_MIN_ABS &&
|
|
3135
|
-
totalJournalEntries > 0 &&
|
|
3136
|
-
intentionalDeleteCandidates.length / totalJournalEntries >=
|
|
3137
|
-
PULL_INTENTIONAL_DELETE_RATIO;
|
|
3138
|
-
for (const cand of intentionalDeleteCandidates) {
|
|
3139
|
-
if (intentionalDeleteIsBulk) {
|
|
3140
|
-
items.push({
|
|
3141
|
-
action: "download",
|
|
3142
|
-
remoteFile: cand.remoteFile,
|
|
3143
|
-
localPath: cand.localPath,
|
|
3144
|
-
isNew: true,
|
|
3145
|
-
localSnapshot: { kind: "absent" },
|
|
3146
|
-
});
|
|
3147
|
-
} else {
|
|
3148
|
-
tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
|
|
3149
|
-
}
|
|
3150
|
-
}
|
|
3151
|
-
|
|
3152
|
-
let filesToDownload = 0;
|
|
3153
|
-
let bytesToDownload = 0;
|
|
3154
|
-
let filesToSkip = 0;
|
|
3155
|
-
let filesToConflict = 0;
|
|
3156
|
-
let filesExcludedByPolicy = 0;
|
|
3157
|
-
let filesOutOfScope = 0;
|
|
3158
|
-
let filesToTombstoneDelete = 0;
|
|
3159
|
-
const newFiles: Array<{ path: string; bytes: number }> = [];
|
|
3160
|
-
for (const item of items) {
|
|
3161
|
-
if (item.action === "download") {
|
|
3162
|
-
filesToDownload++;
|
|
3163
|
-
bytesToDownload += item.remoteFile.size;
|
|
3164
|
-
if (item.isNew) {
|
|
3165
|
-
newFiles.push({ path: item.remoteFile.key, bytes: item.remoteFile.size });
|
|
3166
|
-
}
|
|
3167
|
-
} else if (item.action === "conflict") {
|
|
3168
|
-
filesToConflict++;
|
|
3169
|
-
} else if (item.action === "tombstone-delete") {
|
|
3170
|
-
// Authoritative FILE_TOMBSTONE delete — its own axis so it never inflates
|
|
3171
|
-
// the "unchanged" tally. Surfaced via the plan event's filesToDelete.
|
|
3172
|
-
filesToTombstoneDelete++;
|
|
3173
|
-
} else if (item.action === "skip-excluded-policy") {
|
|
3174
|
-
filesExcludedByPolicy++;
|
|
3175
|
-
// Excluded-policy items don't roll into filesToSkip — they're a
|
|
3176
|
-
// distinct class surfaced via filesExcludedByPolicy so consumers
|
|
3177
|
-
// can render a "N refused by policy" line independently of the
|
|
3178
|
-
// generic "N unchanged" tally.
|
|
3179
|
-
} else if (item.action === "skip-out-of-scope") {
|
|
3180
|
-
// Out-of-scope items get their own axis too, mirroring excluded-policy:
|
|
3181
|
-
// they're "deliberately not downloaded (sync scope)", not "unchanged".
|
|
3182
|
-
filesOutOfScope++;
|
|
3183
|
-
} else {
|
|
3184
|
-
filesToSkip++;
|
|
3185
|
-
}
|
|
3186
|
-
}
|
|
3187
|
-
|
|
3188
|
-
// Bug #9 — cross-machine delete propagation. The peer's push leg
|
|
3189
|
-
// already removed the file from S3 (verified in the deep-test
|
|
3190
|
-
// addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
|
|
3191
|
-
// the pull side enumerated remote keys and downloaded anything
|
|
3192
|
-
// missing locally, but never enumerated "what's missing-from-remote-
|
|
3193
|
-
// that-was-there-before" — so the file stayed forever on every
|
|
3194
|
-
// receiver and `filesTombstoned: 0` showed up on every pull.
|
|
3195
|
-
//
|
|
3196
|
-
// Closing the loop: walk the journal, find every entry whose key is
|
|
3197
|
-
// NOT in the remote LIST set, AND whose path passes the current
|
|
3198
|
-
// ignore filter (paths newly excluded by .hqignore must not trigger
|
|
3199
|
-
// mass-delete), AND — in personalMode — survives the same companies/*
|
|
3200
|
-
// gating the download branch applies. The executor will apply each
|
|
3201
|
-
// as a local delete + journal removal. Symmetric to the push side's
|
|
3202
|
-
// `propagateDeletes` plan in share.ts.
|
|
3203
|
-
const remoteKeySet = new Set<string>();
|
|
3204
|
-
// Skill-farm twin protection (companion to the junk-spelling pull guard):
|
|
3205
|
-
// when the LIST holds only a junk spelling of a journaled canonical key,
|
|
3206
|
-
// the logical object is still remotely present — a raw has() would class
|
|
3207
|
-
// the canonical journal entry as remote-deleted, unlink the good local
|
|
3208
|
-
// wrapper, and drop the journal entry (after which the next pull would
|
|
3209
|
-
// materialize the junk spelling the guard just refused). Track canonical
|
|
3210
|
-
// forms of listed skill keys and consult them below.
|
|
3211
|
-
const remoteCanonicalSkillKeys = new Set<string>();
|
|
3212
|
-
for (const rf of remoteFiles) {
|
|
3213
|
-
remoteKeySet.add(rf.key);
|
|
3214
|
-
if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
|
|
3215
|
-
remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
|
|
3216
|
-
}
|
|
3217
|
-
}
|
|
3218
|
-
const tombstones: PullPlan["tombstones"] = [];
|
|
3219
|
-
for (const key of Object.keys(journal.files)) {
|
|
3220
|
-
// Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
|
|
3221
|
-
// carry backslash separators; the remote LIST is always forward-slash, so a
|
|
3222
|
-
// raw `has(key)` would miss the match and class a live local file as
|
|
3223
|
-
// remote-deleted. normalizeJournalKeys() defuses this on load, but the
|
|
3224
|
-
// POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
|
|
3225
|
-
const posixKey = toPosixKey(key);
|
|
3226
|
-
if (remoteKeySet.has(posixKey)) continue;
|
|
3227
|
-
if (
|
|
3228
|
-
posixKey.startsWith(SKILLS_KEY_PREFIX) &&
|
|
3229
|
-
remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
|
|
3230
|
-
// An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
|
|
3231
|
-
// delete and wins over the twin protection — a lingering junk
|
|
3232
|
-
// spelling in the LIST must not keep a deliberately-deleted wrapper
|
|
3233
|
-
// alive forever. (The key is absent from the LIST here, so there is
|
|
3234
|
-
// no newer re-create to defer to.)
|
|
3235
|
-
fileTombstones.get(posixKey) === undefined
|
|
3236
|
-
) {
|
|
3237
|
-
continue;
|
|
3238
|
-
}
|
|
3239
|
-
const localPath = resolveContainedVaultPath(companyRoot, key);
|
|
3240
|
-
if (localPath === null) continue;
|
|
3241
|
-
// Scope-invalid journal keys (company mode, `companies/…` prefix): the
|
|
3242
|
-
// download branch skips them (presign rejects), but a journaled entry whose
|
|
3243
|
-
// key is absent from the remote LIST is doubled-tree poison that must
|
|
3244
|
-
// drain — local copy under companies/{slug}/companies/{slug}/… removed and
|
|
3245
|
-
// journal dropped. HEAD verify is skipped in verifyPlannedJournalTombstones
|
|
3246
|
-
// (presign would reject); see the auto-verified branch there.
|
|
3247
|
-
const scopeInvalidJournalKey = isForbiddenCompanyVaultKey(key, personalMode);
|
|
3248
|
-
// PersonalMode key gating — mirror the download branch. Local (non-cloud)
|
|
3249
|
-
// company keys are tombstone-eligible (a peer's delete should propagate),
|
|
3250
|
-
// but team-synced orphans are left alone (the team-bucket pull owns those
|
|
3251
|
-
// disk paths), and the manifest — the routing source-of-truth — is never
|
|
3252
|
-
// auto-tombstoned out of the personal vault.
|
|
3253
|
-
if (personalMode && key.startsWith("companies/")) {
|
|
3254
|
-
const slug = key.split("/")[1] ?? "";
|
|
3255
|
-
const isTeamSyncedOrphan =
|
|
3256
|
-
teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
|
|
3257
|
-
if (isTeamSyncedOrphan || key === PERSONAL_VAULT_MANIFEST_KEY) continue;
|
|
3258
|
-
}
|
|
3259
|
-
// Ephemeral keys are filtered both directions; never tombstone-
|
|
3260
|
-
// propagate a conflict-mirror.
|
|
3261
|
-
if (isEphemeralPath(key)) continue;
|
|
3262
|
-
// Honor the current ignore filter — if a path was previously synced
|
|
3263
|
-
// but is now ignored (operator edited .hqignore), do NOT delete
|
|
3264
|
-
// the local copy. They're keeping it deliberately. Scope-invalid poison
|
|
3265
|
-
// bypasses this: the doubled-tree copy must not survive silently.
|
|
3266
|
-
if (
|
|
3267
|
-
!scopeInvalidJournalKey &&
|
|
3268
|
-
!shouldSync(localPath, false) &&
|
|
3269
|
-
!shouldSync(localPath, true)
|
|
3270
|
-
) {
|
|
3271
|
-
continue;
|
|
3272
|
-
}
|
|
3273
|
-
// Codex P1 (PR #24 round 3): detect local edits before tombstoning.
|
|
3274
|
-
// Delete-vs-local-edit race: peer deleted the file remotely while
|
|
3275
|
-
// this machine edited it locally before the next sync. Without
|
|
3276
|
-
// this check, the tombstone executor would unlink the locally-
|
|
3277
|
-
// edited file and drop the journal entry, silently destroying the
|
|
3278
|
-
// user's unsynced work. Compare local hash against the journal
|
|
3279
|
-
// baseline; if they diverge, defer the tombstone. The next pull
|
|
3280
|
-
// run after the operator resolves (re-pushes / overwrites) the
|
|
3281
|
-
// local state will re-evaluate.
|
|
3282
|
-
try {
|
|
3283
|
-
const lstat = fs.lstatSync(localPath);
|
|
3284
|
-
if (lstat.isFile()) {
|
|
3285
|
-
const localHash = hashFile(localPath);
|
|
3286
|
-
const journalEntry = journal.files[key];
|
|
3287
|
-
if (journalEntry?.hash && journalEntry.hash !== localHash) {
|
|
3288
|
-
// Local has unsynced edits — defer this tombstone.
|
|
3289
|
-
continue;
|
|
3290
|
-
}
|
|
3291
|
-
} else if (lstat.isSymbolicLink()) {
|
|
3292
|
-
// Codex P1 (PR #24 round 4): symlinks have target strings
|
|
3293
|
-
// that can be locally edited too. Pre-fix, `isFile()` was
|
|
3294
|
-
// false for symlinks so the divergence guard skipped them,
|
|
3295
|
-
// and a locally-edited symlink (`ln -sfn new-target old-link`
|
|
3296
|
-
// before the peer's remote-delete) was unlinked silently —
|
|
3297
|
-
// the exact race the guard is meant to prevent. Compare the
|
|
3298
|
-
// readlink hash against the journal baseline; defer on
|
|
3299
|
-
// divergence. readlink errors (race / EACCES) also defer.
|
|
3300
|
-
try {
|
|
3301
|
-
const localHash = hashSymlinkTarget(fs.readlinkSync(localPath));
|
|
3302
|
-
const journalEntry = journal.files[key];
|
|
3303
|
-
if (journalEntry?.hash && journalEntry.hash !== localHash) {
|
|
3304
|
-
continue;
|
|
3305
|
-
}
|
|
3306
|
-
} catch {
|
|
3307
|
-
continue;
|
|
3308
|
-
}
|
|
3309
|
-
}
|
|
3310
|
-
// Directories: no hash comparison. The tombstone executor's
|
|
3311
|
-
// dir branch doesn't recursively rm-rf — it just drops the
|
|
3312
|
-
// journal entry, which is the safe-by-default behavior.
|
|
3313
|
-
} catch (err) {
|
|
3314
|
-
const code =
|
|
3315
|
-
err && typeof err === "object" && "code" in err
|
|
3316
|
-
? (err as { code?: string }).code
|
|
3317
|
-
: undefined;
|
|
3318
|
-
// ENOENT → local already gone; safe to drop journal entry via
|
|
3319
|
-
// the executor's tombstone path.
|
|
3320
|
-
// Other lstat errors (EACCES on parent dir, etc.) → defer:
|
|
3321
|
-
// we can't read local state, so we can't safely decide.
|
|
3322
|
-
if (code !== "ENOENT") continue;
|
|
3323
|
-
}
|
|
3324
|
-
const localSnapshot = snapshotLocalState(localPath);
|
|
3325
|
-
if (localSnapshot === null) continue;
|
|
3326
|
-
tombstones.push({ key, localPath, localSnapshot });
|
|
3327
|
-
}
|
|
3328
|
-
|
|
3329
|
-
return {
|
|
3330
|
-
items,
|
|
3331
|
-
filesToDownload,
|
|
3332
|
-
bytesToDownload,
|
|
3333
|
-
filesToSkip,
|
|
3334
|
-
filesToConflict,
|
|
3335
|
-
newFiles,
|
|
3336
|
-
newFilesCount: newFiles.length,
|
|
3337
|
-
filesExcludedByPolicy,
|
|
3338
|
-
filesOutOfScope,
|
|
3339
|
-
filesToTombstoneDelete,
|
|
3340
|
-
tombstones,
|
|
3341
|
-
invalidRemoteKeys,
|
|
3342
|
-
};
|
|
3343
|
-
}
|
|
3344
|
-
|
|
3345
|
-
/**
|
|
3346
|
-
* Default human-readable event rendering. Preserves the exact output format
|
|
3347
|
-
* that `hq sync` emitted before SyncProgressEvent was introduced, so callers
|
|
3348
|
-
* without an `onEvent` see no behavioral change.
|
|
3349
|
-
*/
|
|
3350
|
-
function defaultConsoleLogger(event: SyncProgressEvent): void {
|
|
3351
|
-
if (event.type === "plan") {
|
|
3352
|
-
// Terse single line so humans see what's about to happen without
|
|
3353
|
-
// drowning the per-file output that follows. Skip when there's
|
|
3354
|
-
// nothing to do — no signal, no noise.
|
|
3355
|
-
const movement = event.filesToDownload + event.filesToUpload + event.filesToConflict;
|
|
3356
|
-
if (movement > 0) {
|
|
3357
|
-
const parts: string[] = [];
|
|
3358
|
-
if (event.filesToDownload > 0) {
|
|
3359
|
-
parts.push(`${event.filesToDownload} to download (${event.bytesToDownload} bytes)`);
|
|
3360
|
-
}
|
|
3361
|
-
if (event.filesToUpload > 0) {
|
|
3362
|
-
parts.push(`${event.filesToUpload} to upload (${event.bytesToUpload} bytes)`);
|
|
3363
|
-
}
|
|
3364
|
-
if (event.filesToConflict > 0) {
|
|
3365
|
-
parts.push(`${event.filesToConflict} conflict(s)`);
|
|
3366
|
-
}
|
|
3367
|
-
parts.push(`${event.filesToSkip} unchanged`);
|
|
3368
|
-
console.log(`Plan: ${parts.join(", ")}`);
|
|
3369
|
-
}
|
|
3370
|
-
} else if (event.type === "progress") {
|
|
3371
|
-
if (event.message) {
|
|
3372
|
-
console.log(` ✓ ${event.path} — "${event.message}"`);
|
|
3373
|
-
} else {
|
|
3374
|
-
console.log(` ✓ ${event.path}`);
|
|
3375
|
-
}
|
|
3376
|
-
} else if (event.type === "error") {
|
|
3377
|
-
console.error(` ✗ ${event.path} — ${event.message}`);
|
|
3378
|
-
} else if (event.type === "conflict") {
|
|
3379
|
-
console.error(
|
|
3380
|
-
` ⚠ conflict (${event.direction}): ${event.path} — ${event.resolution}`,
|
|
3381
|
-
);
|
|
3382
|
-
} else if (event.type === "reconciled") {
|
|
3383
|
-
console.log(
|
|
3384
|
-
` = reconciled (${event.direction}): ${event.path} — remote bytes identical to local, journal re-stamped`,
|
|
3385
|
-
);
|
|
3386
|
-
} else if (event.type === "new-files") {
|
|
3387
|
-
if (event.files.length > 0) {
|
|
3388
|
-
console.log(`${event.files.length} new file${event.files.length === 1 ? "" : "s"}`);
|
|
3389
|
-
const MAX_SHOWN = 20;
|
|
3390
|
-
const shown = event.files.slice(0, MAX_SHOWN);
|
|
3391
|
-
for (const f of shown) {
|
|
3392
|
-
const who = f.addedBy ? ` (added by ${f.addedBy})` : "";
|
|
3393
|
-
console.log(` + ${f.path}${who}`);
|
|
3394
|
-
}
|
|
3395
|
-
if (event.files.length > MAX_SHOWN) {
|
|
3396
|
-
console.log(` ... and ${event.files.length - MAX_SHOWN} more`);
|
|
3397
|
-
}
|
|
3398
|
-
}
|
|
3399
|
-
} else if (event.type === "scope-excluded") {
|
|
3400
|
-
console.log(
|
|
3401
|
-
` ~ ${event.count} path${event.count === 1 ? "" : "s"} skipped — outside your granted access (synced the in-scope subset):`,
|
|
3402
|
-
);
|
|
3403
|
-
for (const p of event.samplePaths) {
|
|
3404
|
-
console.log(` · ${p}`);
|
|
3405
|
-
}
|
|
3406
|
-
if (event.count > event.samplePaths.length) {
|
|
3407
|
-
console.log(` ... and ${event.count - event.samplePaths.length} more`);
|
|
3408
|
-
}
|
|
3409
|
-
} else if (event.type === "scope-materialization-gap") {
|
|
3410
|
-
console.warn(
|
|
3411
|
-
` ! ${event.count} item${event.count === 1 ? "" : "s"} in this vault did NOT sync to this device — your access level is "${event.syncMode}", so only shared paths materialize. To pull everything you have access to, set this membership's access level to "all":`,
|
|
3412
|
-
);
|
|
3413
|
-
for (const p of event.samplePaths) {
|
|
3414
|
-
console.warn(` · ${p}`);
|
|
3415
|
-
}
|
|
3416
|
-
if (event.count > event.samplePaths.length) {
|
|
3417
|
-
console.warn(` ... and ${event.count - event.samplePaths.length} more`);
|
|
3418
|
-
}
|
|
3419
|
-
} else if (event.type === "ignore-excluded") {
|
|
3420
|
-
console.warn(
|
|
3421
|
-
` ! ${event.count} path${event.count === 1 ? "" : "s"} were EXCLUDED from sync by an ignore rule and did NOT reach the vault (review in case this is unintended):`,
|
|
3422
|
-
);
|
|
3423
|
-
for (const p of event.samplePaths) {
|
|
3424
|
-
console.warn(` · ${p}`);
|
|
3425
|
-
}
|
|
3426
|
-
if (event.count > event.samplePaths.length) {
|
|
3427
|
-
console.warn(` ... and ${event.count - event.samplePaths.length} more`);
|
|
3428
|
-
}
|
|
3429
|
-
} else if (event.type === "skip-size-limit") {
|
|
3430
|
-
const mb = (event.bytes / (1024 * 1024)).toFixed(1);
|
|
3431
|
-
console.warn(
|
|
3432
|
-
` ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`,
|
|
3433
|
-
);
|
|
3434
|
-
} else if (event.type === "skip-invalid-scoped-key") {
|
|
3435
|
-
console.warn(
|
|
3436
|
-
` ! ${event.path} skipped — invalid vault key (${event.errorCode ?? "INVALID_KEY_COMPANIES_SCOPED"})`,
|
|
3437
|
-
);
|
|
3438
|
-
} else if (event.type === "skip-junk-key-spelling") {
|
|
3439
|
-
console.warn(
|
|
3440
|
-
` ! ${event.path} skipped — junk key spelling; this logical path is already synced as ${event.journaledKey} (mixed-version peer artifact; the vault doctor collapses the family)`,
|
|
3441
|
-
);
|
|
3442
|
-
} else if (event.type === "not-shipped") {
|
|
3443
|
-
if (event.reason === "unreachable-path") {
|
|
3444
|
-
console.warn(
|
|
3445
|
-
` ! ${event.count} named path${event.count === 1 ? "" : "s"} could NOT be pushed — the file exists but is not reachable under the company folder (nothing was uploaded for ${event.count === 1 ? "it" : "them"}):`,
|
|
3446
|
-
);
|
|
3447
|
-
} else if (event.reason === "unreadable-link") {
|
|
3448
|
-
console.warn(
|
|
3449
|
-
` ! ${event.count} symbolic link${event.count === 1 ? "" : "s"} could NOT be read and was skipped without dereferencing its target:`,
|
|
3450
|
-
);
|
|
3451
|
-
} else {
|
|
3452
|
-
console.warn(
|
|
3453
|
-
` ! ${event.count} linked subtree${event.count === 1 ? "" : "s"} recorded but NOT uploaded — contents sync via their own repo, not the vault:`,
|
|
3454
|
-
);
|
|
3455
|
-
}
|
|
3456
|
-
for (const p of event.samplePaths) {
|
|
3457
|
-
console.warn(` · ${p}`);
|
|
3458
|
-
}
|
|
3459
|
-
if (event.count > event.samplePaths.length) {
|
|
3460
|
-
console.warn(` ... and ${event.count - event.samplePaths.length} more`);
|
|
3461
|
-
}
|
|
3462
|
-
if (event.reason === "linked-subtree") {
|
|
3463
|
-
console.warn(
|
|
3464
|
-
` To put a file from a linked subtree in the vault, push it by name:\n` +
|
|
3465
|
-
` hq sync push <path> --company <slug>\n` +
|
|
3466
|
-
` That uploads a point-in-time copy; re-run it after each edit (regular sync will not track or update it).`,
|
|
3467
|
-
);
|
|
3468
|
-
}
|
|
3469
|
-
}
|
|
3470
|
-
}
|