@indigoai-us/hq-cloud 6.15.0 → 6.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. package/dist/bin/sync-mutation.d.ts +16 -0
  2. package/dist/bin/sync-mutation.d.ts.map +1 -0
  3. package/dist/bin/sync-mutation.js +60 -0
  4. package/dist/bin/sync-mutation.js.map +1 -0
  5. package/dist/bin/sync-mutation.test.d.ts +2 -0
  6. package/dist/bin/sync-mutation.test.d.ts.map +1 -0
  7. package/dist/bin/sync-mutation.test.js +165 -0
  8. package/dist/bin/sync-mutation.test.js.map +1 -0
  9. package/dist/bin/sync-runner-company.d.ts +8 -0
  10. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-company.js +16 -0
  12. package/dist/bin/sync-runner-company.js.map +1 -1
  13. package/dist/bin/sync-runner-company.test.d.ts +2 -0
  14. package/dist/bin/sync-runner-company.test.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-company.test.js +36 -0
  16. package/dist/bin/sync-runner-company.test.js.map +1 -0
  17. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  18. package/dist/bin/sync-runner-watch-loop.js +98 -8
  19. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  20. package/dist/bin/sync-runner.d.ts +17 -0
  21. package/dist/bin/sync-runner.d.ts.map +1 -1
  22. package/dist/bin/sync-runner.js.map +1 -1
  23. package/dist/bin/sync-runner.test.js +109 -0
  24. package/dist/bin/sync-runner.test.js.map +1 -1
  25. package/dist/cli/conflict-recovery.test.d.ts +2 -0
  26. package/dist/cli/conflict-recovery.test.d.ts.map +1 -0
  27. package/dist/cli/conflict-recovery.test.js +201 -0
  28. package/dist/cli/conflict-recovery.test.js.map +1 -0
  29. package/dist/cli/conflict.d.ts +60 -0
  30. package/dist/cli/conflict.d.ts.map +1 -1
  31. package/dist/cli/conflict.js +333 -0
  32. package/dist/cli/conflict.js.map +1 -1
  33. package/dist/cli/sync.d.ts +35 -0
  34. package/dist/cli/sync.d.ts.map +1 -1
  35. package/dist/cli/sync.js +100 -0
  36. package/dist/cli/sync.js.map +1 -1
  37. package/dist/cli/sync.test.js +85 -1
  38. package/dist/cli/sync.test.js.map +1 -1
  39. package/dist/index.d.ts +2 -0
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +1 -0
  42. package/dist/index.js.map +1 -1
  43. package/dist/skill-telemetry.d.ts +6 -0
  44. package/dist/skill-telemetry.d.ts.map +1 -1
  45. package/dist/skill-telemetry.js +14 -2
  46. package/dist/skill-telemetry.js.map +1 -1
  47. package/dist/skill-telemetry.test.js +79 -0
  48. package/dist/skill-telemetry.test.js.map +1 -1
  49. package/dist/sync/candidate-uploader.d.ts +88 -0
  50. package/dist/sync/candidate-uploader.d.ts.map +1 -0
  51. package/dist/sync/candidate-uploader.js +212 -0
  52. package/dist/sync/candidate-uploader.js.map +1 -0
  53. package/dist/sync/candidate-uploader.test.d.ts +2 -0
  54. package/dist/sync/candidate-uploader.test.d.ts.map +1 -0
  55. package/dist/sync/candidate-uploader.test.js +132 -0
  56. package/dist/sync/candidate-uploader.test.js.map +1 -0
  57. package/dist/sync/delta-client.d.ts +73 -0
  58. package/dist/sync/delta-client.d.ts.map +1 -0
  59. package/dist/sync/delta-client.js +201 -0
  60. package/dist/sync/delta-client.js.map +1 -0
  61. package/dist/sync/delta-client.test.d.ts +2 -0
  62. package/dist/sync/delta-client.test.d.ts.map +1 -0
  63. package/dist/sync/delta-client.test.js +97 -0
  64. package/dist/sync/delta-client.test.js.map +1 -0
  65. package/dist/sync/durable-apply.d.ts +76 -0
  66. package/dist/sync/durable-apply.d.ts.map +1 -0
  67. package/dist/sync/durable-apply.js +530 -0
  68. package/dist/sync/durable-apply.js.map +1 -0
  69. package/dist/sync/durable-apply.test.d.ts +2 -0
  70. package/dist/sync/durable-apply.test.d.ts.map +1 -0
  71. package/dist/sync/durable-apply.test.js +180 -0
  72. package/dist/sync/durable-apply.test.js.map +1 -0
  73. package/dist/sync/event-sync.d.ts +33 -1
  74. package/dist/sync/event-sync.d.ts.map +1 -1
  75. package/dist/sync/event-sync.js +149 -1
  76. package/dist/sync/event-sync.js.map +1 -1
  77. package/dist/sync/event-sync.test.js +142 -1
  78. package/dist/sync/event-sync.test.js.map +1 -1
  79. package/dist/sync/index.d.ts +2 -0
  80. package/dist/sync/index.d.ts.map +1 -1
  81. package/dist/sync/index.js +1 -0
  82. package/dist/sync/index.js.map +1 -1
  83. package/dist/sync/multipart-uploader.d.ts +99 -0
  84. package/dist/sync/multipart-uploader.d.ts.map +1 -0
  85. package/dist/sync/multipart-uploader.js +447 -0
  86. package/dist/sync/multipart-uploader.js.map +1 -0
  87. package/dist/sync/multipart-uploader.test.d.ts +2 -0
  88. package/dist/sync/multipart-uploader.test.d.ts.map +1 -0
  89. package/dist/sync/multipart-uploader.test.js +119 -0
  90. package/dist/sync/multipart-uploader.test.js.map +1 -0
  91. package/dist/sync/mutation-client.d.ts +85 -0
  92. package/dist/sync/mutation-client.d.ts.map +1 -0
  93. package/dist/sync/mutation-client.js +245 -0
  94. package/dist/sync/mutation-client.js.map +1 -0
  95. package/dist/sync/mutation-client.test.d.ts +2 -0
  96. package/dist/sync/mutation-client.test.d.ts.map +1 -0
  97. package/dist/sync/mutation-client.test.js +51 -0
  98. package/dist/sync/mutation-client.test.js.map +1 -0
  99. package/dist/sync/push-receiver.d.ts +45 -0
  100. package/dist/sync/push-receiver.d.ts.map +1 -1
  101. package/dist/sync/push-receiver.js +101 -0
  102. package/dist/sync/push-receiver.js.map +1 -1
  103. package/dist/sync/push-receiver.test.js +54 -2
  104. package/dist/sync/push-receiver.test.js.map +1 -1
  105. package/dist/sync/scope-inventory-client.d.ts +69 -0
  106. package/dist/sync/scope-inventory-client.d.ts.map +1 -0
  107. package/dist/sync/scope-inventory-client.js +210 -0
  108. package/dist/sync/scope-inventory-client.js.map +1 -0
  109. package/dist/sync/scope-inventory-client.test.d.ts +2 -0
  110. package/dist/sync/scope-inventory-client.test.d.ts.map +1 -0
  111. package/dist/sync/scope-inventory-client.test.js +94 -0
  112. package/dist/sync/scope-inventory-client.test.js.map +1 -0
  113. package/dist/sync/snapshot-client.d.ts +98 -0
  114. package/dist/sync/snapshot-client.d.ts.map +1 -0
  115. package/dist/sync/snapshot-client.js +402 -0
  116. package/dist/sync/snapshot-client.js.map +1 -0
  117. package/dist/sync/snapshot-client.test.d.ts +2 -0
  118. package/dist/sync/snapshot-client.test.d.ts.map +1 -0
  119. package/dist/sync/snapshot-client.test.js +169 -0
  120. package/dist/sync/snapshot-client.test.js.map +1 -0
  121. package/dist/sync/uploader-finalization.d.ts +97 -0
  122. package/dist/sync/uploader-finalization.d.ts.map +1 -0
  123. package/dist/sync/uploader-finalization.js +273 -0
  124. package/dist/sync/uploader-finalization.js.map +1 -0
  125. package/dist/sync/uploader-finalization.test.d.ts +2 -0
  126. package/dist/sync/uploader-finalization.test.d.ts.map +1 -0
  127. package/dist/sync/uploader-finalization.test.js +92 -0
  128. package/dist/sync/uploader-finalization.test.js.map +1 -0
  129. package/dist/telemetry.d.ts +11 -1
  130. package/dist/telemetry.d.ts.map +1 -1
  131. package/dist/telemetry.js +21 -2
  132. package/dist/telemetry.js.map +1 -1
  133. package/dist/telemetry.test.js +80 -0
  134. package/dist/telemetry.test.js.map +1 -1
  135. package/package.json +6 -1
  136. package/.claude/policies/hq-cloud-esm-cannot-spy-fs-builtins.md +0 -30
  137. package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +0 -22
  138. package/.github/workflows/ci.yml +0 -84
  139. package/.github/workflows/publish.yml +0 -56
  140. package/.github/workflows/unreleased-commits-nag.yml +0 -256
  141. package/eslint.config.js +0 -67
  142. package/pnpm-workspace.yaml +0 -2
  143. package/scripts/presign-transport-e2e.mjs +0 -250
  144. package/scripts/vault-rebaseline.sh +0 -323
  145. package/scripts/vault-rescue.sh +0 -332
  146. package/src/active-company.test.ts +0 -188
  147. package/src/active-company.ts +0 -168
  148. package/src/agent-codex-instructions.test.ts +0 -332
  149. package/src/agent-codex-instructions.ts +0 -309
  150. package/src/auth.ts +0 -146
  151. package/src/backup-prune.test.ts +0 -98
  152. package/src/backup-prune.ts +0 -182
  153. package/src/bin/backup-prune-runner.ts +0 -33
  154. package/src/bin/rescue-runner.ts +0 -25
  155. package/src/bin/sync-runner-company.ts +0 -695
  156. package/src/bin/sync-runner-events.test.ts +0 -143
  157. package/src/bin/sync-runner-events.ts +0 -55
  158. package/src/bin/sync-runner-planning.test.ts +0 -311
  159. package/src/bin/sync-runner-planning.ts +0 -258
  160. package/src/bin/sync-runner-rollup.test.ts +0 -37
  161. package/src/bin/sync-runner-rollup.ts +0 -97
  162. package/src/bin/sync-runner-telemetry.ts +0 -15
  163. package/src/bin/sync-runner-watch-loop.ts +0 -1235
  164. package/src/bin/sync-runner-watch-routes.test.ts +0 -71
  165. package/src/bin/sync-runner-watch-routes.ts +0 -184
  166. package/src/bin/sync-runner.test.ts +0 -8767
  167. package/src/bin/sync-runner.ts +0 -2190
  168. package/src/cli/accept.ts +0 -124
  169. package/src/cli/conflict.ts +0 -119
  170. package/src/cli/doctor.test.ts +0 -581
  171. package/src/cli/doctor.ts +0 -642
  172. package/src/cli/index.ts +0 -49
  173. package/src/cli/invite.test.ts +0 -250
  174. package/src/cli/invite.ts +0 -214
  175. package/src/cli/promote.ts +0 -157
  176. package/src/cli/reindex-knowledge.test.ts +0 -307
  177. package/src/cli/reindex-knowledge.ts +0 -450
  178. package/src/cli/reindex.test.ts +0 -957
  179. package/src/cli/reindex.ts +0 -979
  180. package/src/cli/rescue-classify-ordering.test.ts +0 -548
  181. package/src/cli/rescue-clone-diagnostics.test.ts +0 -120
  182. package/src/cli/rescue-core.ts +0 -3011
  183. package/src/cli/rescue-drift-reconcile.test.ts +0 -179
  184. package/src/cli/rescue-drop-dir-symlink.test.ts +0 -224
  185. package/src/cli/rescue-exec-bit-preserve.test.ts +0 -187
  186. package/src/cli/rescue-hq-root-guard.test.ts +0 -232
  187. package/src/cli/rescue-journal-reconcile.test.ts +0 -215
  188. package/src/cli/rescue-mtime-preserve.test.ts +0 -203
  189. package/src/cli/rescue-settings-reconcile.test.ts +0 -637
  190. package/src/cli/rescue-snapshot.test.ts +0 -57
  191. package/src/cli/rescue-snapshot.ts +0 -51
  192. package/src/cli/rescue.reindex.test.ts +0 -63
  193. package/src/cli/rescue.test.ts +0 -131
  194. package/src/cli/rescue.ts +0 -182
  195. package/src/cli/share.test.ts +0 -7843
  196. package/src/cli/share.ts +0 -3663
  197. package/src/cli/sync-scope.test.ts +0 -652
  198. package/src/cli/sync.test.ts +0 -5207
  199. package/src/cli/sync.ts +0 -3470
  200. package/src/cli/tombstones.ts +0 -106
  201. package/src/cli/watch-event-push-conflict.test.ts +0 -234
  202. package/src/client-info.test.ts +0 -214
  203. package/src/client-info.ts +0 -121
  204. package/src/cognito-auth.test.ts +0 -712
  205. package/src/cognito-auth.ts +0 -1422
  206. package/src/company-resolver.test.ts +0 -618
  207. package/src/company-resolver.ts +0 -521
  208. package/src/context.test.ts +0 -583
  209. package/src/context.ts +0 -378
  210. package/src/daemon-worker.ts +0 -26
  211. package/src/daemon.ts +0 -99
  212. package/src/entity-resolver.test.ts +0 -315
  213. package/src/entity-resolver.ts +0 -180
  214. package/src/ignore.test.ts +0 -466
  215. package/src/ignore.ts +0 -469
  216. package/src/index.ts +0 -439
  217. package/src/journal.test.ts +0 -968
  218. package/src/journal.ts +0 -765
  219. package/src/lib/cloud-authoritative.test.ts +0 -45
  220. package/src/lib/cloud-authoritative.ts +0 -59
  221. package/src/lib/conflict-file.ts +0 -86
  222. package/src/lib/conflict-index.ts +0 -289
  223. package/src/lib/conflict.test.ts +0 -348
  224. package/src/lib/describe-error.test.ts +0 -100
  225. package/src/lib/describe-error.ts +0 -58
  226. package/src/lib/exit-codes.ts +0 -24
  227. package/src/lib/machine-id.test.ts +0 -231
  228. package/src/lib/machine-id.ts +0 -175
  229. package/src/lib/net-errors.test.ts +0 -65
  230. package/src/lib/net-errors.ts +0 -86
  231. package/src/lib/readlink-safe.test.ts +0 -43
  232. package/src/lib/readlink-safe.ts +0 -29
  233. package/src/local-path-codec.test.ts +0 -138
  234. package/src/local-path-codec.ts +0 -161
  235. package/src/machine-auth.test.ts +0 -1323
  236. package/src/manifest-reconcile.test.ts +0 -1123
  237. package/src/manifest-reconcile.ts +0 -518
  238. package/src/object-io.test.ts +0 -1221
  239. package/src/object-io.ts +0 -1306
  240. package/src/operation-lock.test.ts +0 -484
  241. package/src/operation-lock.ts +0 -680
  242. package/src/outcome-telemetry.test.ts +0 -498
  243. package/src/outcome-telemetry.ts +0 -639
  244. package/src/personal-vault-exclusions.test.ts +0 -308
  245. package/src/personal-vault-exclusions.ts +0 -354
  246. package/src/personal-vault.test.ts +0 -756
  247. package/src/personal-vault.ts +0 -496
  248. package/src/prefix-coalesce.test.ts +0 -240
  249. package/src/prefix-coalesce.ts +0 -273
  250. package/src/public-surface.test.ts +0 -117
  251. package/src/qmd-reindex.test.ts +0 -877
  252. package/src/qmd-reindex.ts +0 -842
  253. package/src/read-only-state-dir.test.ts +0 -188
  254. package/src/remote-pull.test.ts +0 -1130
  255. package/src/remote-pull.ts +0 -618
  256. package/src/s3.symlink-materialize.test.ts +0 -492
  257. package/src/s3.test.ts +0 -1789
  258. package/src/s3.ts +0 -1532
  259. package/src/schemas/signal-types.test.ts +0 -82
  260. package/src/schemas/signal-types.ts +0 -38
  261. package/src/schemas/source-channels.test.ts +0 -82
  262. package/src/schemas/source-channels.ts +0 -53
  263. package/src/scope-shrink.test.ts +0 -633
  264. package/src/scope-shrink.ts +0 -481
  265. package/src/signals/get.test.ts +0 -310
  266. package/src/signals/get.ts +0 -75
  267. package/src/signals/internals.ts +0 -195
  268. package/src/signals/list.test.ts +0 -420
  269. package/src/signals/list.ts +0 -79
  270. package/src/signals/parse.ts +0 -8
  271. package/src/signals/types.ts +0 -91
  272. package/src/skill-telemetry.test.ts +0 -1825
  273. package/src/skill-telemetry.ts +0 -1439
  274. package/src/sources/get.test.ts +0 -293
  275. package/src/sources/get.ts +0 -66
  276. package/src/sources/internals.ts +0 -198
  277. package/src/sources/list.test.ts +0 -402
  278. package/src/sources/list.ts +0 -84
  279. package/src/sources/parse.ts +0 -43
  280. package/src/sources/types.ts +0 -84
  281. package/src/sync/event-sync.test.ts +0 -594
  282. package/src/sync/event-sync.ts +0 -545
  283. package/src/sync/feature-flags.test.ts +0 -378
  284. package/src/sync/feature-flags.ts +0 -62
  285. package/src/sync/index.ts +0 -76
  286. package/src/sync/lease-client.test.ts +0 -128
  287. package/src/sync/lease-client.ts +0 -207
  288. package/src/sync/logger.test.ts +0 -242
  289. package/src/sync/logger.ts +0 -79
  290. package/src/sync/metrics.test.ts +0 -462
  291. package/src/sync/metrics.ts +0 -213
  292. package/src/sync/pull-scope.ts +0 -265
  293. package/src/sync/push-event.test.ts +0 -266
  294. package/src/sync/push-event.ts +0 -224
  295. package/src/sync/push-receiver.test.ts +0 -566
  296. package/src/sync/push-receiver.ts +0 -1048
  297. package/src/sync/push-transport.ts +0 -231
  298. package/src/sync/realtime-rollout.test.ts +0 -86
  299. package/src/sync/realtime-rollout.ts +0 -262
  300. package/src/sync/state-store.test.ts +0 -194
  301. package/src/sync/state-store.ts +0 -727
  302. package/src/sync-core.ts +0 -58
  303. package/src/sync-progress.test.ts +0 -94
  304. package/src/sync-progress.ts +0 -140
  305. package/src/telemetry-events.test.ts +0 -88
  306. package/src/telemetry-events.ts +0 -205
  307. package/src/telemetry.test.ts +0 -1280
  308. package/src/telemetry.ts +0 -1109
  309. package/src/types.ts +0 -314
  310. package/src/vault-client.test.ts +0 -1380
  311. package/src/vault-client.ts +0 -1694
  312. package/src/version.ts +0 -24
  313. package/src/watch-roots.test.ts +0 -278
  314. package/src/watch-roots.ts +0 -162
  315. package/src/watcher-event-gate.test.ts +0 -212
  316. package/src/watcher.test.ts +0 -1079
  317. package/src/watcher.ts +0 -1741
  318. package/test/e2e/sync/cross-tenant-isolation.test.ts +0 -630
  319. package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +0 -124
  320. package/test/e2e/sync/transient-company-leg.test.ts +0 -384
  321. package/test/e2e/sync/windows-unreadable-link-leg.test.ts +0 -191
  322. package/test/e2e/watcher-real-chokidar.test.ts +0 -165
  323. package/test/e2e/watcher-recursive-backend.test.ts +0 -181
  324. package/test/e2e/watcher-scoped-coverage.test.ts +0 -381
  325. package/test/invite-flow.integration.test.ts +0 -244
  326. package/test/joiner-manifest-reconcile.integration.test.ts +0 -322
  327. package/test/share-sync.integration.test.ts +0 -213
  328. package/tsconfig.json +0 -19
  329. package/vitest.config.ts +0 -22
package/src/cli/sync.ts DELETED
@@ -1,3470 +0,0 @@
1
- /**
2
- * `hq sync` command — pull everything allowed from entity vault (VLT-5 US-002).
3
- *
4
- * Pulls all files the caller's STS session policy permits.
5
- * Never auto-overwrites local changes — prompts on conflict.
6
- */
7
-
8
- import * as fs from "fs";
9
- import * as path from "path";
10
- import type {
11
- EntityContext,
12
- VaultServiceConfig,
13
- SyncJournal,
14
- } from "../types.js";
15
- import { VaultAuthError, VaultClient, type SyncMode } from "../vault-client.js";
16
- import { readlinkOrNull } from "../lib/readlink-safe.js";
17
- import {
18
- emitCloudTelemetry,
19
- type TelemetryClaims,
20
- } from "../telemetry-events.js";
21
- import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
22
- import { createSyncProgressRecorder } from "../sync-progress.js";
23
- import {
24
- canonicalVaultKeySpelling,
25
- localPathForVaultKey,
26
- vaultKeyForLocalPath,
27
- } from "../local-path-codec.js";
28
- import { SKILLS_KEY_PREFIX } from "./doctor.js";
29
- import {
30
- downloadFile,
31
- listRemoteFiles,
32
- headRemoteFile,
33
- primeObjectTransport,
34
- toPosixKey,
35
- classifyVaultKey,
36
- DanglingSymlinkParentError,
37
- } from "../s3.js";
38
- import type { DownloadModeWarning, RemoteFile } from "../s3.js";
39
- import {
40
- readJournal,
41
- writeJournal,
42
- hashFile,
43
- hashSymlinkTarget,
44
- updateEntry,
45
- removeEntry,
46
- getEntry,
47
- normalizeEtag,
48
- migrateToV2,
49
- gcTombstones,
50
- isTombstone,
51
- tombstoneEntry,
52
- lastPullRecord,
53
- appendPullRecord,
54
- generatePullId,
55
- PERSONAL_VAULT_JOURNAL_SLUG,
56
- migratePersonalVaultJournal,
57
- } from "../journal.js";
58
- import {
59
- isGeneratedCoreMirrorKey,
60
- PERSONAL_VAULT_MANIFEST_KEY,
61
- } from "../personal-vault.js";
62
- import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
63
- import {
64
- buildScopeShrinkPlan,
65
- applyScopeShrink,
66
- ScopeShrinkBlockedError,
67
- ScopeShrinkLargePruneError,
68
- type ScopeShrinkAdviceContext,
69
- } from "../scope-shrink.js";
70
- import {
71
- coalescePrefixes,
72
- isCoveredByAny,
73
- type ScopePrefixInput,
74
- } from "../prefix-coalesce.js";
75
- import { createIgnoreFilter } from "../ignore.js";
76
- import {
77
- hasRemoteChanged,
78
- isAccessDenied,
79
- resolveActiveCompany,
80
- resolveTransferConcurrency,
81
- } from "../sync-core.js";
82
- import { isEphemeralPath, isForbiddenCompanyVaultKey, isMalformedVaultKey } from "./share.js";
83
- import { resolveConflict } from "./conflict.js";
84
- import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
85
- import {
86
- buildConflictId,
87
- buildConflictPath,
88
- readShortMachineId,
89
- } from "../lib/conflict-file.js";
90
- import { appendConflictEntry } from "../lib/conflict-index.js";
91
- import { isCloudAuthoritative } from "../lib/cloud-authoritative.js";
92
- import { reindex } from "./reindex.js";
93
- import { withOperationLock } from "../operation-lock.js";
94
- import {
95
- fetchCompanyTombstones,
96
- type CompanyTombstone,
97
- } from "./tombstones.js";
98
-
99
- /**
100
- * Per-file events emitted by `sync()` as it progresses.
101
- *
102
- * When `SyncOptions.onEvent` is set, these events are delivered to the caller
103
- * in place of the default human-readable `console.log` / `console.error`
104
- * output. This is the seam that lets `hq-sync-runner` stream ndjson to the
105
- * AppBar menubar without the engine knowing anything about ndjson (ADR-0001).
106
- *
107
- * The human CLI (`hq sync`) leaves `onEvent` undefined and falls through to
108
- * `defaultConsoleLogger` below, which preserves the existing tty output.
109
- *
110
- * A single `plan` event is emitted once at the start of every run, before
111
- * any `progress`/`conflict`/`error` events. It carries the totals derived
112
- * from a Stage-1 classification pass so consumers can render an accurate
113
- * progress denominator before transfers begin (the menubar's "Preparing
114
- * sync…" pre-pass becomes obsolete once the runner forwards this).
115
- */
116
- export type SyncProgressEvent =
117
- | {
118
- type: "plan";
119
- /** Files this run intends to download (pull-only; 0 from share). */
120
- filesToDownload: number;
121
- bytesToDownload: number;
122
- /** Files this run intends to upload (push-only; 0 from sync). */
123
- filesToUpload: number;
124
- bytesToUpload: number;
125
- /** Files classified as no-op (ignored, unchanged, local-only on pull). */
126
- filesToSkip: number;
127
- /**
128
- * Files known up-front to be conflicts. Pull-side fills this from the
129
- * 3-way merge against the journal; push-side leaves it 0 because
130
- * conflict detection requires a remote HEAD that runs in Stage 2.
131
- */
132
- filesToConflict: number;
133
- /**
134
- * Remote keys this run intends to delete from S3 (push-only with
135
- * `propagateDeletes`; 0 from sync). A key is scheduled for deletion when
136
- * its journal entry exists, the local file is gone, and the key falls
137
- * within the share scope. The bucket has versioning enabled so the
138
- * delete is soft (a delete-marker is written; prior versions remain
139
- * recoverable).
140
- */
141
- filesToDelete: number;
142
- }
143
- | {
144
- type: "progress";
145
- path: string;
146
- bytes: number;
147
- message?: string;
148
- /** True when this event reports a remote DeleteObject (no upload). */
149
- deleted?: boolean;
150
- /**
151
- * Transfer direction for this file, stamped by the runner's per-company
152
- * tagger from the in-flight phase: `"up"` for a push (local→S3 upload),
153
- * `"down"` for a pull (S3→local download). Optional because the inner
154
- * `share()`/`sync()` emitters don't know the run-level phase — only the
155
- * runner's `tagAndEmit` does, and it sets this when re-emitting to stdout.
156
- * Consumers (the menubar activity log) use it to label each file
157
- * uploaded vs downloaded.
158
- */
159
- direction?: "up" | "down";
160
- /**
161
- * Email of the file's author, read from the S3 object's `created-by`
162
- * user-metadata. Only set on the download/pull path (a downloaded file
163
- * was authored by whoever uploaded it); push-side progress has no author
164
- * because the uploader is the local user. `null`/absent when the object
165
- * carries no `created-by`. The menubar activity log shows this so the
166
- * user sees who authored each file they received.
167
- */
168
- author?: string | null;
169
- }
170
- | { type: "error"; path: string; message: string }
171
- | {
172
- type: "conflict";
173
- path: string;
174
- direction: "pull" | "push";
175
- resolution: ConflictResolution;
176
- }
177
- | {
178
- /**
179
- * Emitted when a planner-flagged pull conflict turned out to be a false
180
- * positive: the remote bytes fetched for the conflict mirror hashed
181
- * byte-for-byte identical to the local file, so there is nothing to
182
- * resolve. The planner can only see journal-relative deltas (local hash
183
- * != journal hash AND remote etag != journal etag), so a stale journal
184
- * baseline makes identical content look like a both-sides change. Known
185
- * stale-baseline triggers: a shared-journal cross-root collision (the
186
- * personal vault + `companies/personal` sharing one journal), an
187
- * mtime-rounding fast-path miss, KMS/multipart etag churn on a no-op
188
- * re-upload, a second machine advancing S3 + its own journal, or a
189
- * manual revert. The executor re-stamps the journal baseline (so neither
190
- * side looks changed next run) and skips — no conflict counted, no
191
- * `.conflict-*` mirror written. Counted as a skip; surfaced here purely
192
- * for observability so a UI can explain why N "conflicts" silently
193
- * resolved themselves.
194
- */
195
- type: "reconciled";
196
- path: string;
197
- direction: "pull" | "push";
198
- }
199
- | {
200
- type: "new-files";
201
- files: Array<{ path: string; bytes: number; addedBy: string | null }>;
202
- }
203
- | {
204
- /**
205
- * Emitted by the `currency-gated` delete policy when a delete candidate
206
- * is refused. Two reasons, discriminated by `reason`:
207
- *
208
- * - `"stale-etag"`: the local file is missing but the remote object's
209
- * current ETag no longer matches the journal's last-recorded etag.
210
- * Some other device (or another out-of-band write) modified the
211
- * remote object since this machine last synced it. The pull leg of
212
- * `sync now` will re-pull naturally via the same `hasRemoteChanged`
213
- * path that powers conflict detection. Journal entry is left intact
214
- * so the pull can use it as the baseline for the 3-way merge.
215
- * `journalEtag` and `remoteEtag` are real ETag strings.
216
- *
217
- * - `"legacy-no-etag"`: the journal entry predates remoteEtag tracking
218
- * (no `remoteEtag` recorded). We can't prove currency without an
219
- * etag, so the delete is refused in the safe direction. A future
220
- * sync that picks up an etag for this entry can re-evaluate.
221
- * `journalEtag` and `remoteEtag` are sentinel strings
222
- * (`<legacy-no-etag>` / `<unknown>`) — do not render as ETags.
223
- * Consumers should branch on `reason`, not on the etag values.
224
- *
225
- * - `"bulk-asymmetry"`: the bulk-asymmetry circuit-breaker tripped
226
- * (see `delete-refused-bulk-asymmetry`). Each candidate that would
227
- * have been deleted also surfaces here for path-level visibility.
228
- * `journalEtag` and `remoteEtag` are sentinel strings
229
- * (`<bulk-asymmetry>` / `<not-checked>`) since no HEAD was issued.
230
- */
231
- type: "delete-refused-stale-etag";
232
- path: string;
233
- journalEtag: string;
234
- remoteEtag: string;
235
- reason:
236
- | "stale-etag"
237
- | "legacy-no-etag"
238
- | "bulk-asymmetry"
239
- | "divergent-local"
240
- | "missing-delete-intent"
241
- | "intent-changed"
242
- | "recreated-locally";
243
- }
244
- | {
245
- /**
246
- * Emitted at most ONCE per `share()` call when the bulk-asymmetry
247
- * circuit-breaker tripped: a large fraction of the in-scope journal
248
- * entries are missing locally, suggesting the local mirror is corrupt
249
- * (moved hqRoot, partial restore, fresh clone over inherited state,
250
- * unmounted volume, accidental `rm -rf`) rather than a deliberate
251
- * delete. The engine refuses to convert the missing entries into
252
- * remote `DeleteObject` calls; every refused candidate also surfaces
253
- * via `delete-refused-stale-etag` with `reason: "bulk-asymmetry"`.
254
- *
255
- * Trip condition: `candidates >= 10 AND candidates / inScope >= 0.10`.
256
- * Bypass: set `HQ_SYNC_DELETE_BULK_OVERRIDE=1` (truthy: `1|true|yes`,
257
- * case-insensitive) OR pass `propagateDeletePolicy: "all"` (the
258
- * existing emergency-reconcile policy which already opts out of
259
- * safety gates).
260
- *
261
- * `candidates` is the number of journal entries whose local file is
262
- * missing AND would have been delete-candidates absent the guard;
263
- * `inScope` is the total journal entries under the share's scope
264
- * roots; `ratio = candidates / inScope`; `samplePaths` carries up to
265
- * 10 refused keys for diagnostic display.
266
- */
267
- type: "delete-refused-bulk-asymmetry";
268
- candidates: number;
269
- inScope: number;
270
- ratio: number;
271
- samplePaths: string[];
272
- }
273
- | {
274
- /**
275
- * Emitted at most ONCE per `share()` call (push leg of a sync run) when
276
- * `personalMode === true` and the personal-vault default-exclusion list
277
- * blocked one or more files that would otherwise have uploaded. Gives
278
- * the UI a single summary signal — "N files quietly excluded by default
279
- * policy" — without firing one event per excluded file (which would
280
- * dominate the event stream on first-sync of a dirty tree).
281
- *
282
- * `count` is the total number of paths the exclusion filter rejected
283
- * (deduplicated across the walk). `samplePaths` carries up to 10
284
- * forward-slash-separated relative paths for diagnostic display. `byId`
285
- * is a per-exclusion-rule breakdown so the UI can render which class
286
- * of exclusion did the work (secret / machine-local / scratch / …).
287
- *
288
- * Not emitted when `count === 0` — silent on a clean tree.
289
- */
290
- type: "personal-vault-out-of-policy";
291
- count: number;
292
- samplePaths: string[];
293
- byId: Record<string, number>;
294
- }
295
- | {
296
- /**
297
- * Emitted at most ONCE per `share()` call (push leg) when the run is
298
- * scoped to a member/guest's ACL prefixes (`prefixSet`) and one or more
299
- * candidate paths fell OUTSIDE the granted prefixes. Those paths are
300
- * skipped from the upload (and delete) plan instead of being PUT — the
301
- * vended child credential is scoped to the granted prefixes, so pushing
302
- * them would draw the server's correct 403 `SCOPE_EXCEEDS_PARENT` and
303
- * (pre-fix) abort the WHOLE company. This is the push-side analogue of
304
- * the pull-side `skip-out-of-scope` action.
305
- *
306
- * `count` is the number of distinct company-relative paths/prefixes the
307
- * scope filter excluded; `samplePaths` carries up to 10 for diagnostic
308
- * display. Informational, NOT an error — the company still syncs its
309
- * in-scope subset and the run exits 0.
310
- *
311
- * Not emitted when `count === 0` — silent on a fully in-scope tree.
312
- */
313
- type: "scope-excluded";
314
- count: number;
315
- samplePaths: string[];
316
- }
317
- | {
318
- /**
319
- * Emitted at most ONCE per PULL leg when the leg ran under a
320
- * membership-scoped `syncMode` (`"shared"` or `"custom"` — never `"all"`)
321
- * AND one or more remote keys were withheld BY THAT MEMBERSHIP SCOPE
322
- * (`skip-out-of-scope` with `reason: "membership-scope"`). This turns the
323
- * silent shared-vs-all gap into a
324
- * VISIBLE, actionable surface. The reporter of feedback_d2082110 lost
325
- * files across devices precisely because four memberships defaulted to
326
- * `shared` rather than `all`, so not all vault content materialized on the
327
- * second device — and nothing told them; they had to notice the gap and
328
- * flip each membership to `all` by hand. This event names the gap and the
329
- * lever (raise the membership's access level to `all`) so FULL
330
- * materialization is a deliberate, visible choice rather than a silent
331
- * omission.
332
- *
333
- * `count` is the number of remote keys the MEMBERSHIP SCOPE withheld on
334
- * this leg; `samplePaths` carries up to 10 company-relative keys for
335
- * display; `syncMode` is the active scoped mode. Push-only keys
336
- * (`sessions/`, US-006) are deliberately EXCLUDED from both — they are
337
- * withheld in every mode including `all`, so the lever this event names
338
- * would not materialize them. `count` is therefore ≤
339
- * `SyncResult.filesOutOfScope`, which keeps its both-causes meaning.
340
- * Distinct from the push-side `scope-excluded` (which reports what a
341
- * grantee could not PUSH). Not emitted in `all` mode or when nothing was
342
- * withheld by the membership scope — no gap, no noise.
343
- */
344
- type: "scope-materialization-gap";
345
- count: number;
346
- samplePaths: string[];
347
- syncMode: SyncMode;
348
- }
349
- | {
350
- /**
351
- * Emitted at most ONCE per `share()` push leg when the base ignore
352
- * filter dropped one or more NOTEWORTHY paths: content that does not
353
- * match expected build/VCS/cache noise classes. Surfaces an over-broad
354
- * exclusion that would otherwise drop content silently (DEV-1791).
355
- *
356
- * `count` is the number of distinct noteworthy paths; `totalExcluded`
357
- * is all base-ignore rejections including expected noise; `samplePaths`
358
- * carries up to 10 noteworthy paths. Not emitted when `count === 0`.
359
- */
360
- type: "ignore-excluded";
361
- count: number;
362
- totalExcluded: number;
363
- samplePaths: string[];
364
- }
365
- | {
366
- /**
367
- * Emitted by the PUSH leg (`share()`) once per key whose upload was
368
- * suppressed because an authoritative FILE_TOMBSTONE marks it deleted and
369
- * the local copy is still the deleted baseline (journal hash unchanged).
370
- * Without this, a behind peer that still holds the file would re-upload it,
371
- * and the pull planner's timestamp-only re-create heuristic
372
- * (`isRemoteRecreateAfterTombstone`) would treat the re-upload as a genuine
373
- * re-create and resurrect the key for everyone. The deleter's own stale
374
- * local copy is cleaned by the pull leg's `tombstone-delete`; this event
375
- * records that the push refused to resurrect it. `deletedAt` is the
376
- * tombstone's delete time.
377
- */
378
- type: "upload-suppressed-tombstone";
379
- path: string;
380
- deletedAt: string;
381
- }
382
- | {
383
- /**
384
- * Emitted by the PUSH leg (`share()`) once per local file skipped because
385
- * it exceeds the max sync size (`isWithinSizeLimit`, 50 MB default). This
386
- * is a permanent, benign, user-policy outcome — the file is too big to
387
- * sync and will be over the cap on every pass — so it is NOT an error and
388
- * the run exits 0. It was previously emitted as `type: "error"`, which
389
- * pushed it into the runner's `errors[]` and made every watch pass return
390
- * exit 2; the menubar supervisor then reported that as
391
- * "auto-sync watcher exited unexpectedly (code=Some(2))" on every tick —
392
- * the HQ-SYNC-4 flood across user machines. Surfaced here purely for visibility
393
- * (`bytes` is the offending file's size) so a UI/CLI can tell the user
394
- * which file was too big without treating it as a failure.
395
- */
396
- type: "skip-size-limit";
397
- path: string;
398
- bytes: number;
399
- }
400
- | {
401
- /**
402
- * Emitted once per key skipped because it is permanently unstorable.
403
- *
404
- * PULL leg: a remote key the server will never presign (for example, a
405
- * `companies/…`-prefixed key in a company-scoped vault or a key with
406
- * control characters) — bucket poisoning from an outdated direct-S3
407
- * client. PUSH leg: a local key the upload validator would reject, so
408
- * the upload can only ever throw — a doubled-tree `companies/…` key, or
409
- * a control-character path such as a macOS Finder `Icon\r` file. Like
410
- * `skip-size-limit`, this is deliberately NOT `type: "error"` — one bad
411
- * object must never error a whole company sync (erroring gated
412
- * heartbeat toolset refreshes fleet-wide in the incident). Surfaced for
413
- * visibility + cloud telemetry; per policy
414
- * hq-alert-baseline-calibration the healthy baseline is ZERO such keys,
415
- * so any occurrence is signal.
416
- */
417
- type: "skip-invalid-scoped-key";
418
- path: string;
419
- /** Server-compatible invalid-key code, such as INVALID_KEY_CONTROL_CHARS. */
420
- errorCode?: string;
421
- }
422
- | {
423
- /**
424
- * Emitted by the PULL leg once per remote key skipped because its
425
- * spelling percent-decodes to the same logical path as an
426
- * already-journaled key with a DIFFERENT spelling — the mixed-version
427
- * amplifier (a pre-codec peer minting `%3A`/`%253A` twins of a
428
- * canonical `:` key). Downloading the twin would materialize a junk
429
- * local dir and fork the journal into a second key family for one
430
- * logical path, so the established spelling wins and the twin is
431
- * skipped. Like `skip-size-limit`, this is deliberately NOT
432
- * `type: "error"` (policy hq-sync-deliberate-skip-not-fatal-error-exit2:
433
- * a recurring benign per-file skip emitted as an error lands in the
434
- * runner's `errors[]`, forces exit 2 on every pass, and the menubar
435
- * Sentry-alerts each time). The US-003 vault doctor is the cleanup
436
- * path that collapses the junk family itself.
437
- */
438
- type: "skip-junk-key-spelling";
439
- path: string;
440
- /** The already-journaled spelling that owns this logical path. */
441
- journaledKey: string;
442
- }
443
- | {
444
- /**
445
- * Emitted when local content is deliberately not transferred and would
446
- * otherwise vanish from sync output without a trace (feedback_258e4a86 /
447
- * feedback_a51cb63d — an hour lost because a "Pushed 0 file(s)" success
448
- * named nothing that was skipped). Push batches by reason; pull emits an
449
- * unreadable link by its remote key. `reason` distinguishes the cause:
450
- *
451
- * - `"unreachable-path"`: a path the caller EXPLICITLY named exists on
452
- * disk but the resolver could not place it under the company folder,
453
- * or could not find it under any base. This is actionable signal — the
454
- * named target was NOT shipped — so the CLI may treat it as an error.
455
- * - `"linked-subtree"`: a directory symlink whose target lives OUTSIDE
456
- * the company folder was recorded as a link but its contents were not
457
- * descended/uploaded. They sync via their own repo, not the vault;
458
- * this is informational, not an error.
459
- * - `"unreadable-link"`: the OS identified a local symbolic link but
460
- * did not expose a readable target. The link is skipped rather than
461
- * dereferenced, and the company leg remains complete.
462
- *
463
- * `count` is the number of distinct paths for that reason; `samplePaths`
464
- * carries up to 10 for display. Not emitted when `count === 0`.
465
- */
466
- type: "not-shipped";
467
- reason: "unreachable-path" | "unreadable-link" | "linked-subtree";
468
- count: number;
469
- samplePaths: string[];
470
- };
471
-
472
- export interface SyncOptions {
473
- /** Company slug or UID (defaults to active company from config) */
474
- company?: string;
475
- /** Non-interactive conflict strategy */
476
- onConflict?: ConflictStrategy;
477
- /** Vault service config */
478
- vaultConfig: VaultServiceConfig;
479
- /** HQ root directory */
480
- hqRoot: string;
481
- /**
482
- * Per-file event callback. When present, suppresses the default
483
- * `console.log`/`console.error` human output — the caller is expected to
484
- * render events themselves (e.g. emit ndjson to stdout). When absent, the
485
- * default human logger is used. See `SyncProgressEvent`.
486
- */
487
- onEvent?: (event: SyncProgressEvent) => void;
488
- /**
489
- * When true, the caller is syncing against the caller's person-entity
490
- * bucket. Pulled keys whose path starts with `companies/` are local
491
- * (non-cloud) companies that sync to the personal vault by default —
492
- * they are allowed through EXCEPT for slugs in `teamSyncedSlugs` (which
493
- * are cloud-backed orphans, see below). `companies/manifest.yaml` is
494
- * always allowed (routing source-of-truth).
495
- */
496
- personalMode?: boolean;
497
- /**
498
- * Slugs of companies the operator has an active team-bucket Membership
499
- * for. Only consulted when `personalMode === true`: keys under
500
- * `companies/{slug}/...` for any slug in this set are dropped as orphans
501
- * from a pre-promotion personal-bucket fallback (the company became
502
- * cloud-true). The push-side decommission cycle removes these from the
503
- * bucket; this filter prevents them from re-downloading into the same
504
- * disk paths the team-bucket pull manages.
505
- */
506
- teamSyncedSlugs?: ReadonlySet<string>;
507
- /**
508
- * Override for the per-slug journal file name. Defaults to `ctx.slug`.
509
- * sync-runner passes `journalSlug: "personal"` for the personal slot so
510
- * TS runner and Rust first-push share idempotency state.
511
- */
512
- journalSlug?: string;
513
- /**
514
- * Effective sync mode for this leg (US-005 wiring). Defaults to `"all"`
515
- * when absent, preserving the legacy full-bucket pull. The runner resolves
516
- * this from the membership's sync-config (`getMembershipSyncConfig`).
517
- *
518
- * SECURITY NOTE: this is a footprint/UX filter, NOT an authorization
519
- * boundary. The security boundary is the server (STS credential scope +
520
- * ACL). An owner's STS is wide (role-bypass), so this client-side scope is
521
- * what makes selective download durable for owners — but it never grants
522
- * access beyond what STS already permits.
523
- */
524
- syncMode?: SyncMode;
525
- /**
526
- * Coalesced, COMPANY-RELATIVE prefixes the current pull is scoped to when
527
- * `syncMode` is `"shared"` or `"custom"` (same namespace as `RemoteFile.key`
528
- * and the per-slug journal keys — e.g. `"knowledge/"`, `"projects/x/"`).
529
- * Ignored when `syncMode` is `"all"`. The runner derives this from the
530
- * caller's explicit grants (`shared`) or `customPaths` (`custom`) and is
531
- * responsible for normalizing into the company-relative namespace.
532
- *
533
- * A `shared` leg with an empty/undefined `prefixSet` means "nothing is
534
- * shared with me" → download nothing. The runner MUST fall back to `"all"`
535
- * (not empty `"shared"`) on any grant-resolution error, so a transient
536
- * failure can never silently prune the local tree.
537
- */
538
- prefixSet?: ScopePrefixInput[];
539
- /**
540
- * Company-relative prefixes SUBTRACTED from the effective pull scope in EVERY
541
- * mode — the push-only set (US-006). Carries `sessions/`: session transcripts
542
- * are pushed into the vault but never auto-pulled onto grantee laptops, even
543
- * under `syncMode: "all"` (which otherwise pulls the whole bucket). A remote
544
- * key downloads iff it is covered by the inclusion scope (`prefixSet`, or all
545
- * of `all` mode) AND NOT covered by any `excludePrefixes` entry; excluded
546
- * keys are classified `skip-out-of-scope`.
547
- *
548
- * CRITICAL: this set is applied ONLY to the download filter. It is
549
- * deliberately NOT fed into the scope-shrink pass (which keeps using the
550
- * inclusion `prefixSet` + pins), so a locally-authored (`direction:"up"`) or
551
- * pinned on-demand-materialized session is NEVER orphaned/pruned — it is
552
- * merely not re-pulled. The runner derives this from
553
- * `resolvePullScope().excludePrefixes`.
554
- */
555
- excludePrefixes?: ScopePrefixInput[];
556
- /**
557
- * When the effective scope shrinks relative to the last pull and the shrink
558
- * would orphan locally-modified ("dirty") files, `sync()` aborts with a
559
- * `ScopeShrinkBlockedError` by default. Set `true` to proceed anyway:
560
- * dirty files are LEFT ON DISK and only their journal entries are
561
- * tombstoned. Mirrors `hq sync narrow --force`.
562
- */
563
- forceScopeShrink?: boolean;
564
- /**
565
- * How `sync()` handles a scope shrink (US-005 / DEV-1768):
566
- *
567
- * - `"block"` (default) — a human is present (foreground `hq sync`). Dirty
568
- * out-of-scope orphans, or a clean prune over the safety cap, raise a
569
- * structured error whose advice is followable from a terminal. Clean
570
- * orphans within the cap are QUARANTINED (moved, not deleted).
571
- * - `"auto-recover"` — the background menubar runner, which can take no
572
- * interactive flag. NEVER throws on a shrink: dirty orphans are kept on
573
- * disk + un-tracked, clean orphans are quarantined, and the bulk-prune
574
- * cap is bypassed (quarantine is non-destructive). This is what clears an
575
- * already-wedged journal on the next sync, idempotently and without data
576
- * loss — the recovery seam for the all→shared seed bug.
577
- *
578
- * Both policies are non-destructive for CLEAN files (quarantine, never
579
- * silent delete) — the deliberate `hq sync narrow --apply` ritual is the only
580
- * path that hard-deletes, and it confirms first.
581
- */
582
- scopeShrinkPolicy?: "block" | "auto-recover";
583
- /**
584
- * The caller's own Cognito `sub`, used by the scope-shrink authorship guard
585
- * so a scope shrink never prunes content the caller authored. Injected by the
586
- * entry point — the runner sources it from its decoded idToken claims (the
587
- * same sub stamped onto uploads as `created-by-sub`). The engine never reads
588
- * it from disk, so it stays pure/hermetic; undefined degrades safely.
589
- */
590
- callerSub?: string;
591
- /**
592
- * Optional decoded Cognito claims used only for action telemetry attribution
593
- * (agentUid when `custom:entityType=agent`). Sync behavior never branches on
594
- * this field.
595
- */
596
- telemetryClaims?: TelemetryClaims | null;
597
- /**
598
- * Skip the post-sync `reindex()` refresh (skill wrappers + personal overlay
599
- * mirrors + workers registry). By default, when a sync changes on-disk
600
- * sources (downloads, tombstones, or scope-orphan removals), `sync()`
601
- * re-runs reindex so the generated `.claude/skills/<ns>:<skill>` wrappers
602
- * stay in sync. An orchestrator syncing many companies in one pass can set
603
- * this and run `reindex()` once itself instead of per-company.
604
- */
605
- skipReindex?: boolean;
606
- /**
607
- * Internal runner seam: true only when the caller already holds the
608
- * per-root operation lock for this sync pass.
609
- */
610
- operationLockAlreadyHeld?: boolean;
611
- }
612
-
613
- export interface SyncResult {
614
- filesDownloaded: number;
615
- bytesDownloaded: number;
616
- filesSkipped: number;
617
- conflicts: number;
618
- /**
619
- * Paths (remote keys) that were detected as conflicts during this run.
620
- * Always populated when `conflicts > 0` so callers can surface them in UI
621
- * or logs without re-streaming the per-file events.
622
- */
623
- conflictPaths: string[];
624
- aborted: boolean;
625
- /**
626
- * Files classified as "new" during pull — i.e. the remote file had no
627
- * local counterpart at classification time. Additive field; empty array
628
- * when no new files were detected or on push-only syncs.
629
- */
630
- newFiles: Array<{ path: string; bytes: number }>;
631
- /** Convenience count: `newFiles.length`. */
632
- newFilesCount: number;
633
- /**
634
- * Count of remote keys refused at planning time because they matched
635
- * `EPHEMERAL_PATH_PATTERN` (conflict-mirror files that must never round-
636
- * trip through the bucket). Mirrors `ShareResult.filesExcludedByPolicy`
637
- * so push and pull report the same shape. Pre-fix this count was always
638
- * 0 on the pull side and legacy `.conflict-*` litter rode every sync —
639
- * see Bug #2 in workspace/reports/hq-cloud-5.33.0-deep-test.md.
640
- */
641
- filesExcludedByPolicy: number;
642
- /**
643
- * Count of journal-known keys applied as local deletes during this pull
644
- * because the remote LIST no longer contains them — the cross-machine
645
- * delete-propagation signal that Bug #9 closes. The peer's push leg
646
- * removed the object from S3 (`hq sync` push side verified-to-work in
647
- * the deep-test addendum), but pre-fix the pull side never enumerated
648
- * "what's missing-from-remote-that-was-there-before", so the file
649
- * lingered locally forever. Always 0 when no journal-known keys have
650
- * disappeared from the remote.
651
- */
652
- filesTombstoned: number;
653
- /**
654
- * Count of remote keys NOT downloaded this run because they fall outside
655
- * the effective `syncMode` scope (US-005). Always 0 in `all` mode. Distinct
656
- * from `filesSkipped` (which measures "unchanged on this run") so consumers
657
- * can render a "N outside your sync scope" line. The matching local cleanup
658
- * of previously-downloaded-now-out-of-scope files is reported via
659
- * `scopeOrphansRemoved`.
660
- */
661
- filesOutOfScope: number;
662
- /**
663
- * Clean local orphans deleted this run because a scope shrink moved them
664
- * outside the effective scope (US-005). 0 when scope did not shrink.
665
- */
666
- scopeOrphansRemoved: number;
667
- /**
668
- * Dirty (locally-modified) orphans that a scope shrink would have pruned.
669
- * When `forceScopeShrink` is false these are surfaced via a thrown
670
- * `ScopeShrinkBlockedError` and the leg never reaches this result; when
671
- * true they are left on disk and tombstoned, and counted here.
672
- */
673
- scopeOrphansBlocked: number;
674
- /**
675
- * Remote keys whose local on-disk counterpart changed during this pull.
676
- * Includes downloads, tombstone deletes, and clean scope-orphan removals.
677
- */
678
- changedPaths?: string[];
679
- }
680
-
681
- type SyncEventEmitter = (event: SyncProgressEvent) => void;
682
-
683
- type PullDownloadItem = Extract<PullPlanItem, { action: "download" }>;
684
-
685
- interface PullRunContext {
686
- options: SyncOptions;
687
- companyRef: string;
688
- vaultConfig: VaultServiceConfig;
689
- hqRoot: string;
690
- emit: SyncEventEmitter;
691
- ctx: EntityContext;
692
- companyRoot: string;
693
- shouldSync: (filePath: string, isDir?: boolean) => boolean;
694
- journalSlug: string;
695
- startedAt: string;
696
- journal: SyncJournal;
697
- remoteFiles: RemoteFile[];
698
- syncMode: SyncMode;
699
- currentPrefixSet: string[];
700
- /**
701
- * Coalesced push-only exclusion prefixes (US-006: `sessions/`), subtracted
702
- * from the download filter in EVERY mode. Kept SEPARATE from
703
- * `currentPrefixSet` on purpose: the scope-shrink pass reads only
704
- * `currentPrefixSet` (+ pins), so excluded-but-authored/pinned session files
705
- * are never orphaned.
706
- */
707
- currentExcludeSet: string[];
708
- fileTombstones: ReadonlyMap<string, CompanyTombstone>;
709
- /** Successful downloads since the last durable journal checkpoint. */
710
- downloadsSinceJournalCheckpoint: number;
711
- }
712
-
713
- interface PullCounters {
714
- filesDownloaded: number;
715
- bytesDownloaded: number;
716
- filesSkipped: number;
717
- conflicts: number;
718
- filesTombstoned: number;
719
- filesOutOfScope: number;
720
- conflictPaths: string[];
721
- changedPaths: string[];
722
- }
723
-
724
- interface ScopeShrinkPlanState {
725
- lastRecord: ReturnType<typeof lastPullRecord>;
726
- shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
727
- autoRecover: boolean;
728
- adviceContext: ScopeShrinkAdviceContext;
729
- effectiveForce: boolean;
730
- }
731
-
732
- interface ScopeShrinkRun {
733
- shrinkPlan: ReturnType<typeof buildScopeShrinkPlan>;
734
- shrinkResult: ReturnType<typeof applyScopeShrink>;
735
- scopeOrphansRemoved: number;
736
- changedPaths: string[];
737
- }
738
-
739
- /**
740
- * Resolve the auto-prune safety cap (US-005 bulk-delete guard). An automatic
741
- * scope shrink that would delete more than this many CLEAN local files in one
742
- * pull is refused with `ScopeShrinkLargePruneError`. Default 100; `0` (or a
743
- * non-positive / unparseable value) disables the cap (unlimited). Override via
744
- * `HQ_SYNC_MAX_AUTO_PRUNE`.
745
- */
746
- export function resolveAutoPruneCap(): number {
747
- const raw = process.env.HQ_SYNC_MAX_AUTO_PRUNE;
748
- if (raw === undefined || raw === "") return 100;
749
- const parsed = Number.parseInt(raw, 10);
750
- // NaN or negative → treat as "unlimited" (0) rather than silently capping.
751
- return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
752
- }
753
-
754
- /** Max time to wait on the best-effort new-files notification POST. */
755
- const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
756
- /** Bound crash recovery to at most this many completed pull downloads. */
757
- const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 8;
758
-
759
- /**
760
- * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
761
- * an oversized batch wholesale, so the client MUST split a large report into
762
- * chunks at or under this size — otherwise a first sync with more than this many
763
- * new files reports none of them, and the same oversized batch re-triggers every
764
- * sync cycle (wasted work + dropped notifications). Keep in lockstep with the
765
- * server-side limit.
766
- */
767
- const NOTIFY_FILE_ADDED_MAX_BATCH = 1000;
768
-
769
- /**
770
- * Best-effort report of the files that were new to this drive during the sync,
771
- * so the HQ Sync app can show a persistent cross-session "new files" history.
772
- *
773
- * POSTs to `${apiUrl}/v1/notify/file-added`, which writes per-recipient
774
- * FILE_EVENT rows for the calling user (the one the files are new for). Fully
775
- * non-fatal: any error, non-2xx, or timeout is swallowed — the durable signal
776
- * is the synced file itself; this is only a notification mirror. Bounded by a
777
- * 5s timeout PER request so a hung endpoint can't stall sync completion. No-op
778
- * when there are no new files.
779
- *
780
- * Large reports are split into chunks of at most NOTIFY_FILE_ADDED_MAX_BATCH
781
- * files (the server's per-report cap). Each chunk is POSTed independently and
782
- * best-effort, so one failing/oversized batch can never block the others or the
783
- * sync. Exported only so the chunking can be unit-tested directly.
784
- */
785
- export async function reportNewFilesToNotify(
786
- vaultConfig: VaultServiceConfig,
787
- companyUid: string,
788
- companySlug: string,
789
- files: Array<{ path: string; bytes: number; addedBy: string | null }>,
790
- telemetryClaims?: TelemetryClaims | null,
791
- ): Promise<void> {
792
- if (files.length === 0) return;
793
-
794
- let token: string;
795
- try {
796
- token =
797
- typeof vaultConfig.authToken === "function"
798
- ? await vaultConfig.authToken()
799
- : vaultConfig.authToken;
800
- } catch (err) {
801
- logNotifyFailure(err);
802
- return;
803
- }
804
- const base = vaultConfig.apiUrl.replace(/\/+$/, "");
805
- const telemetryClient = new VaultClient(vaultConfig);
806
- const batchCount = Math.ceil(files.length / NOTIFY_FILE_ADDED_MAX_BATCH);
807
-
808
- for (let i = 0; i < files.length; i += NOTIFY_FILE_ADDED_MAX_BATCH) {
809
- const batch = files.slice(i, i + NOTIFY_FILE_ADDED_MAX_BATCH);
810
- const batchIndex = Math.floor(i / NOTIFY_FILE_ADDED_MAX_BATCH) + 1;
811
- const controller = new AbortController();
812
- const timer = setTimeout(
813
- () => controller.abort(),
814
- NOTIFY_FILE_ADDED_TIMEOUT_MS,
815
- );
816
- try {
817
- const response = await fetch(`${base}/v1/notify/file-added`, {
818
- method: "POST",
819
- headers: {
820
- Authorization: `Bearer ${token}`,
821
- "Content-Type": "application/json",
822
- },
823
- body: JSON.stringify({
824
- companyUid,
825
- companySlug,
826
- files: batch.map((f) => ({
827
- path: f.path,
828
- bytes: f.bytes,
829
- ...(f.addedBy ? { addedBy: f.addedBy } : {}),
830
- })),
831
- }),
832
- signal: controller.signal,
833
- });
834
- if (!response.ok) {
835
- void emitCloudTelemetry(telemetryClient, {
836
- eventName: "new_files_notification_reported",
837
- source: "hq-sync",
838
- companyUid,
839
- properties: {
840
- status: "failure",
841
- fileCount: batch.length,
842
- batchIndex,
843
- batchCount,
844
- statusCode: response.status,
845
- },
846
- }, { claims: telemetryClaims });
847
- }
848
- } catch (err) {
849
- // Best-effort per chunk: never let notification reporting affect the sync
850
- // result, and a failed chunk must not abort the remaining chunks.
851
- logNotifyFailure(err);
852
- void emitCloudTelemetry(telemetryClient, {
853
- eventName: "new_files_notification_reported",
854
- source: "hq-sync",
855
- companyUid,
856
- properties: {
857
- status: "failure",
858
- fileCount: batch.length,
859
- batchIndex,
860
- batchCount,
861
- errorClass: safeErrorName(err),
862
- },
863
- }, { claims: telemetryClaims });
864
- } finally {
865
- clearTimeout(timer);
866
- }
867
- }
868
- }
869
-
870
- function safeErrorName(err: unknown): string {
871
- return err instanceof Error && err.name ? err.name : "error";
872
- }
873
-
874
- /** Log a non-fatal notify failure without ever throwing out of the logger. */
875
- function logNotifyFailure(err: unknown): void {
876
- try {
877
- console.error(
878
- `[hq-sync] new-files notify report failed (non-fatal): ${
879
- err instanceof Error ? err.message : String(err)
880
- }`,
881
- );
882
- } catch {
883
- // swallow — logging must never break sync
884
- }
885
- }
886
-
887
- /** Log a non-fatal post-sync maintenance failure without changing sync state. */
888
- function logPostSyncMaintenanceFailure(
889
- task: string,
890
- hqRoot: string,
891
- err: unknown,
892
- ): void {
893
- try {
894
- console.error(
895
- `[hq-sync] ${task} failed (non-fatal; hqRoot=${hqRoot}): ${
896
- err instanceof Error ? err.message : String(err)
897
- }`,
898
- );
899
- } catch {
900
- // swallow — logging must never break sync
901
- }
902
- }
903
-
904
- /**
905
- * Sync (pull) all allowed files from the entity vault.
906
- */
907
- export async function sync(options: SyncOptions): Promise<SyncResult> {
908
- if (options.operationLockAlreadyHeld) {
909
- return syncWithOperationLockHeld(options);
910
- }
911
- return withOperationLock(options.hqRoot, "sync", () =>
912
- syncWithOperationLockHeld(options),
913
- );
914
- }
915
-
916
- async function syncWithOperationLockHeld(
917
- options: SyncOptions,
918
- ): Promise<SyncResult> {
919
- const run = await buildPullContext(options);
920
- const plan = planPull(run);
921
-
922
- emitPullPlan(run.emit, plan);
923
- reportInvalidRemoteKeys(run, plan);
924
-
925
- const scopePlan = planScopeShrink(run);
926
- const scopeRun = executeScopeShrink(run, scopePlan);
927
- const counters = createPullCounters();
928
-
929
- const transferConcurrency = resolveTransferConcurrency();
930
- const conflictRun = await executeConflictExecutor(
931
- run,
932
- plan,
933
- scopeRun,
934
- counters,
935
- );
936
- if (conflictRun.abortResult) {
937
- return conflictRun.abortResult;
938
- }
939
-
940
- await executeDownloadExecutor(
941
- run,
942
- conflictRun.downloadItems,
943
- transferConcurrency,
944
- counters,
945
- );
946
-
947
- await emitAndReportNewFiles(run, plan);
948
- await verifyPlannedJournalTombstones(run, plan);
949
- executeJournalTombstoneDeletes(run, plan, counters);
950
-
951
- emitScopeMaterializationGap(run, plan);
952
-
953
- return finalizePullRun(run, plan, scopeRun, counters);
954
- }
955
-
956
- async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
957
- const { company, vaultConfig, hqRoot } = options;
958
- const baseEmit = options.onEvent ?? defaultConsoleLogger;
959
-
960
- const companyRef = company ?? resolveActiveCompany(hqRoot);
961
- if (!companyRef) {
962
- throw new Error(
963
- "No company specified and no active company found. " +
964
- "Use --company <slug> or set up .hq/config.json.",
965
- );
966
- }
967
-
968
- const ctx = await resolveEntityContext(companyRef, vaultConfig);
969
- const companyRoot = options.personalMode === true
970
- ? hqRoot
971
- : path.join(hqRoot, "companies", ctx.slug);
972
- const shouldSync = createIgnoreFilter(hqRoot);
973
- const journalSlug = options.journalSlug ?? ctx.slug;
974
- const startedAt = new Date().toISOString();
975
-
976
- // Mirror every progress event into the shared cross-process snapshot so the
977
- // menubar shows live progress for THIS sync regardless of who launched it
978
- // (auto-sync / Sync Now / CLI). Record the friendly company slug (e.g.
979
- // "indigo") — or null for the personal vault so the menubar shows "Personal"
980
- // — never the raw prs_/cmp_ UID. Best-effort; never affects the sync.
981
- const recordProgress = createSyncProgressRecorder({
982
- company: options.personalMode === true ? null : ctx.slug,
983
- phase: "pull",
984
- });
985
- const emit: SyncEventEmitter = (event) => {
986
- baseEmit(event);
987
- recordProgress(event);
988
- };
989
-
990
- if (journalSlug === PERSONAL_VAULT_JOURNAL_SLUG) migratePersonalVaultJournal();
991
- const journal = migrateToV2(readJournal(journalSlug));
992
- gcTombstones(journal, Date.now());
993
-
994
- const syncMode: SyncMode = options.syncMode ?? "all";
995
- const currentPrefixSet: string[] =
996
- syncMode === "all" ? [""] : coalescePrefixes(options.prefixSet ?? []);
997
- // Push-only exclusions (US-006). Coalesced independently of `currentPrefixSet`
998
- // and applied to EVERY mode's download filter — including `all` — but kept out
999
- // of the scope-shrink inputs (see `planScopeShrink`). Empty when the caller
1000
- // passes nothing (legacy behavior: no exclusions).
1001
- const currentExcludeSet: string[] = coalescePrefixes(
1002
- options.excludePrefixes ?? [],
1003
- );
1004
-
1005
- const remoteFiles = await listRemoteFiles(ctx);
1006
- const fileTombstones =
1007
- options.personalMode === true
1008
- ? new Map<string, CompanyTombstone>()
1009
- : await fetchCompanyTombstones(vaultConfig, ctx.uid);
1010
-
1011
- return {
1012
- options,
1013
- companyRef,
1014
- vaultConfig,
1015
- hqRoot,
1016
- emit,
1017
- ctx,
1018
- companyRoot,
1019
- shouldSync,
1020
- journalSlug,
1021
- startedAt,
1022
- journal,
1023
- remoteFiles,
1024
- syncMode,
1025
- currentPrefixSet,
1026
- currentExcludeSet,
1027
- fileTombstones,
1028
- downloadsSinceJournalCheckpoint: 0,
1029
- };
1030
- }
1031
-
1032
- function planPull(run: PullRunContext): PullPlan {
1033
- return computePullPlan(
1034
- run.remoteFiles,
1035
- run.journal,
1036
- run.companyRoot,
1037
- run.shouldSync,
1038
- run.options.personalMode === true,
1039
- run.options.teamSyncedSlugs ?? null,
1040
- run.currentPrefixSet,
1041
- run.fileTombstones,
1042
- run.currentExcludeSet,
1043
- );
1044
- }
1045
-
1046
- function emitPullPlan(emit: SyncEventEmitter, plan: PullPlan): void {
1047
- emit({
1048
- type: "plan",
1049
- filesToDownload: plan.filesToDownload,
1050
- bytesToDownload: plan.bytesToDownload,
1051
- filesToUpload: 0,
1052
- bytesToUpload: 0,
1053
- filesToSkip: plan.filesToSkip,
1054
- filesToConflict: plan.filesToConflict,
1055
- filesToDelete: plan.filesToTombstoneDelete,
1056
- });
1057
- }
1058
-
1059
- /**
1060
- * Surface permanently-unpresignable remote keys the planner skipped (see the
1061
- * `invalidRemoteKeys` doc on PullPlan). One warning event per key on the
1062
- * existing sync event channel (deliberately NOT `type: "error"` — see the
1063
- * `skip-invalid-scoped-key` event doc), plus one best-effort cloud telemetry
1064
- * event on the channel sync already uses (emitCloudTelemetry). Per policy
1065
- * hq-alert-baseline-calibration the documented healthy baseline is ZERO
1066
- * invalid-key occurrences fleet-wide, so consumers can alert on any non-zero
1067
- * count with no threshold tuning.
1068
- */
1069
- function reportInvalidRemoteKeys(run: PullRunContext, plan: PullPlan): void {
1070
- if (plan.invalidRemoteKeys.length === 0) return;
1071
- const countsByErrorCode = new Map<string, number>();
1072
- for (const { path, errorCode } of plan.invalidRemoteKeys) {
1073
- run.emit({ type: "skip-invalid-scoped-key", path, errorCode });
1074
- countsByErrorCode.set(errorCode, (countsByErrorCode.get(errorCode) ?? 0) + 1);
1075
- }
1076
- for (const [errorCode, count] of countsByErrorCode) {
1077
- void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
1078
- eventName: "sync_invalid_scoped_key_skipped",
1079
- source: "hq-sync",
1080
- companyUid: run.ctx.uid,
1081
- properties: { count, errorCode, leg: "pull" },
1082
- }, { claims: run.options.telemetryClaims });
1083
- }
1084
- }
1085
-
1086
- /**
1087
- * A legacy object without hq-mode can otherwise downgrade a hook to the
1088
- * receiver umask with no durable signal. downloadFile already prints a local
1089
- * warning for every caller; the sync path also records a path-free telemetry
1090
- * event so operators can find vaults that still need the one-time backfill.
1091
- */
1092
- function reportModeWarnings(
1093
- run: PullRunContext,
1094
- warnings: DownloadModeWarning[] | undefined,
1095
- ): void {
1096
- for (const warning of warnings ?? []) {
1097
- void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
1098
- eventName: "sync_mode_guardrail_warning",
1099
- source: "hq-sync",
1100
- companyUid: run.ctx.uid,
1101
- properties: {
1102
- leg: "pull",
1103
- reason: warning.reason,
1104
- fallback: warning.fallback ?? null,
1105
- },
1106
- }, { claims: run.options.telemetryClaims });
1107
- }
1108
- }
1109
-
1110
- function createPullCounters(): PullCounters {
1111
- return {
1112
- filesDownloaded: 0,
1113
- bytesDownloaded: 0,
1114
- filesSkipped: 0,
1115
- conflicts: 0,
1116
- filesTombstoned: 0,
1117
- filesOutOfScope: 0,
1118
- conflictPaths: [],
1119
- changedPaths: [],
1120
- };
1121
- }
1122
-
1123
- function planScopeShrink(run: PullRunContext): ScopeShrinkPlanState {
1124
- const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
1125
- const lastPrefixSet =
1126
- lastRecord && lastRecord.prefixSet.length > 0
1127
- ? lastRecord.prefixSet
1128
- : [""];
1129
- const shrinkPlan = buildScopeShrinkPlan({
1130
- journal: run.journal,
1131
- hqRoot: run.companyRoot,
1132
- lastPrefixSet,
1133
- // INCLUSION scope only (+ pins, already folded into currentPrefixSet by the
1134
- // resolver). The push-only exclude set (US-006: `sessions/`) is DELIBERATELY
1135
- // NOT subtracted here: a session file that was authored locally
1136
- // (`direction:"up"`) or pinned + materialized on demand must never be
1137
- // orphaned just because sessions are excluded from the download filter.
1138
- // Feeding `currentExcludeSet` in here would prune exactly those files.
1139
- currentPrefixSet: run.currentPrefixSet,
1140
- callerSub: run.options.callerSub,
1141
- protectUnknownAuthors: true,
1142
- });
1143
- const scopeShrinkPolicy = run.options.scopeShrinkPolicy ?? "block";
1144
- const autoRecover = scopeShrinkPolicy === "auto-recover";
1145
- const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
1146
- const effectiveForce = run.options.forceScopeShrink === true || autoRecover;
1147
-
1148
- return {
1149
- lastRecord,
1150
- shrinkPlan,
1151
- autoRecover,
1152
- adviceContext,
1153
- effectiveForce,
1154
- };
1155
- }
1156
-
1157
- function executeScopeShrink(
1158
- run: PullRunContext,
1159
- scopePlan: ScopeShrinkPlanState,
1160
- ): ScopeShrinkRun {
1161
- const { lastRecord, shrinkPlan, adviceContext, effectiveForce } = scopePlan;
1162
-
1163
- if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
1164
- throw new ScopeShrinkBlockedError(
1165
- run.ctx.uid,
1166
- lastRecord?.syncMode ?? "unknown",
1167
- run.syncMode,
1168
- shrinkPlan.dirty,
1169
- shrinkPlan.clean,
1170
- adviceContext,
1171
- );
1172
- }
1173
-
1174
- const autoPruneCap = resolveAutoPruneCap();
1175
- if (
1176
- !effectiveForce &&
1177
- autoPruneCap > 0 &&
1178
- shrinkPlan.clean.length > autoPruneCap
1179
- ) {
1180
- throw new ScopeShrinkLargePruneError(
1181
- run.ctx.uid,
1182
- run.syncMode,
1183
- shrinkPlan.clean.length,
1184
- autoPruneCap,
1185
- adviceContext,
1186
- );
1187
- }
1188
-
1189
- const scopeQuarantineRoot = path.join(
1190
- run.hqRoot,
1191
- ".hq",
1192
- "scope-quarantine",
1193
- run.journalSlug,
1194
- );
1195
- const shrinkResult = applyScopeShrink({
1196
- journal: run.journal,
1197
- plan: shrinkPlan,
1198
- hqRoot: run.companyRoot,
1199
- forceScopeShrink: effectiveForce,
1200
- reason: "scope_shrink",
1201
- cleanDisposition: "quarantine",
1202
- quarantineRoot: scopeQuarantineRoot,
1203
- });
1204
-
1205
- for (const relPath of shrinkResult.quarantinedPaths) {
1206
- run.emit({
1207
- type: "progress",
1208
- path: relPath,
1209
- bytes: 0,
1210
- deleted: true,
1211
- message: `scope-narrowed: moved out-of-scope copy to ${scopeQuarantineRoot}`,
1212
- });
1213
- }
1214
- for (const relPath of shrinkResult.removedPaths) {
1215
- run.emit({
1216
- type: "progress",
1217
- path: relPath,
1218
- bytes: 0,
1219
- deleted: true,
1220
- message: "scope-narrowed (removed local copy outside sync scope)",
1221
- });
1222
- }
1223
- for (const relPath of shrinkResult.dirtyKeptPaths) {
1224
- run.emit({
1225
- type: "progress",
1226
- path: relPath,
1227
- bytes: 0,
1228
- message:
1229
- "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
1230
- });
1231
- }
1232
-
1233
- return {
1234
- shrinkPlan,
1235
- shrinkResult,
1236
- scopeOrphansRemoved:
1237
- shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined,
1238
- changedPaths: [
1239
- ...shrinkResult.quarantinedPaths,
1240
- ...shrinkResult.removedPaths,
1241
- ],
1242
- };
1243
- }
1244
-
1245
- async function refreshRunContextIfExpiring(
1246
- run: PullRunContext,
1247
- ): Promise<void> {
1248
- if (isExpiringSoon(run.ctx.expiresAt)) {
1249
- run.ctx = await refreshEntityContext(run.companyRef, run.vaultConfig);
1250
- }
1251
- }
1252
-
1253
- async function executeConflictExecutor(
1254
- run: PullRunContext,
1255
- plan: PullPlan,
1256
- scopeRun: ScopeShrinkRun,
1257
- counters: PullCounters,
1258
- ): Promise<{ downloadItems: PullDownloadItem[]; abortResult: SyncResult | null }> {
1259
- const downloadItems: PullDownloadItem[] = [];
1260
-
1261
- for (const item of plan.items) {
1262
- if (
1263
- item.action === "skip-ignored" ||
1264
- item.action === "skip-personal-mode" ||
1265
- item.action === "skip-unchanged" ||
1266
- item.action === "skip-local-only" ||
1267
- item.action === "skip-stale-overlay-marker" ||
1268
- item.action === "skip-overlay-marker-with-children"
1269
- ) {
1270
- counters.filesSkipped++;
1271
- continue;
1272
- }
1273
- if (item.action === "skip-unreadable-link") {
1274
- // An unreadable local link must stay non-fatal: the runner treats a
1275
- // generic error event as a failed company leg. Surface the exact remote
1276
- // key through the recoverable not-shipped channel instead.
1277
- counters.filesSkipped++;
1278
- run.emit({
1279
- type: "not-shipped",
1280
- reason: "unreadable-link",
1281
- count: 1,
1282
- samplePaths: [item.remoteFile.key],
1283
- });
1284
- continue;
1285
- }
1286
- if (item.action === "skip-excluded-policy") {
1287
- continue;
1288
- }
1289
- if (item.action === "skip-out-of-scope") {
1290
- counters.filesOutOfScope++;
1291
- continue;
1292
- }
1293
- if (item.action === "skip-junk-key-spelling") {
1294
- // Deliberate, recoverable skip — counted as skipped and surfaced via a
1295
- // dedicated warning event, never `type: "error"` (see the event doc /
1296
- // policy hq-sync-deliberate-skip-not-fatal-error-exit2).
1297
- counters.filesSkipped++;
1298
- run.emit({
1299
- type: "skip-junk-key-spelling",
1300
- path: item.remoteFile.key,
1301
- journaledKey: item.journaledKey,
1302
- });
1303
- continue;
1304
- }
1305
- if (item.action === "tombstone-delete") {
1306
- executeFileTombstoneDelete(run, item, counters);
1307
- continue;
1308
- }
1309
- if (item.action === "download") {
1310
- downloadItems.push(item);
1311
- continue;
1312
- }
1313
-
1314
- const abortResult = await executeConflictItem(
1315
- run,
1316
- plan,
1317
- scopeRun,
1318
- counters,
1319
- downloadItems,
1320
- item,
1321
- );
1322
- if (abortResult) {
1323
- return { downloadItems, abortResult };
1324
- }
1325
- }
1326
-
1327
- return { downloadItems, abortResult: null };
1328
- }
1329
-
1330
- function executeFileTombstoneDelete(
1331
- run: PullRunContext,
1332
- item: Extract<PullPlanItem, { action: "tombstone-delete" }>,
1333
- counters: PullCounters,
1334
- ): void {
1335
- const tombstoneKey = item.remoteFile.key;
1336
- const tombstonePath = item.localPath;
1337
- if (!isDownloadWritePathStillContained(run.companyRoot, tombstoneKey, tombstonePath)) {
1338
- run.emit({
1339
- type: "error",
1340
- path: tombstoneKey,
1341
- message: "tombstone-suppress unlink skipped: local parent escaped the sync root",
1342
- });
1343
- return;
1344
- }
1345
- try {
1346
- const lstat = fs.lstatSync(tombstonePath);
1347
- if (!matchesLocalSnapshot(tombstonePath, item.localSnapshot)) {
1348
- reportLocalSnapshotConflict(run, counters, tombstoneKey);
1349
- return;
1350
- }
1351
- if (lstat.isSymbolicLink() || lstat.isFile()) {
1352
- fs.unlinkSync(tombstonePath);
1353
- }
1354
- } catch (err: unknown) {
1355
- const code =
1356
- err && typeof err === "object" && "code" in err
1357
- ? (err as { code?: string }).code
1358
- : undefined;
1359
- if (code !== "ENOENT") {
1360
- run.emit({
1361
- type: "error",
1362
- path: tombstoneKey,
1363
- message: `tombstone-suppress unlink failed: ${
1364
- err instanceof Error ? err.message : String(err)
1365
- }`,
1366
- });
1367
- return;
1368
- }
1369
- }
1370
- removeEntry(run.journal, tombstoneKey);
1371
- counters.filesTombstoned++;
1372
- counters.changedPaths.push(tombstoneKey);
1373
- run.emit({ type: "progress", path: tombstoneKey, bytes: 0 });
1374
- }
1375
-
1376
- async function executeConflictItem(
1377
- run: PullRunContext,
1378
- plan: PullPlan,
1379
- scopeRun: ScopeShrinkRun,
1380
- counters: PullCounters,
1381
- downloadItems: PullDownloadItem[],
1382
- item: Extract<PullPlanItem, { action: "conflict" }>,
1383
- ): Promise<SyncResult | null> {
1384
- const { remoteFile, localPath } = item;
1385
-
1386
- // Cloud-authoritative paths (server-regenerated: company-brief.md, board.json,
1387
- // ontology/ signals/ sources/) are PULL-WINS: the cloud copy is the source of
1388
- // truth, so a divergence here is not a real conflict. Resolve it as a normal
1389
- // overwrite-from-cloud — queue a plain download (which stamps the journal with
1390
- // the REMOTE hash/etag) and skip the conflict path entirely. This removes both
1391
- // the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
1392
- // divergent local content) that the keep path produced for these files.
1393
- if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
1394
- downloadItems.push({
1395
- action: "download",
1396
- remoteFile,
1397
- localPath,
1398
- isNew: false,
1399
- localSnapshot: item.localSnapshot,
1400
- });
1401
- run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1402
- return null;
1403
- }
1404
-
1405
- await refreshRunContextIfExpiring(run);
1406
-
1407
- const detectedAt = new Date().toISOString();
1408
- const machineId = readShortMachineId(run.hqRoot);
1409
- const originalRelative = vaultKeyForLocalPath(run.hqRoot, localPath);
1410
- const conflictRelative = buildConflictPath(
1411
- originalRelative,
1412
- detectedAt,
1413
- machineId,
1414
- );
1415
- const conflictAbs = localPathForVaultKey(run.hqRoot, conflictRelative);
1416
- const conflictKey = vaultKeyForLocalPath(run.companyRoot, conflictAbs);
1417
-
1418
- if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
1419
- counters.filesSkipped++;
1420
- run.emit({
1421
- type: "error",
1422
- path: remoteFile.key,
1423
- message: "conflict mirror skipped: local parent escaped the sync root",
1424
- });
1425
- return null;
1426
- }
1427
-
1428
- let remoteFetched = false;
1429
- let converged = false;
1430
- try {
1431
- const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
1432
- remoteFetched = true;
1433
- if (fs.lstatSync(conflictAbs).isSymbolicLink()) {
1434
- const target = readlinkOrNull(conflictAbs);
1435
- if (target === null) {
1436
- counters.filesSkipped++;
1437
- run.emit({
1438
- type: "not-shipped",
1439
- reason: "unreadable-link",
1440
- count: 1,
1441
- samplePaths: [remoteFile.key],
1442
- });
1443
- return null;
1444
- } else {
1445
- converged = hashSymlinkTarget(target) === item.localHash;
1446
- }
1447
- } else {
1448
- converged = (downloaded.contentHash ?? hashFile(conflictAbs)) === item.localHash;
1449
- }
1450
- } catch (probeErr) {
1451
- if (probeErr instanceof VaultAuthError) throw probeErr;
1452
- run.emit({
1453
- type: "error",
1454
- path: remoteFile.key,
1455
- message: `conflict convergence probe failed: ${
1456
- probeErr instanceof Error ? probeErr.message : String(probeErr)
1457
- }`,
1458
- });
1459
- }
1460
-
1461
- if (converged) {
1462
- if (remoteFetched) {
1463
- try {
1464
- fs.rmSync(conflictAbs, { force: true });
1465
- } catch {
1466
- /* best-effort cleanup; a stray identical mirror is harmless */
1467
- }
1468
- }
1469
- updateEntry(
1470
- run.journal,
1471
- remoteFile.key,
1472
- item.localHash,
1473
- item.localSize,
1474
- "down",
1475
- localPath,
1476
- {
1477
- remoteEtag: remoteFile.etag,
1478
- mtimeMs: item.localMtime.getTime(),
1479
- kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
1480
- },
1481
- );
1482
- run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
1483
- counters.filesSkipped++;
1484
- return null;
1485
- }
1486
-
1487
- counters.conflicts++;
1488
- counters.conflictPaths.push(remoteFile.key);
1489
-
1490
- const resolution = await resolveConflict(
1491
- {
1492
- path: remoteFile.key,
1493
- localHash: item.localHash,
1494
- remoteModified: remoteFile.lastModified,
1495
- localModified: item.localMtime,
1496
- direction: "pull",
1497
- },
1498
- run.options.onConflict,
1499
- );
1500
-
1501
- run.emit({
1502
- type: "conflict",
1503
- path: remoteFile.key,
1504
- direction: "pull",
1505
- resolution,
1506
- });
1507
-
1508
- if (resolution !== "abort" && resolution !== "overwrite") {
1509
- if (remoteFetched) {
1510
- try {
1511
- appendConflictEntry(run.hqRoot, {
1512
- id: buildConflictId(originalRelative, detectedAt),
1513
- originalPath: originalRelative,
1514
- conflictPath: conflictRelative,
1515
- detectedAt,
1516
- side: "pull",
1517
- machineId,
1518
- localHash: item.localHash,
1519
- remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
1520
- });
1521
- } catch (mirrorErr) {
1522
- run.emit({
1523
- type: "error",
1524
- path: remoteFile.key,
1525
- message: `conflict mirror index write failed: ${
1526
- mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
1527
- }`,
1528
- });
1529
- }
1530
- }
1531
- } else if (remoteFetched) {
1532
- try {
1533
- fs.rmSync(conflictAbs, { force: true });
1534
- } catch {
1535
- /* best-effort; a leftover mirror is cosmetic, not corrupting */
1536
- }
1537
- }
1538
-
1539
- if (resolution === "abort") {
1540
- run.emit({ type: "new-files", files: [] });
1541
- writeJournal(run.journalSlug, run.journal);
1542
- return {
1543
- filesDownloaded: counters.filesDownloaded,
1544
- bytesDownloaded: counters.bytesDownloaded,
1545
- filesSkipped: counters.filesSkipped,
1546
- conflicts: counters.conflicts,
1547
- conflictPaths: counters.conflictPaths,
1548
- aborted: true,
1549
- newFiles: plan.newFiles,
1550
- newFilesCount: plan.newFilesCount,
1551
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
1552
- filesTombstoned: 0,
1553
- filesOutOfScope: counters.filesOutOfScope,
1554
- scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1555
- scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1556
- changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
1557
- };
1558
- }
1559
-
1560
- if (resolution === "keep" || resolution === "skip") {
1561
- counters.filesSkipped++;
1562
- updateEntry(
1563
- run.journal,
1564
- remoteFile.key,
1565
- item.localHash,
1566
- item.localSize,
1567
- "down",
1568
- localPath,
1569
- {
1570
- remoteEtag: remoteFile.etag,
1571
- mtimeMs: item.localMtime.getTime(),
1572
- kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
1573
- },
1574
- );
1575
- // Journal-honesty: we recorded the remote etag so this conflict can't
1576
- // re-fire (#137), but the KEPT local copy diverges from that remote — it
1577
- // never matched it. Flag the entry so the currency-gated delete planner
1578
- // refuses to propagate a delete for it (its currency would falsely match on
1579
- // HEAD, and the delete would destroy the divergent remote version). Any
1580
- // genuine future download clears the flag by replacing the entry.
1581
- const keptEntry = getEntry(run.journal, remoteFile.key);
1582
- if (keptEntry) keptEntry.localDiverges = true;
1583
- return null;
1584
- }
1585
-
1586
- downloadItems.push({
1587
- action: "download",
1588
- remoteFile,
1589
- localPath,
1590
- isNew: false,
1591
- localSnapshot: item.localSnapshot,
1592
- });
1593
- return null;
1594
- }
1595
-
1596
- async function executeDownloadExecutor(
1597
- run: PullRunContext,
1598
- downloadItems: PullDownloadItem[],
1599
- transferConcurrency: number,
1600
- counters: PullCounters,
1601
- ): Promise<void> {
1602
- if (downloadItems.length === 0) return;
1603
-
1604
- await primeObjectTransport(
1605
- run.ctx,
1606
- "get",
1607
- downloadItems.map((d) => d.remoteFile.key),
1608
- );
1609
-
1610
- const queue = [...downloadItems];
1611
- const inFlight: Set<Promise<unknown>> = new Set();
1612
- const workerErrors: Error[] = [];
1613
-
1614
- while (queue.length > 0 || inFlight.size > 0) {
1615
- while (inFlight.size < transferConcurrency && queue.length > 0) {
1616
- const downloadItem = queue.shift()!;
1617
- const p: Promise<void> = downloadOne(run, downloadItem, counters)
1618
- .catch((err: unknown) => {
1619
- workerErrors.push(err instanceof Error ? err : new Error(String(err)));
1620
- })
1621
- .finally(() => {
1622
- inFlight.delete(p);
1623
- });
1624
- inFlight.add(p);
1625
- }
1626
- if (inFlight.size > 0) {
1627
- await Promise.race(Array.from(inFlight));
1628
- }
1629
- }
1630
-
1631
- if (workerErrors.length > 0) {
1632
- writeJournal(run.journalSlug, run.journal);
1633
- const first = workerErrors[0]!;
1634
- if (workerErrors.length > 1) {
1635
- first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
1636
- }
1637
- throw first;
1638
- }
1639
- }
1640
-
1641
- async function downloadOne(
1642
- run: PullRunContext,
1643
- downloadItem: PullDownloadItem,
1644
- counters: PullCounters,
1645
- ): Promise<void> {
1646
- const { remoteFile, localPath } = downloadItem;
1647
-
1648
- await refreshRunContextIfExpiring(run);
1649
-
1650
- if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
1651
- counters.filesSkipped++;
1652
- run.emit({
1653
- type: "error",
1654
- path: remoteFile.key,
1655
- message: "download skipped: local parent escaped the sync root",
1656
- });
1657
- return;
1658
- }
1659
-
1660
- try {
1661
- const { metadata, contentHash, contentSize, modeWarnings } = await downloadFile(
1662
- run.ctx,
1663
- remoteFile.key,
1664
- localPath,
1665
- {
1666
- beforeReplace: () => {
1667
- if (!matchesLocalSnapshot(localPath, downloadItem.localSnapshot)) {
1668
- throw new LocalSnapshotChangedError(remoteFile.key);
1669
- }
1670
- },
1671
- },
1672
- );
1673
- reportModeWarnings(run, modeWarnings);
1674
- const author = metadata?.["created-by"] ?? null;
1675
- const createdBySub = metadata?.["created-by-sub"];
1676
-
1677
- const localLstat = fs.lstatSync(localPath);
1678
- const isLocalSymlink = localLstat.isSymbolicLink();
1679
- let hash: string;
1680
- if (isLocalSymlink) {
1681
- const target = readlinkOrNull(localPath);
1682
- if (target === null) {
1683
- counters.filesSkipped++;
1684
- run.emit({
1685
- type: "not-shipped",
1686
- reason: "unreadable-link",
1687
- count: 1,
1688
- samplePaths: [remoteFile.key],
1689
- });
1690
- return;
1691
- }
1692
- hash = hashSymlinkTarget(target);
1693
- } else {
1694
- hash = contentHash ?? hashFile(localPath);
1695
- }
1696
- const size = isLocalSymlink ? 0 : (contentSize ?? fs.statSync(localPath).size);
1697
-
1698
- updateEntry(
1699
- run.journal,
1700
- remoteFile.key,
1701
- hash,
1702
- size,
1703
- "down",
1704
- localPath,
1705
- {
1706
- remoteEtag: remoteFile.etag,
1707
- mtimeMs: localLstat.mtimeMs,
1708
- ...(createdBySub !== undefined ? { createdBySub } : {}),
1709
- kind: isLocalSymlink ? "symlink" : "file",
1710
- },
1711
- );
1712
-
1713
- const priorEntry = getEntry(run.journal, remoteFile.key);
1714
- const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
1715
- run.emit({
1716
- type: "progress",
1717
- path: remoteFile.key,
1718
- bytes: size,
1719
- ...(remoteJournalMessage ? { message: remoteJournalMessage } : {}),
1720
- ...(author ? { author } : {}),
1721
- });
1722
-
1723
- counters.filesDownloaded++;
1724
- counters.bytesDownloaded += size;
1725
- counters.changedPaths.push(remoteFile.key);
1726
- } catch (err) {
1727
- if (err instanceof VaultAuthError) throw err;
1728
- if (err instanceof LocalSnapshotChangedError) {
1729
- reportLocalSnapshotConflict(run, counters, remoteFile.key);
1730
- } else if (isAccessDenied(err)) {
1731
- counters.filesSkipped++;
1732
- } else if (err instanceof DanglingSymlinkParentError) {
1733
- // Per-object condition, not a company failure. The pre-flight
1734
- // containment check can miss this: it runs before the download, and a
1735
- // sibling worker in the SAME batch may materialize the parent symlink in
1736
- // between (the vault stores the link and the files beneath it as
1737
- // separate objects, ordered arbitrarily). Skip loudly — never silently —
1738
- // so the run stays `complete` instead of `errored` on every cycle.
1739
- counters.filesSkipped++;
1740
- run.emit({
1741
- type: "error",
1742
- path: remoteFile.key,
1743
- message: err.message,
1744
- });
1745
- } else {
1746
- run.emit({
1747
- type: "error",
1748
- path: remoteFile.key,
1749
- message: err instanceof Error ? err.message : String(err),
1750
- });
1751
- }
1752
- return;
1753
- }
1754
-
1755
- run.downloadsSinceJournalCheckpoint++;
1756
- if (
1757
- run.downloadsSinceJournalCheckpoint >=
1758
- DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE
1759
- ) {
1760
- // writeJournal is already fsync + rename atomic. Persist completed work
1761
- // before scheduling more objects so an abrupt process exit only replays a
1762
- // small tail, not an entire bulk download.
1763
- writeJournal(run.journalSlug, run.journal);
1764
- run.downloadsSinceJournalCheckpoint = 0;
1765
- }
1766
- }
1767
-
1768
- async function emitAndReportNewFiles(
1769
- run: PullRunContext,
1770
- plan: PullPlan,
1771
- ): Promise<void> {
1772
- const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
1773
- // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
1774
- // created-by HEADs below reuse the cache instead of each minting its own
1775
- // presign. Without this, a big catch-up pull (hundreds of new files) bursts
1776
- // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
1777
- // then fails. Mirrors the tombstone HEAD-verify pre-prime.
1778
- await primeObjectTransport(
1779
- run.ctx,
1780
- "get",
1781
- plan.newFiles.map((nf) => nf.path),
1782
- );
1783
- const HEAD_CONCURRENCY = 5;
1784
- for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
1785
- const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
1786
- const results = await Promise.all(
1787
- batch.map(async (nf) => {
1788
- let addedBy: string | null = null;
1789
- try {
1790
- const head = await headRemoteFile(run.ctx, nf.path);
1791
- if (head?.metadata?.["created-by"]) {
1792
- addedBy = head.metadata["created-by"];
1793
- }
1794
- } catch (headErr) {
1795
- if (headErr instanceof VaultAuthError) throw headErr;
1796
- try {
1797
- console.error(
1798
- `[hq-sync] HeadObject failed for ${nf.path}: ${
1799
- headErr instanceof Error ? headErr.message : String(headErr)
1800
- }`,
1801
- );
1802
- } catch {
1803
- // Swallow — logging must never break sync.
1804
- }
1805
- }
1806
- return { path: nf.path, bytes: nf.bytes, addedBy };
1807
- }),
1808
- );
1809
- enrichedNewFiles.push(...results);
1810
- }
1811
- run.emit({ type: "new-files", files: enrichedNewFiles });
1812
- await reportNewFilesToNotify(
1813
- run.vaultConfig,
1814
- run.ctx.uid,
1815
- run.ctx.slug,
1816
- enrichedNewFiles,
1817
- run.options.telemetryClaims,
1818
- );
1819
- }
1820
-
1821
- async function verifyPlannedJournalTombstones(
1822
- run: PullRunContext,
1823
- plan: PullPlan,
1824
- ): Promise<void> {
1825
- if (plan.tombstones.length === 0) return;
1826
-
1827
- // Scope-invalid journal keys (company mode, `companies/…` prefix) skip the
1828
- // HEAD verify: the presign transport's server validator rejects such keys
1829
- // outright (INVALID_KEY_COMPANIES_SCOPED), so a HEAD can never succeed and
1830
- // would defer the tombstone forever — leaving the poisoned journal entry
1831
- // (and its doubled-tree local file) re-firing on every sync. The key can
1832
- // never legitimately exist in a company vault, so treating it as verified
1833
- // is safe: the executor cleans the local copy + journal entry instead of
1834
- // the push leg ever re-uploading it (tombstone-not-reupload).
1835
- const autoVerified: PullPlan["tombstones"] = [];
1836
- const toHeadVerify: PullPlan["tombstones"] = [];
1837
- for (const tombstone of plan.tombstones) {
1838
- if (run.options.personalMode !== true && tombstone.key.startsWith("companies/")) {
1839
- autoVerified.push(tombstone);
1840
- } else {
1841
- toHeadVerify.push(tombstone);
1842
- }
1843
- }
1844
- plan.tombstones = toHeadVerify;
1845
- if (plan.tombstones.length === 0) {
1846
- plan.tombstones = autoVerified;
1847
- return;
1848
- }
1849
-
1850
- await primeObjectTransport(run.ctx, "get", plan.tombstones.map((t) => t.key));
1851
-
1852
- const HEAD_VERIFY_CONCURRENCY = 5;
1853
- const verified: PullPlan["tombstones"] = [];
1854
- for (let i = 0; i < plan.tombstones.length; i += HEAD_VERIFY_CONCURRENCY) {
1855
- const batch = plan.tombstones.slice(i, i + HEAD_VERIFY_CONCURRENCY);
1856
- const results = await Promise.all(
1857
- batch.map(async (tombstone) => {
1858
- try {
1859
- const head = await headRemoteFile(run.ctx, tombstone.key);
1860
- return head === null ? tombstone : null;
1861
- } catch (err) {
1862
- if (err instanceof VaultAuthError) throw err;
1863
- if (isAccessDenied(err)) return null;
1864
- run.emit({
1865
- type: "error",
1866
- path: tombstone.key,
1867
- message: `tombstone HEAD verify failed (deferring): ${
1868
- err instanceof Error ? err.message : String(err)
1869
- }`,
1870
- });
1871
- return null;
1872
- }
1873
- }),
1874
- );
1875
- for (const tombstone of results) {
1876
- if (tombstone !== null) verified.push(tombstone);
1877
- }
1878
- }
1879
- plan.tombstones = [...autoVerified, ...verified];
1880
- }
1881
-
1882
- function executeJournalTombstoneDeletes(
1883
- run: PullRunContext,
1884
- plan: PullPlan,
1885
- counters: PullCounters,
1886
- ): void {
1887
- for (const tombstone of plan.tombstones) {
1888
- const { key, localPath, localSnapshot } = tombstone;
1889
- if (!isDownloadWritePathStillContained(run.companyRoot, key, localPath)) {
1890
- run.emit({
1891
- type: "error",
1892
- path: key,
1893
- message: "tombstone unlink skipped: local parent escaped the sync root",
1894
- });
1895
- continue;
1896
- }
1897
- let removedSomething = false;
1898
- try {
1899
- const lstat = fs.lstatSync(localPath);
1900
- if (!matchesLocalSnapshot(localPath, localSnapshot)) {
1901
- reportLocalSnapshotConflict(run, counters, key);
1902
- continue;
1903
- }
1904
- if (lstat.isSymbolicLink() || lstat.isFile()) {
1905
- fs.unlinkSync(localPath);
1906
- removedSomething = true;
1907
- } else if (lstat.isDirectory()) {
1908
- // A dir at a key is converged by dropping only the journal entry.
1909
- }
1910
- } catch (err: unknown) {
1911
- const code =
1912
- err && typeof err === "object" && "code" in err
1913
- ? (err as { code?: string }).code
1914
- : undefined;
1915
- if (code !== "ENOENT") {
1916
- run.emit({
1917
- type: "error",
1918
- path: key,
1919
- message: `tombstone unlink failed: ${
1920
- err instanceof Error ? err.message : String(err)
1921
- }`,
1922
- });
1923
- continue;
1924
- }
1925
- }
1926
- removeEntry(run.journal, key);
1927
- counters.filesTombstoned++;
1928
- counters.changedPaths.push(key);
1929
- run.emit({
1930
- type: "progress",
1931
- path: key,
1932
- bytes: 0,
1933
- deleted: true,
1934
- message: removedSomething ? "tombstone (cross-machine delete)" : "tombstone (already absent locally)",
1935
- });
1936
- }
1937
- }
1938
-
1939
- function finalizePullRun(
1940
- run: PullRunContext,
1941
- plan: PullPlan,
1942
- scopeRun: ScopeShrinkRun,
1943
- counters: PullCounters,
1944
- ): SyncResult {
1945
- appendPullRecord(run.journal, {
1946
- pullId: generatePullId(),
1947
- companyUid: run.ctx.uid,
1948
- startedAt: run.startedAt,
1949
- completedAt: new Date().toISOString(),
1950
- syncMode: run.syncMode,
1951
- prefixSet: run.currentPrefixSet,
1952
- scopeChangeDetected: scopeRun.shrinkPlan.scopeChangeDetected,
1953
- orphansRemoved: scopeRun.scopeOrphansRemoved,
1954
- orphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1955
- });
1956
-
1957
- run.journal.lastSync = new Date().toISOString();
1958
- writeJournal(run.journalSlug, run.journal);
1959
-
1960
- const changedOnDisk =
1961
- counters.filesDownloaded > 0 ||
1962
- counters.filesTombstoned > 0 ||
1963
- scopeRun.scopeOrphansRemoved > 0;
1964
- if (!run.options.skipReindex && changedOnDisk) {
1965
- try {
1966
- const result = reindex({ repoRoot: run.hqRoot, skipLock: true });
1967
- void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
1968
- eventName: "reindex_run",
1969
- source: "hq-sync",
1970
- companyUid: run.ctx.uid,
1971
- properties: {
1972
- result: result.status === 0 ? "completed" : "failed",
1973
- statusCode: result.status,
1974
- lockResult: "skipped",
1975
- changedCount:
1976
- counters.changedPaths.length + scopeRun.changedPaths.length,
1977
- },
1978
- }, { claims: run.options.telemetryClaims });
1979
- } catch (err) {
1980
- logPostSyncMaintenanceFailure("post-sync reindex", run.hqRoot, err);
1981
- // best-effort: a post-sync refresh failure never fails the sync
1982
- }
1983
- }
1984
-
1985
- return {
1986
- filesDownloaded: counters.filesDownloaded,
1987
- bytesDownloaded: counters.bytesDownloaded,
1988
- filesSkipped: counters.filesSkipped,
1989
- conflicts: counters.conflicts,
1990
- conflictPaths: counters.conflictPaths,
1991
- aborted: false,
1992
- newFiles: plan.newFiles,
1993
- newFilesCount: plan.newFilesCount,
1994
- filesExcludedByPolicy: plan.filesExcludedByPolicy,
1995
- filesTombstoned: counters.filesTombstoned,
1996
- filesOutOfScope: counters.filesOutOfScope,
1997
- scopeOrphansRemoved: scopeRun.scopeOrphansRemoved,
1998
- scopeOrphansBlocked: scopeRun.shrinkResult.dirtyTombstoned,
1999
- changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
2000
- };
2001
- }
2002
-
2003
- /**
2004
- * Surface the shared-vs-all materialization gap (feedback_d2082110). When a
2005
- * pull leg ran under a membership-scoped `syncMode` and skipped one or more
2006
- * remote keys as out-of-scope, emit a single summary event so the operator
2007
- * SEES that not all vault content materialized on this device and knows the
2008
- * lever — raise the membership's access level to `all`. Silent in `all` mode
2009
- * (nothing is scoped away) and when the membership scope withheld nothing: no
2010
- * gap, no noise. Pure — count and samples are both read off the plan's
2011
- * `skip-out-of-scope` items, no I/O.
2012
- *
2013
- * Counts ONLY `reason: "membership-scope"` items. `push-only` keys
2014
- * (`sessions/`, US-006) are also classified `skip-out-of-scope` and also land
2015
- * on the `filesOutOfScope` axis, but they are withheld in EVERY mode — `all`
2016
- * included — so raising the access level does NOT materialize them. Counting
2017
- * them here would inflate the gap, could fill `samplePaths` entirely with
2018
- * session keys, and could fire the event (and its "set this membership to
2019
- * `all`" advice) when the membership scope withheld nothing at all — sending
2020
- * the operator back into exactly the flip-everything-to-`all`-and-still-miss
2021
- * -files loop this event exists to end. `SyncResult.filesOutOfScope` keeps its
2022
- * original both-causes meaning; only this advisory is narrowed.
2023
- */
2024
- function emitScopeMaterializationGap(run: PullRunContext, plan: PullPlan): void {
2025
- if (run.syncMode === "all") return;
2026
- const withheldByMembership = plan.items.filter(
2027
- (item) => item.action === "skip-out-of-scope" && item.reason === "membership-scope",
2028
- );
2029
- if (withheldByMembership.length === 0) return;
2030
- run.emit({
2031
- type: "scope-materialization-gap",
2032
- count: withheldByMembership.length,
2033
- samplePaths: withheldByMembership.slice(0, 10).map((item) => item.remoteFile.key),
2034
- syncMode: run.syncMode,
2035
- });
2036
- }
2037
-
2038
- /**
2039
- * Decide whether a remote object present in the LIST is a GENUINE RE-CREATE
2040
- * written AFTER a FILE_TOMBSTONE — in which case the tombstone is stale and the
2041
- * object must still sync (the tombstone is not permanent suppression). Returns
2042
- * true to ALLOW the download (re-create), false to honor the tombstone.
2043
- *
2044
- * Decisive signal: the remote object's `lastModified` strictly newer than the
2045
- * tombstone's `deletedAt`. A delete-marker + tombstone are written together at
2046
- * delete time, so any object that post-dates the tombstone is a new write at
2047
- * that key. An object at-or-before the tombstone is the deleted version (a
2048
- * stale re-push or an un-propagated delete-marker) → honor the tombstone.
2049
- *
2050
- * Fail-OPEN (treat as re-create, allow download) when the comparison can't be
2051
- * made — a malformed `deletedAt`, or no remote `lastModified`. Hiding a file the
2052
- * user can see is worse than re-pulling one they deleted; the latter is
2053
- * recoverable, the former looks like data loss.
2054
- */
2055
- function isRemoteRecreateAfterTombstone(
2056
- remote: { lastModified?: Date },
2057
- tombstone: CompanyTombstone,
2058
- ): boolean {
2059
- const deletedAtMs = Date.parse(tombstone.deletedAt);
2060
- if (Number.isNaN(deletedAtMs)) return true; // malformed tombstone → don't suppress
2061
- const remoteMs =
2062
- remote.lastModified instanceof Date
2063
- ? remote.lastModified.getTime()
2064
- : NaN;
2065
- if (Number.isNaN(remoteMs)) return true; // no remote timestamp → don't suppress
2066
- return remoteMs > deletedAtMs;
2067
- }
2068
-
2069
- function hasTraversalSegment(key: string): boolean {
2070
- return key.split("/").some((segment) => segment === "..");
2071
- }
2072
-
2073
- function isPathWithin(root: string, candidate: string): boolean {
2074
- const relative = path.relative(root, candidate);
2075
- return (
2076
- relative === "" ||
2077
- (!relative.startsWith("..") && !path.isAbsolute(relative))
2078
- );
2079
- }
2080
-
2081
- function deepestExistingAncestor(start: string): string | null {
2082
- let current = start;
2083
- for (;;) {
2084
- try {
2085
- fs.lstatSync(current);
2086
- return current;
2087
- } catch (err: unknown) {
2088
- const code =
2089
- err && typeof err === "object" && "code" in err
2090
- ? (err as { code?: string }).code
2091
- : undefined;
2092
- if (code !== "ENOENT" && code !== "ENOTDIR") return null;
2093
- }
2094
-
2095
- const parent = path.dirname(current);
2096
- if (parent === current) return null;
2097
- current = parent;
2098
- }
2099
- }
2100
-
2101
- function resolveContainedVaultPath(root: string, key: string): string | null {
2102
- if (isMalformedVaultKey(key) || hasTraversalSegment(key)) return null;
2103
-
2104
- const resolvedRoot = path.resolve(root);
2105
- const resolvedLocal = localPathForVaultKey(resolvedRoot, key);
2106
- if (!isPathWithin(resolvedRoot, resolvedLocal)) return null;
2107
-
2108
- let realRoot: string;
2109
- try {
2110
- realRoot = fs.realpathSync.native(resolvedRoot);
2111
- } catch {
2112
- // If the vault root does not exist yet, no below-root symlink component can
2113
- // already exist to redirect this key. Preserve first-pull behavior.
2114
- return resolvedLocal;
2115
- }
2116
-
2117
- const existingAncestor = deepestExistingAncestor(path.dirname(resolvedLocal));
2118
- if (existingAncestor === null) return null;
2119
- try {
2120
- const realAncestor = fs.realpathSync.native(existingAncestor);
2121
- if (!isPathWithin(realRoot, realAncestor)) return null;
2122
- } catch {
2123
- return null;
2124
- }
2125
- return resolvedLocal;
2126
- }
2127
-
2128
- function isDownloadWritePathStillContained(
2129
- root: string,
2130
- key: string,
2131
- localPath: string,
2132
- ): boolean {
2133
- const resolved = resolveContainedVaultPath(root, key);
2134
- return resolved !== null && path.resolve(resolved) === path.resolve(localPath);
2135
- }
2136
-
2137
- function snapshotLocalState(localPath: string): LocalSnapshot | null {
2138
- try {
2139
- const lstat = fs.lstatSync(localPath);
2140
- if (lstat.isSymbolicLink()) {
2141
- return { kind: "symlink", hash: hashSymlinkTarget(fs.readlinkSync(localPath)) };
2142
- }
2143
- if (lstat.isFile()) {
2144
- return { kind: "file", hash: hashFile(localPath) };
2145
- }
2146
- if (lstat.isDirectory()) return { kind: "directory" };
2147
- return { kind: "other" };
2148
- } catch (err: unknown) {
2149
- const code =
2150
- err && typeof err === "object" && "code" in err
2151
- ? (err as { code?: string }).code
2152
- : undefined;
2153
- return code === "ENOENT" ? { kind: "absent" } : null;
2154
- }
2155
- }
2156
-
2157
- function matchesLocalSnapshot(localPath: string, expected: LocalSnapshot): boolean {
2158
- const actual = snapshotLocalState(localPath);
2159
- if (actual === null || actual.kind !== expected.kind) return false;
2160
- return (
2161
- (actual.kind !== "file" && actual.kind !== "symlink") ||
2162
- actual.hash === (expected as Extract<LocalSnapshot, { hash: string }>).hash
2163
- );
2164
- }
2165
-
2166
- function hasCurrentLocalDeleteIntent(
2167
- entry: SyncJournal["files"][string] | undefined,
2168
- ): boolean {
2169
- const intent = entry?.localDeleteIntent;
2170
- return !!(
2171
- intent &&
2172
- intent.version === 1 &&
2173
- entry.remoteEtag &&
2174
- entry.kind &&
2175
- intent.remoteEtag === entry.remoteEtag &&
2176
- intent.localHash === entry.hash &&
2177
- intent.localKind === entry.kind
2178
- );
2179
- }
2180
-
2181
- function reportLocalSnapshotConflict(
2182
- run: PullRunContext,
2183
- counters: PullCounters,
2184
- key: string,
2185
- ): void {
2186
- counters.conflicts++;
2187
- counters.conflictPaths.push(key);
2188
- run.emit({ type: "conflict", path: key, direction: "pull", resolution: "skip" });
2189
- }
2190
-
2191
- class LocalSnapshotChangedError extends Error {
2192
- constructor(key: string) {
2193
- super(`local state changed while downloading ${key}; replanning required`);
2194
- }
2195
- }
2196
-
2197
- /**
2198
- * Stage-1 classification for a single remote object. Each remote file falls
2199
- * into exactly one bucket; the executor in `sync()` switches on `action` to
2200
- * decide what to do. `localHash` is carried on `conflict` items so the
2201
- * executor can hand it to `resolveConflict` without re-hashing.
2202
- */
2203
- type LocalSnapshot =
2204
- | { kind: "absent" | "directory" | "other" }
2205
- | { kind: "file" | "symlink"; hash: string };
2206
-
2207
- /**
2208
- * Why a remote key was classified `skip-out-of-scope`. The two causes look
2209
- * identical in the count but have OPPOSITE remedies, so they must never be
2210
- * reported as one: `membership-scope` is fixed by raising the membership's
2211
- * access level to `all`; `push-only` is not fixed by that at all.
2212
- */
2213
- type OutOfScopeReason = "membership-scope" | "push-only";
2214
-
2215
- type PullPlanItem =
2216
- | {
2217
- action: "download";
2218
- remoteFile: RemoteFile;
2219
- localPath: string;
2220
- isNew: boolean;
2221
- localSnapshot: LocalSnapshot;
2222
- }
2223
- | { action: "skip-ignored"; remoteFile: RemoteFile; localPath: string }
2224
- | { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
2225
- | { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
2226
- | { action: "skip-local-only"; remoteFile: RemoteFile; localPath: string }
2227
- // A local symlink was classified as a link but Windows (or a filesystem
2228
- // race) would not expose its target. This stays distinct from generic
2229
- // skip-local-only so the executor can report the exact path without making
2230
- // the whole company leg fail.
2231
- | { action: "skip-unreadable-link"; remoteFile: RemoteFile; localPath: string }
2232
- // A stale personal-overlay symlink marker in the vault sitting at a key that
2233
- // is ALSO a real, release-shipped core directory (e.g. an old
2234
- // `core/knowledge/public/agent-browser` overlay marker after a release
2235
- // shipped a real directory of the same name). The local release-shipped
2236
- // directory is authoritative; the marker is inert derived state that must
2237
- // NEVER overwrite core. Distinct from skip-local-only so the benign,
2238
- // no-action-needed case is never surfaced with the alarming "rm -rf the
2239
- // local directory" reconciliation advice.
2240
- | { action: "skip-stale-overlay-marker"; remoteFile: RemoteFile; localPath: string }
2241
- // A directory-overlay marker (a single vault object at a key that ALSO has
2242
- // child objects under it in the same LIST — a symlink record for a directory
2243
- // mirrored from another tree) colliding with a local REAL directory whose
2244
- // contents are already materialized. The marker can never be renamed over the
2245
- // directory; its children carry the real content and materialize via their own
2246
- // plan items. A benign, no-action skip — distinct from skip-local-only so the
2247
- // case is never surfaced with the alarming (and, for a materialized directory,
2248
- // data-destroying) "rm -rf the local directory to pull" advice. feedback_d2082110.
2249
- | { action: "skip-overlay-marker-with-children"; remoteFile: RemoteFile; localPath: string }
2250
- // Remote keys refused by ephemeral-mirror policy. The push walker has
2251
- // refused to upload these since 5.33.0; the pull walker now refuses to
2252
- // download them so legacy litter in cloud staging drains naturally.
2253
- | { action: "skip-excluded-policy"; remoteFile: RemoteFile; localPath: string }
2254
- // Remote keys present in the remote LIST (and accessible per STS) but
2255
- // deliberately not downloaded. Two DIFFERENT causes share this action and
2256
- // the `filesOutOfScope` count, so `reason` discriminates them:
2257
- //
2258
- // `membership-scope` — outside the effective `syncMode` prefix set
2259
- // (US-005). RAISING the membership's access level to `all` WOULD
2260
- // materialize these keys.
2261
- // `push-only` — under a push-only exclude prefix (`sessions/`,
2262
- // US-006). These are withheld in EVERY mode, `all` included; raising
2263
- // the access level does NOT materialize them (`hq files get` does).
2264
- //
2265
- // Only `membership-scope` items may be attributed to the shared-vs-all gap
2266
- // — see `emitScopeMaterializationGap`. feedback_d2082110.
2267
- | {
2268
- action: "skip-out-of-scope";
2269
- remoteFile: RemoteFile;
2270
- localPath: string;
2271
- reason: OutOfScopeReason;
2272
- }
2273
- | {
2274
- // Mixed-version amplifier guard — see the `skip-junk-key-spelling`
2275
- // event doc. Carries the journaled spelling for the warning surface.
2276
- action: "skip-junk-key-spelling";
2277
- remoteFile: RemoteFile;
2278
- localPath: string;
2279
- journaledKey: string;
2280
- }
2281
- // Remote key present in the LIST but carrying a FILE_TOMBSTONE that marks it
2282
- // intentionally deleted (and the remote object is NOT a newer re-create). The
2283
- // executor deletes any local copy and drops the journal entry — the
2284
- // authoritative-delete arm of delete-resync. Unlike `PullPlan.tombstones`
2285
- // (keys ABSENT from the LIST, HEAD-verified before delete), these are
2286
- // suppressed purely on the FILE_TOMBSTONE authority and skip HEAD-verify (the
2287
- // remote object is present by definition).
2288
- | {
2289
- action: "tombstone-delete";
2290
- remoteFile: RemoteFile;
2291
- localPath: string;
2292
- localSnapshot: LocalSnapshot;
2293
- }
2294
- | {
2295
- action: "conflict";
2296
- remoteFile: RemoteFile;
2297
- localPath: string;
2298
- localHash: string;
2299
- // Captured from the planner's lstat (does NOT follow symlinks),
2300
- // so a dangling symlink doesn't make the conflict executor
2301
- // statSync-then-ENOENT and abort the whole pull. See the
2302
- // executor: prior to this field, the prompt builder called
2303
- // fs.statSync(localPath).mtime to populate the conflict UI's
2304
- // "local modified at X" line, which crashed for dangling links.
2305
- localMtime: Date;
2306
- // Symlink-aware size for the post-resolution journal stamp.
2307
- // For a symlink, journal convention is 0 (the wire body is
2308
- // empty + target metadata; matches what the push side stamps).
2309
- // For a regular file, mirrors the on-disk byte length. The
2310
- // executor's keep/skip branch reads this instead of calling
2311
- // fs.statSync(localPath), which would follow a dangling link
2312
- // and silently fail to stamp the journal — leaving the
2313
- // conflict to re-fire forever.
2314
- localSize: number;
2315
- localSnapshot: LocalSnapshot;
2316
- };
2317
-
2318
- interface PullPlan {
2319
- items: PullPlanItem[];
2320
- filesToDownload: number;
2321
- bytesToDownload: number;
2322
- filesToSkip: number;
2323
- filesToConflict: number;
2324
- /** Files classified as new (no local counterpart at classification time). */
2325
- newFiles: Array<{ path: string; bytes: number }>;
2326
- newFilesCount: number;
2327
- /** Count of remote keys refused by ephemeral-mirror policy. */
2328
- filesExcludedByPolicy: number;
2329
- /** Count of remote keys skipped because they fall outside the sync scope. */
2330
- filesOutOfScope: number;
2331
- /**
2332
- * Journal-known keys missing from the remote LIST. The executor will
2333
- * apply each as a local delete (file or symlink) + journal removal,
2334
- * propagating the peer's push-side delete cross-machine (Bug #9).
2335
- * Carried on the plan so the executor can iterate without re-walking.
2336
- */
2337
- tombstones: Array<{
2338
- key: string;
2339
- localPath: string;
2340
- localSnapshot: LocalSnapshot;
2341
- }>;
2342
- /**
2343
- * Count of `tombstone-delete` items — remote keys present in the LIST but
2344
- * suppressed by a FILE_TOMBSTONE (delete-resync). Surfaced on the plan event's
2345
- * `filesToDelete` axis; the per-item executor applies the local delete.
2346
- */
2347
- filesToTombstoneDelete: number;
2348
- /**
2349
- * Remote keys skipped because the server will never presign them. Counted in
2350
- * `filesExcludedByPolicy`; carried with the server-compatible error code so
2351
- * the caller can log one warning per key and emit telemetry. Healthy
2352
- * baseline is an empty array (policy
2353
- * hq-alert-baseline-calibration: zero expected, any occurrence is signal).
2354
- */
2355
- invalidRemoteKeys: Array<{ path: string; errorCode: string }>;
2356
- }
2357
-
2358
- /**
2359
- * Stage-1 planning pass: classify every remote file into download / skip /
2360
- * conflict buckets without performing any S3 transfers. Local hashes are
2361
- * computed here (not in the transfer loop) so the totals returned reflect
2362
- * the real outcome of the upcoming Stage-2 execution rather than an
2363
- * upper-bound guess.
2364
- *
2365
- * Pure function: no S3 calls, no journal writes, no event emission. The
2366
- * caller (`sync()`) is responsible for emitting the resulting plan event
2367
- * before iterating `items`.
2368
- */
2369
- // Pull-leg intentional-delete bulk-asymmetry breaker. Mirrors the push leg's
2370
- // BULK_ASYMMETRY_* (share.ts): an abnormally large fraction of clean files
2371
- // vanishing at once is a corrupt mirror / bulk op / unmount (the ridge-incident
2372
- // class), not intent — so at/above the threshold the pull leg drift-restores
2373
- // every candidate instead of tombstoning it. Below it, a missing clean file is
2374
- // an intentional delete.
2375
- const PULL_INTENTIONAL_DELETE_MIN_ABS = 10;
2376
- const PULL_INTENTIONAL_DELETE_RATIO = 0.1;
2377
-
2378
- /**
2379
- * win32 only: true when an existing local symlink's TARGET exists but the
2380
- * link cannot be traversed as that target's type — the file-flavor-link-at-
2381
- * a-directory shape minted while the target was absent (or by an older
2382
- * client). A dangling link is NOT broken-flavored (there is nothing to
2383
- * repair until the target appears). POSIX links are flavorless: always
2384
- * false off win32.
2385
- */
2386
- export function win32SymlinkFlavorBroken(
2387
- localPath: string,
2388
- win32: boolean = process.platform === "win32",
2389
- ): boolean {
2390
- if (!win32) return false;
2391
- let target: string;
2392
- try {
2393
- target = fs.readlinkSync(localPath);
2394
- } catch {
2395
- return false;
2396
- }
2397
- const resolved = path.resolve(path.dirname(localPath), target);
2398
- let targetIsDir: boolean;
2399
- try {
2400
- targetIsDir = fs.statSync(resolved).isDirectory();
2401
- } catch {
2402
- return false; // dangling target — nothing to repair yet
2403
- }
2404
- try {
2405
- return fs.statSync(localPath).isDirectory() !== targetIsDir;
2406
- } catch {
2407
- return true; // target exists but the link cannot traverse: wrong flavor
2408
- }
2409
- }
2410
-
2411
- function computePullPlan(
2412
- remoteFiles: RemoteFile[],
2413
- journal: SyncJournal,
2414
- companyRoot: string,
2415
- shouldSync: (filePath: string, isDir?: boolean) => boolean,
2416
- personalMode: boolean,
2417
- teamSyncedSlugs: ReadonlySet<string> | null,
2418
- // Coalesced, company-relative prefixes the pull is scoped to (US-005).
2419
- // `[""]` (the `all`-mode value) covers everything via `isCoveredByAny`, so
2420
- // the scope filter below becomes a no-op and legacy behavior is preserved.
2421
- prefixSet: readonly ScopePrefixInput[],
2422
- // FILE_TOMBSTONE records (POSIX-keyed) for the company — the durable
2423
- // "this key was intentionally deleted" signal the planner consults before
2424
- // re-downloading a key, so a deleted folder does not resync back in
2425
- // (delete-resync). An empty map (the default / degraded-fetch case)
2426
- // preserves the legacy behavior bit-for-bit. (Named `fileTombstones` to avoid
2427
- // shadowing the local `tombstones` string[] — the journal-vs-LIST delete set.)
2428
- fileTombstones: ReadonlyMap<string, CompanyTombstone> = new Map(),
2429
- // Push-only exclusion prefixes (US-006: `sessions/`). Keys covered by any of
2430
- // these are classified `skip-out-of-scope` in EVERY mode — including `all` —
2431
- // AFTER passing the inclusion check. Empty (the default) preserves legacy
2432
- // behavior bit-for-bit. Applied ONLY here (the download filter), never to
2433
- // scope-shrink, so an authored/pinned excluded key is not re-pulled yet also
2434
- // never orphaned.
2435
- excludePrefixes: readonly ScopePrefixInput[] = [],
2436
- ): PullPlan {
2437
- const items: PullPlanItem[] = [];
2438
- // Remote keys the server will never presign — collected so the caller can
2439
- // warn + emit telemetry without allowing one poison object to fail a pull.
2440
- const invalidRemoteKeys: Array<{ path: string; errorCode: string }> = [];
2441
- // Clean/current/non-divergent files missing locally — collected here, then
2442
- // resolved post-loop by the bulk-asymmetry decision (tombstone vs. restore).
2443
- const intentionalDeleteCandidates: Array<{
2444
- remoteFile: RemoteFile;
2445
- localPath: string;
2446
- }> = [];
2447
-
2448
- // Junk-spelling guard (mixed-version amplifier): index every journaled key
2449
- // by its canonical (percent-decode fixpoint) form so a remote twin minted
2450
- // under a different spelling is recognized before it materializes. When
2451
- // several journaled spellings already share one canonical form (a
2452
- // pre-guard fork), the exactly-canonical spelling is preferred as the
2453
- // representative; entries journaled under their own exact spelling are
2454
- // never gated by this map (see the guard below), so pre-existing families
2455
- // keep syncing untouched until the vault doctor (US-003) collapses them.
2456
- const journaledKeyByCanonical = new Map<string, string>();
2457
- for (const journaledKey of Object.keys(journal.files)) {
2458
- if (!journaledKey.startsWith(SKILLS_KEY_PREFIX)) continue;
2459
- const canonical = canonicalVaultKeySpelling(journaledKey);
2460
- const existing = journaledKeyByCanonical.get(canonical);
2461
- if (existing === undefined || journaledKey === canonical) {
2462
- journaledKeyByCanonical.set(canonical, journaledKey);
2463
- }
2464
- }
2465
-
2466
- // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
2467
- // keys that carry child objects under `${key}/…` in this same LIST. The
2468
- // vault stores a directory OVERLAY (a symlink into another tree, e.g. a
2469
- // company skill mirrored from a shared repo) as a symlink RECORD at the
2470
- // directory's own key AND stores the directory's contents as separate child
2471
- // objects beneath it. A REAL directory, by contrast, only ever syncs as
2472
- // those child objects — never as a single object AT its own key. So a single
2473
- // remote object sitting at a key that also has children is unambiguously a
2474
- // directory-overlay marker, not a regular file. Precomputed once (pure path
2475
- // derivation, no I/O — the planner is synchronous) so the dir-vs-object
2476
- // collision branch can tell an inert overlay marker apart from a genuine
2477
- // file-vs-directory structural collision. feedback_d2082110.
2478
- const remoteKeysWithChildren = new Set<string>();
2479
- for (const rf of remoteFiles) {
2480
- const key = toPosixKey(rf.key);
2481
- let slash = key.indexOf("/");
2482
- while (slash !== -1) {
2483
- remoteKeysWithChildren.add(key.slice(0, slash));
2484
- slash = key.indexOf("/", slash + 1);
2485
- }
2486
- }
2487
-
2488
- for (const remoteFile of remoteFiles) {
2489
- const invalidKey = classifyVaultKey(
2490
- remoteFile.key,
2491
- personalMode ? "personal" : "company",
2492
- );
2493
- if (invalidKey !== null) {
2494
- invalidRemoteKeys.push({
2495
- path: remoteFile.key,
2496
- errorCode: invalidKey.code,
2497
- });
2498
- items.push({
2499
- action: "skip-excluded-policy",
2500
- remoteFile,
2501
- localPath: companyRoot,
2502
- });
2503
- continue;
2504
- }
2505
-
2506
- const localPath = resolveContainedVaultPath(companyRoot, remoteFile.key);
2507
- if (localPath === null) {
2508
- items.push({ action: "skip-excluded-policy", remoteFile, localPath: companyRoot });
2509
- continue;
2510
- }
2511
-
2512
- // Ephemeral-mirror filter — symmetric with the push-side walker. Bug #2
2513
- // in the 5.33.0 deep-test: the push side has refused to upload conflict
2514
- // mirrors since 5.33.0, but the pull side downloaded them freely from
2515
- // staging, so legacy `.conflict-*` litter rode every sync into clean
2516
- // trees. Refuse them here; the V2 verification test (direct S3 inject
2517
- // bypassing the push filter entirely) is the regression contract.
2518
- if (isEphemeralPath(remoteFile.key)) {
2519
- items.push({ action: "skip-excluded-policy", remoteFile, localPath });
2520
- continue;
2521
- }
2522
-
2523
- // Company-vault double-scoped-key filter. A company vault is already
2524
- // anchored at its company root, so a remote key under `companies/...` is a
2525
- // corrupt doubly-scoped object the vault-service refuses to presign
2526
- // (INVALID_KEY_COMPANIES_SCOPED). Without this, the puller enqueues it, the
2527
- // per-file GET fails, and the whole company wedges at "errored" (runner
2528
- // exit 2) on every run (frogbear, 2026-06-16). Refuse at plan time,
2529
- // symmetric with the malformed/ephemeral filters above. Personal vaults
2530
- // handle `companies/...` keys in the dedicated branch below.
2531
- if (isForbiddenCompanyVaultKey(remoteFile.key, personalMode)) {
2532
- items.push({ action: "skip-excluded-policy", remoteFile, localPath });
2533
- continue;
2534
- }
2535
-
2536
- // Personal-vault policy must be symmetric across both transfer legs.
2537
- // The push walker refuses derived/machine-local/sensitive paths, but old
2538
- // objects can remain in the bucket from clients that predate an exclusion.
2539
- // Refuse those objects here too so a pull cannot rematerialize them (or,
2540
- // for generated skill wrappers, fight reindex over a directory link).
2541
- if (personalMode && isPersonalVaultExcluded(remoteFile.key)) {
2542
- items.push({ action: "skip-excluded-policy", remoteFile, localPath });
2543
- continue;
2544
- }
2545
-
2546
- if (
2547
- personalMode &&
2548
- remoteFile.key.startsWith("companies/") &&
2549
- // EXEMPTION: companies/manifest.yaml is the routing source-of-truth
2550
- // carved INTO the personal vault on the push side
2551
- // (computePersonalVaultPaths). It must round-trip on the pull leg too —
2552
- // skipping it here leaves it forever unjournaled, which re-fires a
2553
- // transient push-side conflict every sync (no journal baseline). Let it
2554
- // fall through to download + journal like any personal file.
2555
- remoteFile.key !== PERSONAL_VAULT_MANIFEST_KEY
2556
- ) {
2557
- // Local (non-cloud) companies sync to the personal vault: keys under
2558
- // `companies/{slug}/...` are legitimate content a peer machine pushed
2559
- // for a company that is not cloud-backed. Allow them through EXCEPT
2560
- // for slugs the operator has an active team-bucket Membership for:
2561
- // those are orphan remnants from before the company became cloud-true,
2562
- // and downloading them would clash with the team-bucket pull at the
2563
- // same disk path.
2564
- //
2565
- // Symmetric to the push-side `decommissionPrefixes` logic in
2566
- // share.ts — both target the same orphan class, both honor the
2567
- // same `teamSyncedSlugs` set derived from the operator's live
2568
- // membership at runtime.
2569
- const slug = remoteFile.key.split("/")[1] ?? "";
2570
- const isTeamSyncedOrphan =
2571
- teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
2572
- if (isTeamSyncedOrphan) {
2573
- items.push({ action: "skip-personal-mode", remoteFile, localPath });
2574
- continue;
2575
- }
2576
- }
2577
-
2578
- // Scope filter (US-005). Keys outside the effective `syncMode` prefix set
2579
- // are not downloaded. `prefixSet` is `[""]` in `all` mode, which
2580
- // `isCoveredByAny` treats as covering everything — so this is a no-op for
2581
- // `all` and preserves the legacy full-bucket pull bit-for-bit. The
2582
- // previously-downloaded counterparts of these keys (if scope just shrank)
2583
- // are pruned separately by the scope-shrink pass in `sync()`.
2584
- // A key can fail the inclusion check AND sit under a push-only exclude
2585
- // prefix. Attribute it to `push-only` in that case: widening the
2586
- // membership to `all` would move it into `prefixSet` but the exclusion
2587
- // below would still withhold it, so it is NOT part of the shared-vs-all
2588
- // gap and must not be advertised as fixable by raising the access level.
2589
- if (!isCoveredByAny(remoteFile.key, prefixSet)) {
2590
- items.push({
2591
- action: "skip-out-of-scope",
2592
- remoteFile,
2593
- localPath,
2594
- reason: isCoveredByAny(remoteFile.key, excludePrefixes)
2595
- ? "push-only"
2596
- : "membership-scope",
2597
- });
2598
- continue;
2599
- }
2600
-
2601
- // Push-only exclusion filter (US-006). A key that PASSED the inclusion check
2602
- // above is STILL not downloaded if it falls under a push-only exclude prefix
2603
- // (`sessions/`). This is what makes sessions push-only in EVERY mode — it
2604
- // subtracts from `all` (whose `prefixSet` is `[""]`, covering everything)
2605
- // and from an owner's role-wide scope alike. Empty `excludePrefixes` (the
2606
- // default) makes this a no-op. Deliberately AFTER inclusion and reported on
2607
- // the same `filesOutOfScope` axis. The matching local file (if a session was
2608
- // previously pulled, authored locally, or pinned+materialized) is NOT pruned
2609
- // here or by scope-shrink — see `planScopeShrink`, which never sees this set.
2610
- if (isCoveredByAny(remoteFile.key, excludePrefixes)) {
2611
- items.push({
2612
- action: "skip-out-of-scope",
2613
- remoteFile,
2614
- localPath,
2615
- reason: "push-only",
2616
- });
2617
- continue;
2618
- }
2619
-
2620
- // LIST gives us no kind signal for the remote object — we don't
2621
- // know whether this key is a regular file or a symlink record
2622
- // until we either HEAD it (expensive — N extra calls per pull) or
2623
- // download it. So probe the ignore filter with BOTH isDir hints
2624
- // and include the entry if either probe passes. Without this,
2625
- // a symlink record at a dir-only-allowlisted path (e.g. an
2626
- // .hqinclude of `companies/*/knowledge/`) would be classified
2627
- // skip-ignored and the symlinks freshly uploaded by the
2628
- // push-side fix would never propagate. Mirrors the dual-hint
2629
- // probe in walkDir/collectFiles. Pure path lookup, no I/O.
2630
- if (!shouldSync(localPath, false) && !shouldSync(localPath, true)) {
2631
- items.push({ action: "skip-ignored", remoteFile, localPath });
2632
- continue;
2633
- }
2634
-
2635
- const journalEntry = getEntry(journal, remoteFile.key);
2636
-
2637
- // Reindex and package wiring own symlinks under these `core/<type>`
2638
- // surfaces; their canonical sources are elsewhere. A prior pull records
2639
- // the remote object as a symlink, then reindex removes the generated link.
2640
- // Without this classification the next pull sees only an absent local path
2641
- // and downloads the same marker forever. Retain regular core files: the
2642
- // journal kind is the discriminator, because LIST exposes no object type.
2643
- // An unchanged ETag is also required: a remote writer can replace a legacy
2644
- // symlink with canonical file content at the same key, and that replacement
2645
- // must still be pulled.
2646
- if (
2647
- personalMode &&
2648
- journalEntry?.kind === "symlink" &&
2649
- isGeneratedCoreMirrorKey(remoteFile.key) &&
2650
- journalEntry.remoteEtag !== undefined &&
2651
- !hasRemoteChanged(remoteFile, journalEntry)
2652
- ) {
2653
- items.push({ action: "skip-excluded-policy", remoteFile, localPath });
2654
- continue;
2655
- }
2656
-
2657
- // Junk-spelling guard — SKILL-FARM KEYS ONLY. The amplifier is specific
2658
- // to `.claude/skills/<ns>:<skill>` wrapper spellings; outside that prefix,
2659
- // keys whose percent-decode fixpoints collide (`notes/a b.md` vs a genuine
2660
- // `notes/a%20b.md`) are DISTINCT files that must both pull.
2661
- // Within the farm: a remote key that is NOT journaled under its own
2662
- // spelling but whose percent-decode fixpoint matches an already-journaled
2663
- // key with a different spelling is a mixed-version twin (`%3A`/`%253A`
2664
- // generations of a canonical `:` key). Materializing it would mint junk
2665
- // local dirs and fork the journal into a second key family for the same
2666
- // logical path — skip it (loudly, never fatally; see the
2667
- // `skip-junk-key-spelling` event doc) and leave the journaled spelling
2668
- // authoritative for this machine.
2669
- if (
2670
- journalEntry === undefined &&
2671
- remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
2672
- // Gate NON-canonical remote spellings only. A remote key that already
2673
- // IS the canonical spelling must always be downloadable — after the
2674
- // doctor collapses the vault, a machine whose journal still holds only
2675
- // the junk spelling would otherwise skip the canonical object forever
2676
- // (and the delete-resync twin protection keeps that stale junk entry),
2677
- // deadlocking it out of every future update.
2678
- canonicalVaultKeySpelling(remoteFile.key) !== remoteFile.key
2679
- ) {
2680
- const journaledTwin = journaledKeyByCanonical.get(
2681
- canonicalVaultKeySpelling(remoteFile.key),
2682
- );
2683
- if (journaledTwin !== undefined && journaledTwin !== remoteFile.key) {
2684
- items.push({
2685
- action: "skip-junk-key-spelling",
2686
- remoteFile,
2687
- localPath,
2688
- journaledKey: journaledTwin,
2689
- });
2690
- continue;
2691
- }
2692
- }
2693
-
2694
- // ── FILE_TOMBSTONE consult (delete-resync) ───────────────────────────────
2695
- // A remote object present in the LIST may be an intentionally-deleted key
2696
- // (a peer re-pushed it, or its delete-marker hasn't propagated to this
2697
- // caller's view). `tombstoneSuppresses` is true when a FILE_TOMBSTONE marks
2698
- // this key deleted AND the remote object is NOT a newer re-create — i.e. the
2699
- // object is the stale/deleted version and must NOT be downloaded. A genuine
2700
- // re-create (object newer than the tombstone) leaves this false so the
2701
- // normal download/merge path runs (the tombstone is not permanent
2702
- // suppression). An empty tombstone map (the default / degraded-fetch case)
2703
- // makes this always false — legacy behavior preserved bit-for-bit.
2704
- const tombstone = fileTombstones.get(toPosixKey(remoteFile.key));
2705
- const tombstoneSuppresses =
2706
- tombstone !== undefined &&
2707
- !isRemoteRecreateAfterTombstone(remoteFile, tombstone);
2708
- // LOCAL journal tombstone — this machine's own intentional-delete record,
2709
- // independent of the (best-effort, intermittently-degraded) server
2710
- // FILE_TOMBSTONE fetch above. When a clean local delete is classified, the
2711
- // push leg stamps `removedAt` on the journal entry; honoring it here makes
2712
- // the delete-vs-drift-restore outcome DETERMINISTIC, closing the
2713
- // intermittent respawn where a degraded tombstone fetch let the pull leg
2714
- // re-materialize an intentionally-deleted file. Reuses the same re-create
2715
- // guard (a remote object newer than the local delete is a genuine re-create
2716
- // and still downloads). Consulted ONLY on the `!localExists` re-download
2717
- // path below — never to delete a file that is present locally.
2718
- const localTombstoneSuppresses =
2719
- journalEntry !== undefined &&
2720
- isTombstone(journalEntry) &&
2721
- hasCurrentLocalDeleteIntent(journalEntry) &&
2722
- journalEntry.removedAt !== undefined &&
2723
- !isRemoteRecreateAfterTombstone(remoteFile, {
2724
- deletedAt: journalEntry.removedAt,
2725
- });
2726
-
2727
- // lstat (not existsSync/statSync) handles three cases the legacy
2728
- // checks got wrong for symlinks:
2729
- // 1. A valid symlink at localPath: existsSync returns true and the
2730
- // planner falls through to hashFile, which follows the link and
2731
- // hashes the target file's bytes — never matching the journal's
2732
- // sha256(target string), so every pull would mis-classify a
2733
- // pristine symlink as locally changed.
2734
- // 2. A dangling symlink: existsSync returns false (existsSync
2735
- // follows links) so the planner treats the entry as missing and
2736
- // re-downloads, silently clobbering the user's intentional
2737
- // dangling link on every run.
2738
- // 3. A regular file: indistinguishable from current behaviour.
2739
- let localLstat: fs.Stats | null = null;
2740
- let plannedLocalSnapshot: LocalSnapshot = { kind: "absent" };
2741
- let localPathBlockedByFileAncestor = false;
2742
- try {
2743
- localLstat = fs.lstatSync(localPath);
2744
- } catch (err: unknown) {
2745
- const code =
2746
- err && typeof err === "object" && "code" in err
2747
- ? (err as { code?: string }).code
2748
- : undefined;
2749
- // ENOENT → truly absent → treated as "new file from cloud".
2750
- // ENOTDIR → an ancestor of localPath is a regular file (cloud has a
2751
- // dir at that key, local has a file). Pre-fix this threw
2752
- // and aborted the whole company sync (Bug #10 in the
2753
- // 5.33.0 deep-test verification — \`v4-dir-vs-file\` repro
2754
- // wedged the personal company at \"errored\" status,
2755
- // skipping every later file). Recover by classifying as
2756
- // a structural collision the operator must resolve
2757
- // manually, and let the rest of the company process.
2758
- if (code !== "ENOENT" && code !== "ENOTDIR") {
2759
- throw err;
2760
- }
2761
- if (code === "ENOTDIR") {
2762
- localPathBlockedByFileAncestor = true;
2763
- }
2764
- }
2765
- const localExists = localLstat !== null;
2766
-
2767
- if (localPathBlockedByFileAncestor) {
2768
- // Symmetric counterpart to the (local-dir, cloud-file) warning below.
2769
- // Emit the same "manual reconciliation required" surface so the
2770
- // operator sees one consistent message for both topologies; record as
2771
- // skip-local-only so the file is not silently dropped from the
2772
- // count and the executor never tries to lstat/write it.
2773
- console.error(
2774
- ` Warning: an ancestor of ${remoteFile.key} exists locally as a regular file; ` +
2775
- `cloud has a deeper key under that path. Skipping; manual ` +
2776
- `reconciliation required (rm the conflicting local file to pull).`,
2777
- );
2778
- items.push({ action: "skip-local-only", remoteFile, localPath });
2779
- continue;
2780
- }
2781
-
2782
- if (localExists) {
2783
- const isLocalSymlink = localLstat!.isSymbolicLink();
2784
- // Kind-mismatch guard: a remote LIST entry is always a single
2785
- // S3 object (regular file or symlink record), but the local
2786
- // path may be a real directory if the user previously had a
2787
- // dir at this key. Hashing a directory throws EISDIR and
2788
- // aborts the whole pull. We don't auto-replace because that
2789
- // would `rm -rf` the user's directory; surface as a non-fatal
2790
- // skip and warn so the operator can reconcile manually
2791
- // (delete the local dir, or restructure the remote).
2792
- if (localLstat!.isDirectory() && !isLocalSymlink) {
2793
- // Stale personal-overlay marker vs. release-shipped core directory:
2794
- // a single vault object at a `core/<overlay-type>/.../<entry>` key
2795
- // whose local counterpart is a real directory is an orphaned overlay
2796
- // symlink marker (reindex mirror artifact) left behind after a release
2797
- // shipped a real core dir of the same name. The local release-shipped
2798
- // directory is authoritative and the marker can never be materialized
2799
- // over it, so keep local and ignore the marker QUIETLY — the generic
2800
- // "rm -rf the local directory" advice below is actively dangerous here
2801
- // (it would destroy release-shipped core content to pull inert derived
2802
- // state), and it recurred on every sync. feedback_ce9aeede.
2803
- if (isGeneratedCoreMirrorKey(remoteFile.key)) {
2804
- console.error(
2805
- ` Note: ${remoteFile.key} is a stale personal-overlay marker ` +
2806
- `shadowing a release-shipped core directory; keeping the local ` +
2807
- `directory and ignoring the marker (no action needed).`,
2808
- );
2809
- items.push({
2810
- action: "skip-stale-overlay-marker",
2811
- remoteFile,
2812
- localPath,
2813
- });
2814
- continue;
2815
- }
2816
- // Directory-overlay marker vs. a locally-materialized directory:
2817
- // the remote object at this key carries CHILD objects under it in the
2818
- // same LIST (`remoteKeysWithChildren`), so it is a symlink/overlay
2819
- // record for a directory — e.g. the DPP vault's `skills/cody-copywriter`
2820
- // overlay whose 13 child objects are already materialized locally. The
2821
- // marker can never be renamed over the real directory, and its children
2822
- // carry the actual content and materialize via their own plan items, so
2823
- // the marker is a benign, no-action skip. The generic "rm -rf the local
2824
- // directory to pull" advice below would tell the operator to DELETE the
2825
- // materialized (and possibly locally-authored) contents to install an
2826
- // inert link, and it recurred on EVERY sync cycle. feedback_d2082110.
2827
- if (remoteKeysWithChildren.has(toPosixKey(remoteFile.key))) {
2828
- console.error(
2829
- ` Note: ${remoteFile.key} is a directory-overlay marker whose ` +
2830
- `contents already exist locally as a directory; keeping the ` +
2831
- `local directory and ignoring the marker (no action needed).`,
2832
- );
2833
- items.push({
2834
- action: "skip-overlay-marker-with-children",
2835
- remoteFile,
2836
- localPath,
2837
- });
2838
- continue;
2839
- }
2840
- console.error(
2841
- ` Warning: ${remoteFile.key} exists locally as a directory; ` +
2842
- `cloud has a single object at this key. Skipping; manual ` +
2843
- `reconciliation required (rm -rf the local directory to pull).`,
2844
- );
2845
- items.push({ action: "skip-local-only", remoteFile, localPath });
2846
- continue;
2847
- }
2848
- // Fast-path (5.36.0): when the journal entry has an mtimeMs and the
2849
- // local lstat matches both size and mtimeMs, the file hasn't been
2850
- // touched locally since the last sync, so we can reuse the journal's
2851
- // recorded hash without re-reading the file's bytes. Same shape as
2852
- // the share-side fast-path; only kicks in for regular files (symlink
2853
- // hashing through hashSymlinkTarget is already cheap — it hashes a
2854
- // short target string, not file bytes).
2855
- let localHash: string;
2856
- if (
2857
- !isLocalSymlink &&
2858
- journalEntry &&
2859
- journalEntry.mtimeMs !== undefined &&
2860
- journalEntry.hash &&
2861
- localLstat!.size === journalEntry.size &&
2862
- localLstat!.mtimeMs === journalEntry.mtimeMs
2863
- ) {
2864
- localHash = journalEntry.hash;
2865
- } else {
2866
- if (isLocalSymlink) {
2867
- const target = readlinkOrNull(localPath);
2868
- if (target === null) {
2869
- // We cannot prove whether the local link diverged, so never
2870
- // overwrite or tombstone it from this plan. A later pass can retry
2871
- // once the OS exposes a readable target.
2872
- items.push({ action: "skip-unreadable-link", remoteFile, localPath });
2873
- continue;
2874
- }
2875
- localHash = hashSymlinkTarget(target);
2876
- } else {
2877
- localHash = hashFile(localPath);
2878
- }
2879
- }
2880
- const localChanged = !!journalEntry && journalEntry.hash !== localHash;
2881
- plannedLocalSnapshot = {
2882
- kind: isLocalSymlink ? "symlink" : "file",
2883
- hash: localHash,
2884
- };
2885
- const remoteChanged =
2886
- !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
2887
-
2888
- // ── Authoritative delete reaches a machine that still holds the file ──
2889
- // A FILE_TOMBSTONE marks this key deleted and the remote object is not a
2890
- // newer re-create, but this machine still has a local copy. Honor the
2891
- // delete — EXCEPT never destroy unsynced local work: if the local copy
2892
- // diverges from the synced baseline (a post-tombstone edit, or an
2893
- // untracked local file with no journal entry), surface a CONFLICT instead
2894
- // of silently deleting (the safety guard the brief requires + the 3-way
2895
- // merge invariant). A local copy that still matches the deleted baseline
2896
- // is the stale version → delete it + drop the journal entry so it stays
2897
- // gone on this machine.
2898
- if (tombstoneSuppresses) {
2899
- if (localChanged || !journalEntry) {
2900
- items.push({
2901
- action: "conflict",
2902
- remoteFile,
2903
- localPath,
2904
- localHash,
2905
- localMtime: localLstat!.mtime,
2906
- localSize: isLocalSymlink ? 0 : localLstat!.size,
2907
- localSnapshot: plannedLocalSnapshot,
2908
- });
2909
- } else {
2910
- items.push({
2911
- action: "tombstone-delete",
2912
- remoteFile,
2913
- localPath,
2914
- localSnapshot: plannedLocalSnapshot,
2915
- });
2916
- }
2917
- continue;
2918
- }
2919
-
2920
- // ── Untracked-local guard (unjournaled local file) ──────────────
2921
- // Both `localChanged` and `remoteChanged` are gated on `!!journalEntry`,
2922
- // so a key with NO journal entry reads "clean on both sides" and falls
2923
- // all the way through to a plain `download` — silently replacing a local
2924
- // file that exists on disk. There is no conflict, no `.conflict-` mirror
2925
- // and no conflict-index record, so the loss is invisible unless someone
2926
- // happens to know what the file should contain.
2927
- //
2928
- // This destroyed in-progress edits to
2929
- // `companies/*/projects/*/prd.json` three times on 2026-08-01/02, but
2930
- // nothing about it is prd- or project-specific: it fires for ANY synced
2931
- // path whose journal entry is absent. Absence is common — a file created
2932
- // on this machine and on a peer under the same key before either pushed,
2933
- // a journal shard reset/lost/rebuilt (`readJournal` returns an empty
2934
- // `files` map for a missing shard, so EVERY divergent local file is in
2935
- // scope), a run under a different `HQ_STATE_DIR`, or an entry dropped by
2936
- // scope-shrink or rescue.
2937
- //
2938
- // No journal entry means no evidence that local and remote ever agreed,
2939
- // which is exactly when an overwrite is least safe. Route it through the
2940
- // conflict path instead: the executor's convergence probe fetches the
2941
- // remote once and, when the bytes already match (the ordinary
2942
- // fresh-machine / rebuilt-journal case), reconciles silently — no
2943
- // conflict, no mirror, journal stamped. Only genuine divergence surfaces,
2944
- // and there local is PRESERVED as-is with the cloud copy written beside
2945
- // it under `--on-conflict keep`, or the run halts under `abort`.
2946
- // Cloud-authoritative paths (board.json, ontology/, signals/, …) are
2947
- // unaffected — the conflict executor short-circuits those back to a
2948
- // pull-wins download before any mirror is minted.
2949
- if (!journalEntry) {
2950
- items.push({
2951
- action: "conflict",
2952
- remoteFile,
2953
- localPath,
2954
- localHash,
2955
- localMtime: localLstat!.mtime,
2956
- localSize: isLocalSymlink ? 0 : localLstat!.size,
2957
- localSnapshot: plannedLocalSnapshot,
2958
- });
2959
- continue;
2960
- }
2961
-
2962
- // Mirror the original 3-way merge from the inline loop. Tested by
2963
- // `does NOT flag a pull conflict when only local changed since last
2964
- // sync` and `detects conflicts with local changes…`.
2965
- if (localChanged && remoteChanged) {
2966
- items.push({
2967
- action: "conflict",
2968
- remoteFile,
2969
- localPath,
2970
- localHash,
2971
- // localLstat is non-null inside this `if (localExists)`
2972
- // branch — the planner already lstat'd the path. Carry
2973
- // its mtime forward so the executor doesn't have to
2974
- // re-stat (which would follow links and crash on a
2975
- // dangling symlink). Same goes for size: the conflict
2976
- // executor's keep/skip branch needs to stamp the
2977
- // journal with a size, and statSync on a dangling
2978
- // symlink throws ENOENT (silently swallowed pre-fix),
2979
- // leaving the journal stale and re-firing the conflict
2980
- // every sync.
2981
- localMtime: localLstat!.mtime,
2982
- localSize: isLocalSymlink ? 0 : localLstat!.size,
2983
- localSnapshot: plannedLocalSnapshot,
2984
- });
2985
- continue;
2986
- }
2987
- if (journalEntry && localChanged && !remoteChanged) {
2988
- items.push({ action: "skip-local-only", remoteFile, localPath });
2989
- continue;
2990
- }
2991
- if (journalEntry && !localChanged && !remoteChanged) {
2992
- // Flavor-health repair (win32 skill farm): a link journaled while its
2993
- // target was absent was minted file-flavor; once the target directory
2994
- // exists the hashes still match (hash = sha256(target)) so this
2995
- // branch would skip it forever, leaving the wrapper broken. Re-download
2996
- // instead — the materializer's flavor-aware path recreates it 'dir'.
2997
- if (
2998
- isLocalSymlink &&
2999
- remoteFile.key.startsWith(SKILLS_KEY_PREFIX) &&
3000
- win32SymlinkFlavorBroken(localPath)
3001
- ) {
3002
- items.push({
3003
- action: "download",
3004
- remoteFile,
3005
- localPath,
3006
- isNew: false,
3007
- localSnapshot: plannedLocalSnapshot,
3008
- });
3009
- continue;
3010
- }
3011
- items.push({ action: "skip-unchanged", remoteFile, localPath });
3012
- continue;
3013
- }
3014
- if (journalEntry && !localChanged && remoteChanged) {
3015
- // ── Anti-rollback guard (silent newer→older overwrite) ───────────
3016
- // Remote etag moved but local still matches the journal baseline, so
3017
- // the classic 3-way merge calls this a clean `download` and overwrites
3018
- // local with remote. That assumes the remote *advanced*. It can also
3019
- // REGRESS: a second device (or a stale re-push) can replace the S3
3020
- // object with an OLDER, SMALLER copy. `hasRemoteChanged` is pure-ETag
3021
- // and cannot tell an advance from a rollback — ListObjectsV2 carries no
3022
- // content hash, and `lastModified` is the upload time (newer even for
3023
- // stale content), so neither distinguishes the two. A silent download
3024
- // then DESTROYS newer local bytes with no conflict surfaced.
3025
- //
3026
- // This bit append-only bot chat logs on 2026-06-10, the day
3027
- // event-driven bidirectional sync (which lets a peer device push to
3028
- // S3 between this device's pulls) first rolled out: a regressed S3
3029
- // object truncated freshly-appended lines, undetected by the planner.
3030
- //
3031
- // Guard: if the incoming remote object is strictly SMALLER than the
3032
- // local file we already hold (a size regression on a tracked regular
3033
- // file), do NOT silently clobber. Route through the conflict path —
3034
- // the executor's convergence probe fetches the remote once and, on
3035
- // genuine divergence, PRESERVES local as a `.conflict-*` mirror instead
3036
- // of overwriting it (and under `--on-conflict abort` halts loudly
3037
- // rather than losing data). A byte-identical remote can never be
3038
- // smaller, so this never manufactures a false conflict on a no-op
3039
- // re-upload. Symlinks are exempt: their on-disk size is not the target
3040
- // length, so the comparison is meaningless.
3041
- //
3042
- // Direction gate: only guard copies this device AUTHORED (last synced
3043
- // UP). A consumer copy (last synced DOWN) has no local authorship to
3044
- // protect — the remote is authoritative, so a smaller remote is a
3045
- // legitimate update (e.g. the gardener rewriting a shorter brief), not a
3046
- // regression. Without this gate a consumer device floods `.conflict`
3047
- // mirrors and stalls on stale gardener output forever (the 786-file
3048
- // pile-up, 2026-06-22). `"up"` and any missing/unknown direction stay
3049
- // guarded exactly as before — the 2026-06-10 chat-log protection holds.
3050
- if (
3051
- journalEntry.direction !== "down" &&
3052
- !isLocalSymlink &&
3053
- remoteFile.size < localLstat!.size
3054
- ) {
3055
- items.push({
3056
- action: "conflict",
3057
- remoteFile,
3058
- localPath,
3059
- localHash,
3060
- localMtime: localLstat!.mtime,
3061
- localSize: localLstat!.size,
3062
- localSnapshot: plannedLocalSnapshot,
3063
- });
3064
- continue;
3065
- }
3066
- // Remote genuinely advanced (same-or-larger) → normal catch-up
3067
- // download below.
3068
- }
3069
- // No journal entry, or remote-only changed (non-shrinking) → fall
3070
- // through to download.
3071
- }
3072
-
3073
- // Suppress the re-download of an intentionally-deleted key (delete-resync).
3074
- // Reaches here only for the `!localExists` case — the `localExists` branch
3075
- // above already handled (and `continue`d) any suppressing tombstone, so a
3076
- // tombstone that survives to here means "remote present, local absent, not a
3077
- // re-create": the classic resurrection ("remote present → I'm behind →
3078
- // download") the planner must NOT do. Route to `tombstone-delete` so the
3079
- // executor drops any stale journal entry and the key stays gone, instead of
3080
- // pulling the deleted object back in. `localTombstoneSuppresses` adds the
3081
- // same protection driven by this machine's own journal tombstone, so an
3082
- // intentional delete is not resurrected even when the server FILE_TOMBSTONE
3083
- // fetch is degraded.
3084
- if (tombstoneSuppresses || localTombstoneSuppresses) {
3085
- // A persisting `local-delete` tombstone (stamped by the intentional-delete
3086
- // producer below) must KEEP its journal entry — the push leg still needs
3087
- // it to propagate the S3 delete. Skip the re-download without dropping the
3088
- // entry. Other tombstone kinds (scope-shrink / FILE_TOMBSTONE) drop it.
3089
- if (journalEntry?.removedReason === "local-delete") {
3090
- continue;
3091
- }
3092
- items.push({
3093
- action: "tombstone-delete",
3094
- remoteFile,
3095
- localPath,
3096
- localSnapshot: plannedLocalSnapshot,
3097
- });
3098
- continue;
3099
- }
3100
-
3101
- // Intentional-delete producer: a clean, CURRENT (etag matches), non-divergent
3102
- // file that is MISSING locally is the user deleting a genuinely-synced file.
3103
- // Collect it; the post-loop bulk-asymmetry decision tombstones it (intent) or
3104
- // restores it (mass loss = accident). A stale-etag, divergent, tombstoned, or
3105
- // never-journaled missing file falls through to the normal download below.
3106
- if (
3107
- !localExists &&
3108
- journalEntry !== undefined &&
3109
- !isTombstone(journalEntry) &&
3110
- hasCurrentLocalDeleteIntent(journalEntry) &&
3111
- normalizeEtag(remoteFile.etag) === journalEntry.remoteEtag
3112
- ) {
3113
- intentionalDeleteCandidates.push({ remoteFile, localPath });
3114
- continue;
3115
- }
3116
-
3117
- items.push({
3118
- action: "download",
3119
- remoteFile,
3120
- localPath,
3121
- isNew: !localExists,
3122
- localSnapshot: plannedLocalSnapshot,
3123
- });
3124
- }
3125
-
3126
- // ── Intentional-delete bulk-asymmetry decision (pull-leg producer) ─────────
3127
- // Below the breaker → stamp a persisting `local-delete` tombstone (the
3128
- // suppression above keeps it from being re-pulled; the push leg propagates the
3129
- // S3 delete on its next pass). At/above → drift-restore EVERY candidate: a
3130
- // mass disappearance (corrupt mirror / bulk rm / checkout / unmount) is
3131
- // accidental loss, never intent (the ridge-incident guard).
3132
- const totalJournalEntries = Object.keys(journal.files).length;
3133
- const intentionalDeleteIsBulk =
3134
- intentionalDeleteCandidates.length >= PULL_INTENTIONAL_DELETE_MIN_ABS &&
3135
- totalJournalEntries > 0 &&
3136
- intentionalDeleteCandidates.length / totalJournalEntries >=
3137
- PULL_INTENTIONAL_DELETE_RATIO;
3138
- for (const cand of intentionalDeleteCandidates) {
3139
- if (intentionalDeleteIsBulk) {
3140
- items.push({
3141
- action: "download",
3142
- remoteFile: cand.remoteFile,
3143
- localPath: cand.localPath,
3144
- isNew: true,
3145
- localSnapshot: { kind: "absent" },
3146
- });
3147
- } else {
3148
- tombstoneEntry(journal, cand.remoteFile.key, "local-delete");
3149
- }
3150
- }
3151
-
3152
- let filesToDownload = 0;
3153
- let bytesToDownload = 0;
3154
- let filesToSkip = 0;
3155
- let filesToConflict = 0;
3156
- let filesExcludedByPolicy = 0;
3157
- let filesOutOfScope = 0;
3158
- let filesToTombstoneDelete = 0;
3159
- const newFiles: Array<{ path: string; bytes: number }> = [];
3160
- for (const item of items) {
3161
- if (item.action === "download") {
3162
- filesToDownload++;
3163
- bytesToDownload += item.remoteFile.size;
3164
- if (item.isNew) {
3165
- newFiles.push({ path: item.remoteFile.key, bytes: item.remoteFile.size });
3166
- }
3167
- } else if (item.action === "conflict") {
3168
- filesToConflict++;
3169
- } else if (item.action === "tombstone-delete") {
3170
- // Authoritative FILE_TOMBSTONE delete — its own axis so it never inflates
3171
- // the "unchanged" tally. Surfaced via the plan event's filesToDelete.
3172
- filesToTombstoneDelete++;
3173
- } else if (item.action === "skip-excluded-policy") {
3174
- filesExcludedByPolicy++;
3175
- // Excluded-policy items don't roll into filesToSkip — they're a
3176
- // distinct class surfaced via filesExcludedByPolicy so consumers
3177
- // can render a "N refused by policy" line independently of the
3178
- // generic "N unchanged" tally.
3179
- } else if (item.action === "skip-out-of-scope") {
3180
- // Out-of-scope items get their own axis too, mirroring excluded-policy:
3181
- // they're "deliberately not downloaded (sync scope)", not "unchanged".
3182
- filesOutOfScope++;
3183
- } else {
3184
- filesToSkip++;
3185
- }
3186
- }
3187
-
3188
- // Bug #9 — cross-machine delete propagation. The peer's push leg
3189
- // already removed the file from S3 (verified in the deep-test
3190
- // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
3191
- // the pull side enumerated remote keys and downloaded anything
3192
- // missing locally, but never enumerated "what's missing-from-remote-
3193
- // that-was-there-before" — so the file stayed forever on every
3194
- // receiver and `filesTombstoned: 0` showed up on every pull.
3195
- //
3196
- // Closing the loop: walk the journal, find every entry whose key is
3197
- // NOT in the remote LIST set, AND whose path passes the current
3198
- // ignore filter (paths newly excluded by .hqignore must not trigger
3199
- // mass-delete), AND — in personalMode — survives the same companies/*
3200
- // gating the download branch applies. The executor will apply each
3201
- // as a local delete + journal removal. Symmetric to the push side's
3202
- // `propagateDeletes` plan in share.ts.
3203
- const remoteKeySet = new Set<string>();
3204
- // Skill-farm twin protection (companion to the junk-spelling pull guard):
3205
- // when the LIST holds only a junk spelling of a journaled canonical key,
3206
- // the logical object is still remotely present — a raw has() would class
3207
- // the canonical journal entry as remote-deleted, unlink the good local
3208
- // wrapper, and drop the journal entry (after which the next pull would
3209
- // materialize the junk spelling the guard just refused). Track canonical
3210
- // forms of listed skill keys and consult them below.
3211
- const remoteCanonicalSkillKeys = new Set<string>();
3212
- for (const rf of remoteFiles) {
3213
- remoteKeySet.add(rf.key);
3214
- if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
3215
- remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
3216
- }
3217
- }
3218
- const tombstones: PullPlan["tombstones"] = [];
3219
- for (const key of Object.keys(journal.files)) {
3220
- // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
3221
- // carry backslash separators; the remote LIST is always forward-slash, so a
3222
- // raw `has(key)` would miss the match and class a live local file as
3223
- // remote-deleted. normalizeJournalKeys() defuses this on load, but the
3224
- // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
3225
- const posixKey = toPosixKey(key);
3226
- if (remoteKeySet.has(posixKey)) continue;
3227
- if (
3228
- posixKey.startsWith(SKILLS_KEY_PREFIX) &&
3229
- remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
3230
- // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
3231
- // delete and wins over the twin protection — a lingering junk
3232
- // spelling in the LIST must not keep a deliberately-deleted wrapper
3233
- // alive forever. (The key is absent from the LIST here, so there is
3234
- // no newer re-create to defer to.)
3235
- fileTombstones.get(posixKey) === undefined
3236
- ) {
3237
- continue;
3238
- }
3239
- const localPath = resolveContainedVaultPath(companyRoot, key);
3240
- if (localPath === null) continue;
3241
- // Scope-invalid journal keys (company mode, `companies/…` prefix): the
3242
- // download branch skips them (presign rejects), but a journaled entry whose
3243
- // key is absent from the remote LIST is doubled-tree poison that must
3244
- // drain — local copy under companies/{slug}/companies/{slug}/… removed and
3245
- // journal dropped. HEAD verify is skipped in verifyPlannedJournalTombstones
3246
- // (presign would reject); see the auto-verified branch there.
3247
- const scopeInvalidJournalKey = isForbiddenCompanyVaultKey(key, personalMode);
3248
- // PersonalMode key gating — mirror the download branch. Local (non-cloud)
3249
- // company keys are tombstone-eligible (a peer's delete should propagate),
3250
- // but team-synced orphans are left alone (the team-bucket pull owns those
3251
- // disk paths), and the manifest — the routing source-of-truth — is never
3252
- // auto-tombstoned out of the personal vault.
3253
- if (personalMode && key.startsWith("companies/")) {
3254
- const slug = key.split("/")[1] ?? "";
3255
- const isTeamSyncedOrphan =
3256
- teamSyncedSlugs !== null && slug !== "" && teamSyncedSlugs.has(slug);
3257
- if (isTeamSyncedOrphan || key === PERSONAL_VAULT_MANIFEST_KEY) continue;
3258
- }
3259
- // Ephemeral keys are filtered both directions; never tombstone-
3260
- // propagate a conflict-mirror.
3261
- if (isEphemeralPath(key)) continue;
3262
- // Honor the current ignore filter — if a path was previously synced
3263
- // but is now ignored (operator edited .hqignore), do NOT delete
3264
- // the local copy. They're keeping it deliberately. Scope-invalid poison
3265
- // bypasses this: the doubled-tree copy must not survive silently.
3266
- if (
3267
- !scopeInvalidJournalKey &&
3268
- !shouldSync(localPath, false) &&
3269
- !shouldSync(localPath, true)
3270
- ) {
3271
- continue;
3272
- }
3273
- // Codex P1 (PR #24 round 3): detect local edits before tombstoning.
3274
- // Delete-vs-local-edit race: peer deleted the file remotely while
3275
- // this machine edited it locally before the next sync. Without
3276
- // this check, the tombstone executor would unlink the locally-
3277
- // edited file and drop the journal entry, silently destroying the
3278
- // user's unsynced work. Compare local hash against the journal
3279
- // baseline; if they diverge, defer the tombstone. The next pull
3280
- // run after the operator resolves (re-pushes / overwrites) the
3281
- // local state will re-evaluate.
3282
- try {
3283
- const lstat = fs.lstatSync(localPath);
3284
- if (lstat.isFile()) {
3285
- const localHash = hashFile(localPath);
3286
- const journalEntry = journal.files[key];
3287
- if (journalEntry?.hash && journalEntry.hash !== localHash) {
3288
- // Local has unsynced edits — defer this tombstone.
3289
- continue;
3290
- }
3291
- } else if (lstat.isSymbolicLink()) {
3292
- // Codex P1 (PR #24 round 4): symlinks have target strings
3293
- // that can be locally edited too. Pre-fix, `isFile()` was
3294
- // false for symlinks so the divergence guard skipped them,
3295
- // and a locally-edited symlink (`ln -sfn new-target old-link`
3296
- // before the peer's remote-delete) was unlinked silently —
3297
- // the exact race the guard is meant to prevent. Compare the
3298
- // readlink hash against the journal baseline; defer on
3299
- // divergence. readlink errors (race / EACCES) also defer.
3300
- try {
3301
- const localHash = hashSymlinkTarget(fs.readlinkSync(localPath));
3302
- const journalEntry = journal.files[key];
3303
- if (journalEntry?.hash && journalEntry.hash !== localHash) {
3304
- continue;
3305
- }
3306
- } catch {
3307
- continue;
3308
- }
3309
- }
3310
- // Directories: no hash comparison. The tombstone executor's
3311
- // dir branch doesn't recursively rm-rf — it just drops the
3312
- // journal entry, which is the safe-by-default behavior.
3313
- } catch (err) {
3314
- const code =
3315
- err && typeof err === "object" && "code" in err
3316
- ? (err as { code?: string }).code
3317
- : undefined;
3318
- // ENOENT → local already gone; safe to drop journal entry via
3319
- // the executor's tombstone path.
3320
- // Other lstat errors (EACCES on parent dir, etc.) → defer:
3321
- // we can't read local state, so we can't safely decide.
3322
- if (code !== "ENOENT") continue;
3323
- }
3324
- const localSnapshot = snapshotLocalState(localPath);
3325
- if (localSnapshot === null) continue;
3326
- tombstones.push({ key, localPath, localSnapshot });
3327
- }
3328
-
3329
- return {
3330
- items,
3331
- filesToDownload,
3332
- bytesToDownload,
3333
- filesToSkip,
3334
- filesToConflict,
3335
- newFiles,
3336
- newFilesCount: newFiles.length,
3337
- filesExcludedByPolicy,
3338
- filesOutOfScope,
3339
- filesToTombstoneDelete,
3340
- tombstones,
3341
- invalidRemoteKeys,
3342
- };
3343
- }
3344
-
3345
- /**
3346
- * Default human-readable event rendering. Preserves the exact output format
3347
- * that `hq sync` emitted before SyncProgressEvent was introduced, so callers
3348
- * without an `onEvent` see no behavioral change.
3349
- */
3350
- function defaultConsoleLogger(event: SyncProgressEvent): void {
3351
- if (event.type === "plan") {
3352
- // Terse single line so humans see what's about to happen without
3353
- // drowning the per-file output that follows. Skip when there's
3354
- // nothing to do — no signal, no noise.
3355
- const movement = event.filesToDownload + event.filesToUpload + event.filesToConflict;
3356
- if (movement > 0) {
3357
- const parts: string[] = [];
3358
- if (event.filesToDownload > 0) {
3359
- parts.push(`${event.filesToDownload} to download (${event.bytesToDownload} bytes)`);
3360
- }
3361
- if (event.filesToUpload > 0) {
3362
- parts.push(`${event.filesToUpload} to upload (${event.bytesToUpload} bytes)`);
3363
- }
3364
- if (event.filesToConflict > 0) {
3365
- parts.push(`${event.filesToConflict} conflict(s)`);
3366
- }
3367
- parts.push(`${event.filesToSkip} unchanged`);
3368
- console.log(`Plan: ${parts.join(", ")}`);
3369
- }
3370
- } else if (event.type === "progress") {
3371
- if (event.message) {
3372
- console.log(` ✓ ${event.path} — "${event.message}"`);
3373
- } else {
3374
- console.log(` ✓ ${event.path}`);
3375
- }
3376
- } else if (event.type === "error") {
3377
- console.error(` ✗ ${event.path} — ${event.message}`);
3378
- } else if (event.type === "conflict") {
3379
- console.error(
3380
- ` ⚠ conflict (${event.direction}): ${event.path} — ${event.resolution}`,
3381
- );
3382
- } else if (event.type === "reconciled") {
3383
- console.log(
3384
- ` = reconciled (${event.direction}): ${event.path} — remote bytes identical to local, journal re-stamped`,
3385
- );
3386
- } else if (event.type === "new-files") {
3387
- if (event.files.length > 0) {
3388
- console.log(`${event.files.length} new file${event.files.length === 1 ? "" : "s"}`);
3389
- const MAX_SHOWN = 20;
3390
- const shown = event.files.slice(0, MAX_SHOWN);
3391
- for (const f of shown) {
3392
- const who = f.addedBy ? ` (added by ${f.addedBy})` : "";
3393
- console.log(` + ${f.path}${who}`);
3394
- }
3395
- if (event.files.length > MAX_SHOWN) {
3396
- console.log(` ... and ${event.files.length - MAX_SHOWN} more`);
3397
- }
3398
- }
3399
- } else if (event.type === "scope-excluded") {
3400
- console.log(
3401
- ` ~ ${event.count} path${event.count === 1 ? "" : "s"} skipped — outside your granted access (synced the in-scope subset):`,
3402
- );
3403
- for (const p of event.samplePaths) {
3404
- console.log(` · ${p}`);
3405
- }
3406
- if (event.count > event.samplePaths.length) {
3407
- console.log(` ... and ${event.count - event.samplePaths.length} more`);
3408
- }
3409
- } else if (event.type === "scope-materialization-gap") {
3410
- console.warn(
3411
- ` ! ${event.count} item${event.count === 1 ? "" : "s"} in this vault did NOT sync to this device — your access level is "${event.syncMode}", so only shared paths materialize. To pull everything you have access to, set this membership's access level to "all":`,
3412
- );
3413
- for (const p of event.samplePaths) {
3414
- console.warn(` · ${p}`);
3415
- }
3416
- if (event.count > event.samplePaths.length) {
3417
- console.warn(` ... and ${event.count - event.samplePaths.length} more`);
3418
- }
3419
- } else if (event.type === "ignore-excluded") {
3420
- console.warn(
3421
- ` ! ${event.count} path${event.count === 1 ? "" : "s"} were EXCLUDED from sync by an ignore rule and did NOT reach the vault (review in case this is unintended):`,
3422
- );
3423
- for (const p of event.samplePaths) {
3424
- console.warn(` · ${p}`);
3425
- }
3426
- if (event.count > event.samplePaths.length) {
3427
- console.warn(` ... and ${event.count - event.samplePaths.length} more`);
3428
- }
3429
- } else if (event.type === "skip-size-limit") {
3430
- const mb = (event.bytes / (1024 * 1024)).toFixed(1);
3431
- console.warn(
3432
- ` ! ${event.path} skipped — ${mb} MB exceeds the 50 MB sync size limit`,
3433
- );
3434
- } else if (event.type === "skip-invalid-scoped-key") {
3435
- console.warn(
3436
- ` ! ${event.path} skipped — invalid vault key (${event.errorCode ?? "INVALID_KEY_COMPANIES_SCOPED"})`,
3437
- );
3438
- } else if (event.type === "skip-junk-key-spelling") {
3439
- console.warn(
3440
- ` ! ${event.path} skipped — junk key spelling; this logical path is already synced as ${event.journaledKey} (mixed-version peer artifact; the vault doctor collapses the family)`,
3441
- );
3442
- } else if (event.type === "not-shipped") {
3443
- if (event.reason === "unreachable-path") {
3444
- console.warn(
3445
- ` ! ${event.count} named path${event.count === 1 ? "" : "s"} could NOT be pushed — the file exists but is not reachable under the company folder (nothing was uploaded for ${event.count === 1 ? "it" : "them"}):`,
3446
- );
3447
- } else if (event.reason === "unreadable-link") {
3448
- console.warn(
3449
- ` ! ${event.count} symbolic link${event.count === 1 ? "" : "s"} could NOT be read and was skipped without dereferencing its target:`,
3450
- );
3451
- } else {
3452
- console.warn(
3453
- ` ! ${event.count} linked subtree${event.count === 1 ? "" : "s"} recorded but NOT uploaded — contents sync via their own repo, not the vault:`,
3454
- );
3455
- }
3456
- for (const p of event.samplePaths) {
3457
- console.warn(` · ${p}`);
3458
- }
3459
- if (event.count > event.samplePaths.length) {
3460
- console.warn(` ... and ${event.count - event.samplePaths.length} more`);
3461
- }
3462
- if (event.reason === "linked-subtree") {
3463
- console.warn(
3464
- ` To put a file from a linked subtree in the vault, push it by name:\n` +
3465
- ` hq sync push <path> --company <slug>\n` +
3466
- ` That uploads a point-in-time copy; re-run it after each edit (regular sync will not track or update it).`,
3467
- );
3468
- }
3469
- }
3470
- }