@indigoai-us/hq-cloud 6.15.0 → 6.15.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (329) hide show
  1. package/dist/bin/sync-mutation.d.ts +16 -0
  2. package/dist/bin/sync-mutation.d.ts.map +1 -0
  3. package/dist/bin/sync-mutation.js +60 -0
  4. package/dist/bin/sync-mutation.js.map +1 -0
  5. package/dist/bin/sync-mutation.test.d.ts +2 -0
  6. package/dist/bin/sync-mutation.test.d.ts.map +1 -0
  7. package/dist/bin/sync-mutation.test.js +165 -0
  8. package/dist/bin/sync-mutation.test.js.map +1 -0
  9. package/dist/bin/sync-runner-company.d.ts +8 -0
  10. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  11. package/dist/bin/sync-runner-company.js +16 -0
  12. package/dist/bin/sync-runner-company.js.map +1 -1
  13. package/dist/bin/sync-runner-company.test.d.ts +2 -0
  14. package/dist/bin/sync-runner-company.test.d.ts.map +1 -0
  15. package/dist/bin/sync-runner-company.test.js +36 -0
  16. package/dist/bin/sync-runner-company.test.js.map +1 -0
  17. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  18. package/dist/bin/sync-runner-watch-loop.js +98 -8
  19. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  20. package/dist/bin/sync-runner.d.ts +17 -0
  21. package/dist/bin/sync-runner.d.ts.map +1 -1
  22. package/dist/bin/sync-runner.js.map +1 -1
  23. package/dist/bin/sync-runner.test.js +109 -0
  24. package/dist/bin/sync-runner.test.js.map +1 -1
  25. package/dist/cli/conflict-recovery.test.d.ts +2 -0
  26. package/dist/cli/conflict-recovery.test.d.ts.map +1 -0
  27. package/dist/cli/conflict-recovery.test.js +201 -0
  28. package/dist/cli/conflict-recovery.test.js.map +1 -0
  29. package/dist/cli/conflict.d.ts +60 -0
  30. package/dist/cli/conflict.d.ts.map +1 -1
  31. package/dist/cli/conflict.js +333 -0
  32. package/dist/cli/conflict.js.map +1 -1
  33. package/dist/cli/sync.d.ts +35 -0
  34. package/dist/cli/sync.d.ts.map +1 -1
  35. package/dist/cli/sync.js +100 -0
  36. package/dist/cli/sync.js.map +1 -1
  37. package/dist/cli/sync.test.js +85 -1
  38. package/dist/cli/sync.test.js.map +1 -1
  39. package/dist/index.d.ts +2 -0
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +1 -0
  42. package/dist/index.js.map +1 -1
  43. package/dist/skill-telemetry.d.ts +6 -0
  44. package/dist/skill-telemetry.d.ts.map +1 -1
  45. package/dist/skill-telemetry.js +14 -2
  46. package/dist/skill-telemetry.js.map +1 -1
  47. package/dist/skill-telemetry.test.js +79 -0
  48. package/dist/skill-telemetry.test.js.map +1 -1
  49. package/dist/sync/candidate-uploader.d.ts +88 -0
  50. package/dist/sync/candidate-uploader.d.ts.map +1 -0
  51. package/dist/sync/candidate-uploader.js +212 -0
  52. package/dist/sync/candidate-uploader.js.map +1 -0
  53. package/dist/sync/candidate-uploader.test.d.ts +2 -0
  54. package/dist/sync/candidate-uploader.test.d.ts.map +1 -0
  55. package/dist/sync/candidate-uploader.test.js +132 -0
  56. package/dist/sync/candidate-uploader.test.js.map +1 -0
  57. package/dist/sync/delta-client.d.ts +73 -0
  58. package/dist/sync/delta-client.d.ts.map +1 -0
  59. package/dist/sync/delta-client.js +201 -0
  60. package/dist/sync/delta-client.js.map +1 -0
  61. package/dist/sync/delta-client.test.d.ts +2 -0
  62. package/dist/sync/delta-client.test.d.ts.map +1 -0
  63. package/dist/sync/delta-client.test.js +97 -0
  64. package/dist/sync/delta-client.test.js.map +1 -0
  65. package/dist/sync/durable-apply.d.ts +76 -0
  66. package/dist/sync/durable-apply.d.ts.map +1 -0
  67. package/dist/sync/durable-apply.js +530 -0
  68. package/dist/sync/durable-apply.js.map +1 -0
  69. package/dist/sync/durable-apply.test.d.ts +2 -0
  70. package/dist/sync/durable-apply.test.d.ts.map +1 -0
  71. package/dist/sync/durable-apply.test.js +180 -0
  72. package/dist/sync/durable-apply.test.js.map +1 -0
  73. package/dist/sync/event-sync.d.ts +33 -1
  74. package/dist/sync/event-sync.d.ts.map +1 -1
  75. package/dist/sync/event-sync.js +149 -1
  76. package/dist/sync/event-sync.js.map +1 -1
  77. package/dist/sync/event-sync.test.js +142 -1
  78. package/dist/sync/event-sync.test.js.map +1 -1
  79. package/dist/sync/index.d.ts +2 -0
  80. package/dist/sync/index.d.ts.map +1 -1
  81. package/dist/sync/index.js +1 -0
  82. package/dist/sync/index.js.map +1 -1
  83. package/dist/sync/multipart-uploader.d.ts +99 -0
  84. package/dist/sync/multipart-uploader.d.ts.map +1 -0
  85. package/dist/sync/multipart-uploader.js +447 -0
  86. package/dist/sync/multipart-uploader.js.map +1 -0
  87. package/dist/sync/multipart-uploader.test.d.ts +2 -0
  88. package/dist/sync/multipart-uploader.test.d.ts.map +1 -0
  89. package/dist/sync/multipart-uploader.test.js +119 -0
  90. package/dist/sync/multipart-uploader.test.js.map +1 -0
  91. package/dist/sync/mutation-client.d.ts +85 -0
  92. package/dist/sync/mutation-client.d.ts.map +1 -0
  93. package/dist/sync/mutation-client.js +245 -0
  94. package/dist/sync/mutation-client.js.map +1 -0
  95. package/dist/sync/mutation-client.test.d.ts +2 -0
  96. package/dist/sync/mutation-client.test.d.ts.map +1 -0
  97. package/dist/sync/mutation-client.test.js +51 -0
  98. package/dist/sync/mutation-client.test.js.map +1 -0
  99. package/dist/sync/push-receiver.d.ts +45 -0
  100. package/dist/sync/push-receiver.d.ts.map +1 -1
  101. package/dist/sync/push-receiver.js +101 -0
  102. package/dist/sync/push-receiver.js.map +1 -1
  103. package/dist/sync/push-receiver.test.js +54 -2
  104. package/dist/sync/push-receiver.test.js.map +1 -1
  105. package/dist/sync/scope-inventory-client.d.ts +69 -0
  106. package/dist/sync/scope-inventory-client.d.ts.map +1 -0
  107. package/dist/sync/scope-inventory-client.js +210 -0
  108. package/dist/sync/scope-inventory-client.js.map +1 -0
  109. package/dist/sync/scope-inventory-client.test.d.ts +2 -0
  110. package/dist/sync/scope-inventory-client.test.d.ts.map +1 -0
  111. package/dist/sync/scope-inventory-client.test.js +94 -0
  112. package/dist/sync/scope-inventory-client.test.js.map +1 -0
  113. package/dist/sync/snapshot-client.d.ts +98 -0
  114. package/dist/sync/snapshot-client.d.ts.map +1 -0
  115. package/dist/sync/snapshot-client.js +402 -0
  116. package/dist/sync/snapshot-client.js.map +1 -0
  117. package/dist/sync/snapshot-client.test.d.ts +2 -0
  118. package/dist/sync/snapshot-client.test.d.ts.map +1 -0
  119. package/dist/sync/snapshot-client.test.js +169 -0
  120. package/dist/sync/snapshot-client.test.js.map +1 -0
  121. package/dist/sync/uploader-finalization.d.ts +97 -0
  122. package/dist/sync/uploader-finalization.d.ts.map +1 -0
  123. package/dist/sync/uploader-finalization.js +273 -0
  124. package/dist/sync/uploader-finalization.js.map +1 -0
  125. package/dist/sync/uploader-finalization.test.d.ts +2 -0
  126. package/dist/sync/uploader-finalization.test.d.ts.map +1 -0
  127. package/dist/sync/uploader-finalization.test.js +92 -0
  128. package/dist/sync/uploader-finalization.test.js.map +1 -0
  129. package/dist/telemetry.d.ts +11 -1
  130. package/dist/telemetry.d.ts.map +1 -1
  131. package/dist/telemetry.js +21 -2
  132. package/dist/telemetry.js.map +1 -1
  133. package/dist/telemetry.test.js +80 -0
  134. package/dist/telemetry.test.js.map +1 -1
  135. package/package.json +6 -1
  136. package/.claude/policies/hq-cloud-esm-cannot-spy-fs-builtins.md +0 -30
  137. package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +0 -22
  138. package/.github/workflows/ci.yml +0 -84
  139. package/.github/workflows/publish.yml +0 -56
  140. package/.github/workflows/unreleased-commits-nag.yml +0 -256
  141. package/eslint.config.js +0 -67
  142. package/pnpm-workspace.yaml +0 -2
  143. package/scripts/presign-transport-e2e.mjs +0 -250
  144. package/scripts/vault-rebaseline.sh +0 -323
  145. package/scripts/vault-rescue.sh +0 -332
  146. package/src/active-company.test.ts +0 -188
  147. package/src/active-company.ts +0 -168
  148. package/src/agent-codex-instructions.test.ts +0 -332
  149. package/src/agent-codex-instructions.ts +0 -309
  150. package/src/auth.ts +0 -146
  151. package/src/backup-prune.test.ts +0 -98
  152. package/src/backup-prune.ts +0 -182
  153. package/src/bin/backup-prune-runner.ts +0 -33
  154. package/src/bin/rescue-runner.ts +0 -25
  155. package/src/bin/sync-runner-company.ts +0 -695
  156. package/src/bin/sync-runner-events.test.ts +0 -143
  157. package/src/bin/sync-runner-events.ts +0 -55
  158. package/src/bin/sync-runner-planning.test.ts +0 -311
  159. package/src/bin/sync-runner-planning.ts +0 -258
  160. package/src/bin/sync-runner-rollup.test.ts +0 -37
  161. package/src/bin/sync-runner-rollup.ts +0 -97
  162. package/src/bin/sync-runner-telemetry.ts +0 -15
  163. package/src/bin/sync-runner-watch-loop.ts +0 -1235
  164. package/src/bin/sync-runner-watch-routes.test.ts +0 -71
  165. package/src/bin/sync-runner-watch-routes.ts +0 -184
  166. package/src/bin/sync-runner.test.ts +0 -8767
  167. package/src/bin/sync-runner.ts +0 -2190
  168. package/src/cli/accept.ts +0 -124
  169. package/src/cli/conflict.ts +0 -119
  170. package/src/cli/doctor.test.ts +0 -581
  171. package/src/cli/doctor.ts +0 -642
  172. package/src/cli/index.ts +0 -49
  173. package/src/cli/invite.test.ts +0 -250
  174. package/src/cli/invite.ts +0 -214
  175. package/src/cli/promote.ts +0 -157
  176. package/src/cli/reindex-knowledge.test.ts +0 -307
  177. package/src/cli/reindex-knowledge.ts +0 -450
  178. package/src/cli/reindex.test.ts +0 -957
  179. package/src/cli/reindex.ts +0 -979
  180. package/src/cli/rescue-classify-ordering.test.ts +0 -548
  181. package/src/cli/rescue-clone-diagnostics.test.ts +0 -120
  182. package/src/cli/rescue-core.ts +0 -3011
  183. package/src/cli/rescue-drift-reconcile.test.ts +0 -179
  184. package/src/cli/rescue-drop-dir-symlink.test.ts +0 -224
  185. package/src/cli/rescue-exec-bit-preserve.test.ts +0 -187
  186. package/src/cli/rescue-hq-root-guard.test.ts +0 -232
  187. package/src/cli/rescue-journal-reconcile.test.ts +0 -215
  188. package/src/cli/rescue-mtime-preserve.test.ts +0 -203
  189. package/src/cli/rescue-settings-reconcile.test.ts +0 -637
  190. package/src/cli/rescue-snapshot.test.ts +0 -57
  191. package/src/cli/rescue-snapshot.ts +0 -51
  192. package/src/cli/rescue.reindex.test.ts +0 -63
  193. package/src/cli/rescue.test.ts +0 -131
  194. package/src/cli/rescue.ts +0 -182
  195. package/src/cli/share.test.ts +0 -7843
  196. package/src/cli/share.ts +0 -3663
  197. package/src/cli/sync-scope.test.ts +0 -652
  198. package/src/cli/sync.test.ts +0 -5207
  199. package/src/cli/sync.ts +0 -3470
  200. package/src/cli/tombstones.ts +0 -106
  201. package/src/cli/watch-event-push-conflict.test.ts +0 -234
  202. package/src/client-info.test.ts +0 -214
  203. package/src/client-info.ts +0 -121
  204. package/src/cognito-auth.test.ts +0 -712
  205. package/src/cognito-auth.ts +0 -1422
  206. package/src/company-resolver.test.ts +0 -618
  207. package/src/company-resolver.ts +0 -521
  208. package/src/context.test.ts +0 -583
  209. package/src/context.ts +0 -378
  210. package/src/daemon-worker.ts +0 -26
  211. package/src/daemon.ts +0 -99
  212. package/src/entity-resolver.test.ts +0 -315
  213. package/src/entity-resolver.ts +0 -180
  214. package/src/ignore.test.ts +0 -466
  215. package/src/ignore.ts +0 -469
  216. package/src/index.ts +0 -439
  217. package/src/journal.test.ts +0 -968
  218. package/src/journal.ts +0 -765
  219. package/src/lib/cloud-authoritative.test.ts +0 -45
  220. package/src/lib/cloud-authoritative.ts +0 -59
  221. package/src/lib/conflict-file.ts +0 -86
  222. package/src/lib/conflict-index.ts +0 -289
  223. package/src/lib/conflict.test.ts +0 -348
  224. package/src/lib/describe-error.test.ts +0 -100
  225. package/src/lib/describe-error.ts +0 -58
  226. package/src/lib/exit-codes.ts +0 -24
  227. package/src/lib/machine-id.test.ts +0 -231
  228. package/src/lib/machine-id.ts +0 -175
  229. package/src/lib/net-errors.test.ts +0 -65
  230. package/src/lib/net-errors.ts +0 -86
  231. package/src/lib/readlink-safe.test.ts +0 -43
  232. package/src/lib/readlink-safe.ts +0 -29
  233. package/src/local-path-codec.test.ts +0 -138
  234. package/src/local-path-codec.ts +0 -161
  235. package/src/machine-auth.test.ts +0 -1323
  236. package/src/manifest-reconcile.test.ts +0 -1123
  237. package/src/manifest-reconcile.ts +0 -518
  238. package/src/object-io.test.ts +0 -1221
  239. package/src/object-io.ts +0 -1306
  240. package/src/operation-lock.test.ts +0 -484
  241. package/src/operation-lock.ts +0 -680
  242. package/src/outcome-telemetry.test.ts +0 -498
  243. package/src/outcome-telemetry.ts +0 -639
  244. package/src/personal-vault-exclusions.test.ts +0 -308
  245. package/src/personal-vault-exclusions.ts +0 -354
  246. package/src/personal-vault.test.ts +0 -756
  247. package/src/personal-vault.ts +0 -496
  248. package/src/prefix-coalesce.test.ts +0 -240
  249. package/src/prefix-coalesce.ts +0 -273
  250. package/src/public-surface.test.ts +0 -117
  251. package/src/qmd-reindex.test.ts +0 -877
  252. package/src/qmd-reindex.ts +0 -842
  253. package/src/read-only-state-dir.test.ts +0 -188
  254. package/src/remote-pull.test.ts +0 -1130
  255. package/src/remote-pull.ts +0 -618
  256. package/src/s3.symlink-materialize.test.ts +0 -492
  257. package/src/s3.test.ts +0 -1789
  258. package/src/s3.ts +0 -1532
  259. package/src/schemas/signal-types.test.ts +0 -82
  260. package/src/schemas/signal-types.ts +0 -38
  261. package/src/schemas/source-channels.test.ts +0 -82
  262. package/src/schemas/source-channels.ts +0 -53
  263. package/src/scope-shrink.test.ts +0 -633
  264. package/src/scope-shrink.ts +0 -481
  265. package/src/signals/get.test.ts +0 -310
  266. package/src/signals/get.ts +0 -75
  267. package/src/signals/internals.ts +0 -195
  268. package/src/signals/list.test.ts +0 -420
  269. package/src/signals/list.ts +0 -79
  270. package/src/signals/parse.ts +0 -8
  271. package/src/signals/types.ts +0 -91
  272. package/src/skill-telemetry.test.ts +0 -1825
  273. package/src/skill-telemetry.ts +0 -1439
  274. package/src/sources/get.test.ts +0 -293
  275. package/src/sources/get.ts +0 -66
  276. package/src/sources/internals.ts +0 -198
  277. package/src/sources/list.test.ts +0 -402
  278. package/src/sources/list.ts +0 -84
  279. package/src/sources/parse.ts +0 -43
  280. package/src/sources/types.ts +0 -84
  281. package/src/sync/event-sync.test.ts +0 -594
  282. package/src/sync/event-sync.ts +0 -545
  283. package/src/sync/feature-flags.test.ts +0 -378
  284. package/src/sync/feature-flags.ts +0 -62
  285. package/src/sync/index.ts +0 -76
  286. package/src/sync/lease-client.test.ts +0 -128
  287. package/src/sync/lease-client.ts +0 -207
  288. package/src/sync/logger.test.ts +0 -242
  289. package/src/sync/logger.ts +0 -79
  290. package/src/sync/metrics.test.ts +0 -462
  291. package/src/sync/metrics.ts +0 -213
  292. package/src/sync/pull-scope.ts +0 -265
  293. package/src/sync/push-event.test.ts +0 -266
  294. package/src/sync/push-event.ts +0 -224
  295. package/src/sync/push-receiver.test.ts +0 -566
  296. package/src/sync/push-receiver.ts +0 -1048
  297. package/src/sync/push-transport.ts +0 -231
  298. package/src/sync/realtime-rollout.test.ts +0 -86
  299. package/src/sync/realtime-rollout.ts +0 -262
  300. package/src/sync/state-store.test.ts +0 -194
  301. package/src/sync/state-store.ts +0 -727
  302. package/src/sync-core.ts +0 -58
  303. package/src/sync-progress.test.ts +0 -94
  304. package/src/sync-progress.ts +0 -140
  305. package/src/telemetry-events.test.ts +0 -88
  306. package/src/telemetry-events.ts +0 -205
  307. package/src/telemetry.test.ts +0 -1280
  308. package/src/telemetry.ts +0 -1109
  309. package/src/types.ts +0 -314
  310. package/src/vault-client.test.ts +0 -1380
  311. package/src/vault-client.ts +0 -1694
  312. package/src/version.ts +0 -24
  313. package/src/watch-roots.test.ts +0 -278
  314. package/src/watch-roots.ts +0 -162
  315. package/src/watcher-event-gate.test.ts +0 -212
  316. package/src/watcher.test.ts +0 -1079
  317. package/src/watcher.ts +0 -1741
  318. package/test/e2e/sync/cross-tenant-isolation.test.ts +0 -630
  319. package/test/e2e/sync/skill-telemetry-oversized-transcript.test.ts +0 -124
  320. package/test/e2e/sync/transient-company-leg.test.ts +0 -384
  321. package/test/e2e/sync/windows-unreadable-link-leg.test.ts +0 -191
  322. package/test/e2e/watcher-real-chokidar.test.ts +0 -165
  323. package/test/e2e/watcher-recursive-backend.test.ts +0 -181
  324. package/test/e2e/watcher-scoped-coverage.test.ts +0 -381
  325. package/test/invite-flow.integration.test.ts +0 -244
  326. package/test/joiner-manifest-reconcile.integration.test.ts +0 -322
  327. package/test/share-sync.integration.test.ts +0 -213
  328. package/tsconfig.json +0 -19
  329. package/vitest.config.ts +0 -22
package/src/s3.ts DELETED
@@ -1,1532 +0,0 @@
1
- /**
2
- * S3 operations — upload, download, list, delete.
3
- *
4
- * VLT-5: All operations now accept an EntityContext (entity-aware bucket +
5
- * STS-scoped credentials) instead of reading static env config. The caller
6
- * is responsible for resolving the context via resolveEntityContext().
7
- */
8
-
9
- import * as fs from "fs";
10
- import * as path from "path";
11
- import * as crypto from "crypto";
12
- import { Readable, Transform } from "stream";
13
- import { pipeline } from "stream/promises";
14
- import type { EntityContext } from "./types.js";
15
- import {
16
- resolveObjectIO,
17
- type GetObjectStreamResult,
18
- type ObjectIO,
19
- type PutPrecondition,
20
- } from "./object-io.js";
21
-
22
- // Byte/metadata transport is resolved per-call via resolveObjectIO(ctx) — the
23
- // default is the AWS S3 SDK over STS-vended credentials (S3SdkObjectIO), but a
24
- // session may select the presigned-URL transport (PresignObjectIO) via
25
- // setObjectIOFactory. The symlink/mode/mtime/created-at semantics below are
26
- // transport-agnostic: they compose on top of the ObjectIO primitives.
27
-
28
- /**
29
- * Author identity stamped onto S3 user-defined metadata at upload time. The
30
- * vault UI's "CREATED BY" column reads `Metadata['created-by']` back via
31
- * HEAD; uploads without an author leave that column blank.
32
- */
33
- export interface UploadAuthor {
34
- /** Cognito sub — stable join key for per-member rollups. */
35
- userSub: string;
36
- /** Email for human display. */
37
- email: string;
38
- }
39
-
40
- /**
41
- * S3 user metadata is ASCII-only (lowercased on read, capped at 2 KB total).
42
- * Values that fail the printable-ASCII test or would push the keys over the
43
- * cap are elided rather than throwing — partial attribution beats none. The
44
- * shape mirrors `hq-console/src/lib/s3-vault.ts buildAuthorMetadata` so the
45
- * read path on the consumer side stays a single check against
46
- * `Metadata['created-by']`.
47
- */
48
- function buildAuthorMetadata(
49
- author: UploadAuthor,
50
- createdAt: string,
51
- ): Record<string, string> {
52
- const meta: Record<string, string> = {};
53
- const sub = author.userSub.trim();
54
- if (sub && /^[\x20-\x7E]+$/.test(sub)) {
55
- meta["created-by-sub"] = sub;
56
- }
57
- const email = author.email.trim();
58
- if (email && /^[\x20-\x7E]+$/.test(email)) {
59
- meta["created-by"] = email;
60
- }
61
- if (createdAt && /^[\x20-\x7E]+$/.test(createdAt)) {
62
- meta["created-at"] = createdAt;
63
- }
64
- return meta;
65
- }
66
-
67
- /**
68
- * S3 user-metadata header that marks an object as a symlink record.
69
- * The value is now an OPAQUE MARKER ('1') — the target lives in the
70
- * object body. Earlier drafts of this feature stored the target in
71
- * metadata (raw, then base64), but S3 user-metadata is HTTP-header-
72
- * bound: total ≤ 2 KiB across all user-defined keys + values. A
73
- * sufficiently long POSIX target (or one with author-metadata
74
- * adding to the total) would exceed the limit and PutObject would
75
- * reject the upload outright. Moving the target to the body — which
76
- * has no such limit — makes target length bounded only by S3's 5 GB
77
- * object size cap. The metadata header still serves as the read-
78
- * time discriminator (cheaper than peeking at body bytes via HEAD).
79
- *
80
- * Backward compat: downloadFile prefers the body (sliced after
81
- * SYMLINK_BODY_PREFIX) for the target string. If the body doesn't
82
- * carry the prefix (a legacy upload from earlier in this PR's
83
- * lifetime), it falls back to base64-decoding the metadata value —
84
- * the round-trip-validating decoder returns raw or decoded as
85
- * appropriate. Any prior in-flight upload still resolves correctly.
86
- */
87
- export const SYMLINK_TARGET_META_KEY = "hq-symlink-target";
88
-
89
- /**
90
- * Constant value written to SYMLINK_TARGET_META_KEY. Any non-empty
91
- * string would work as a discriminator — '1' is just compact and
92
- * conventional for boolean flags in HTTP headers.
93
- */
94
- export const SYMLINK_MARKER_META_VALUE = "1";
95
-
96
- /**
97
- * Encode a target for the S3 metadata header value. Retained as the
98
- * legacy encoder so a downloader can still receive it and round-trip
99
- * via decodeSymlinkMetadataValue, but new uploads use
100
- * SYMLINK_MARKER_META_VALUE — the target lives in the body now.
101
- */
102
- export function encodeSymlinkMetadataValue(target: string): string {
103
- return Buffer.from(target, "utf-8").toString("base64");
104
- }
105
-
106
- /**
107
- * Decode a target from the S3 metadata header value. Used as a
108
- * legacy fallback when the body doesn't carry SYMLINK_BODY_PREFIX
109
- * (i.e. an in-flight upload from earlier in this PR before the
110
- * marker-only metadata convention). Round-trip-validates: if the
111
- * value isn't valid base64 of UTF-8, returns the raw string.
112
- */
113
- export function decodeSymlinkMetadataValue(value: string): string {
114
- try {
115
- const decoded = Buffer.from(value, "base64").toString("utf-8");
116
- if (Buffer.from(decoded, "utf-8").toString("base64") === value) {
117
- return decoded;
118
- }
119
- } catch {
120
- // fall through
121
- }
122
- return value;
123
- }
124
-
125
- /**
126
- * Magic prefix prepended to symlink-record bodies on the wire. Two
127
- * properties this gives us:
128
- *
129
- * 1. ETag distinguishability. S3 ETag = MD5(body). Without a prefix,
130
- * a symlink whose target string equals some regular file's exact
131
- * contents would produce the same ETag, and the LIST-based pull
132
- * planner (which can't see per-object metadata) would classify a
133
- * symlink ↔ regular-file transition as "no change" and never
134
- * replace the local representation. The prefix makes those two
135
- * shapes ETag-distinguishable for the realistic case (collision
136
- * now requires a regular file whose contents *start* with this
137
- * prefix, which is implausible for any non-malicious source).
138
- *
139
- * 2. Fallback discriminator. If user-metadata is ever lost (S3
140
- * cross-region replication of object data only, manual S3 console
141
- * copy that drops Metadata), the body prefix lets a downloader
142
- * recover the symlink record without needing the metadata header.
143
- * We don't currently rely on this fallback — the metadata header
144
- * is still the primary discriminator on the read path — but the
145
- * prefix keeps the option open and avoids painting us into a
146
- * "metadata is the only signal" corner.
147
- *
148
- * Format: `hq-symlink:` + target string (UTF-8 bytes). No trailing
149
- * newline. The colon separates the marker from the target so a future
150
- * extension can encode additional fields if needed.
151
- */
152
- export const SYMLINK_BODY_PREFIX = "hq-symlink:";
153
- const SYMLINK_BODY_PREFIX_BYTES = Buffer.from(SYMLINK_BODY_PREFIX, "utf-8");
154
-
155
- /**
156
- * S3 user-metadata key carrying the source-side file mode (permission bits
157
- * only — \`mode & 0o777\`) as an octal string ("755", "640", etc.). On
158
- * download, downloadFile parses this with \`parseInt(value, 8)\` and chmods
159
- * the file to the exact source mode after the byte write.
160
- *
161
- * Bug #5 in the 5.33.0 deep-test was originally reported as "exec bit lost
162
- * on sync" but the verification report broadened it: ALL modes (0600 / 0640
163
- * / 0700 / 0750 / 0755) collapsed to the receiver's umask default (0644)
164
- * because no mode signal crossed the wire at all. Stamping the mode in
165
- * metadata is the smallest schema change that preserves the full
166
- * permission bitfield without a per-host umask negotiation.
167
- *
168
- * Symlinks: skipped at upload time (symlink mode is OS-controlled
169
- * lrwxrwxrwx) and skipped on download (\`fs.chmodSync\` follows symlinks
170
- * and would mutate the target's mode instead).
171
- *
172
- * Back-compat: legacy uploads have no \`hq-mode\` header — the receiver
173
- * leaves the umask default in place, matching pre-fix behavior.
174
- */
175
- export const FILE_MODE_META_KEY = "hq-mode";
176
-
177
- /**
178
- * S3 user-metadata key carrying the source-side file modification time
179
- * (mtimeMs) as an integer-millisecond epoch string ("1700000000000"). On
180
- * download, downloadFile parses this with a strict-numeric regex BEFORE
181
- * parseInt, then applies it via `fs.utimesSync(localPath, mtimeDate,
182
- * mtimeDate)` after the byte write.
183
- *
184
- * 5.37.0 symmetric to the 5.34.0 Bug #5 mode preservation: a file's
185
- * modification time should follow it across machines instead of resetting
186
- * to "the time of sync." Without this metadata, every receiver's mtime is
187
- * wall-clock-now at write-time — making "newer than" comparisons,
188
- * mtime-keyed caches, and reproducible builds break across sync.
189
- *
190
- * Symlinks: skipped at upload time (symlink mtime is OS-controlled and
191
- * `lstat` on a symlink already returns the symlink's own times — but we
192
- * don't stamp them because the symlink record wire body is `hq-symlink:`
193
- * + target string, not real file content, so its mtime isn't user-
194
- * meaningful) and skipped on download (`fs.utimesSync` follows symlinks
195
- * and would mutate the target's mtime instead; `lutimesSync` is not in
196
- * stable Node).
197
- *
198
- * Composition with 5.36.0 lstat fast-path: the journal stamp is captured
199
- * AFTER utimesSync runs (the share/sync call sites lstat AFTER
200
- * downloadFile returns), so the journal's mtimeMs matches the post-utimes
201
- * lstat. The next sync's fast-path correctly skips re-hashing.
202
- *
203
- * Clock skew: a peer with a wrong clock pushes file with mtimeMs=<wrong>;
204
- * receivers apply <wrong>. This is the same trade git's "file from the
205
- * future" warning makes — silent in our case, deliberately. Clock skew
206
- * is the user's problem, not the sync engine's.
207
- *
208
- * Back-compat: legacy uploads (pre-5.37.0) have no `hq-mtime` header —
209
- * the receiver leaves the on-disk mtime at write-time, matching pre-
210
- * 5.37.0 behavior. Forward-compat: pre-5.37.0 pullers ignore `hq-mtime`
211
- * and keep their current "mtime = write-time" behavior. Both work; only
212
- * the receiver upgrade unlocks the feature.
213
- */
214
- export const FILE_MTIME_META_KEY = "hq-mtime";
215
-
216
- /**
217
- * S3 user-metadata key carrying the source-side file birthtime (birthtimeMs)
218
- * as an integer-millisecond epoch string. Stamped on upload ONLY when
219
- * `birthtimeMs > 0 && birthtimeMs !== mtimeMs` — many filesystems (Linux
220
- * ext4 historically, tmpfs, some FUSE mounts) return 0 (unsupported) or
221
- * the same value as mtime (no separate creation time tracking). The
222
- * filter keeps the metadata header free of noise on those platforms.
223
- *
224
- * Pull: NO-OP for now. Node has no API to set birthtime on POSIX as of
225
- * v24 (no `lbirthtime`, no `birthtimeSync`). The push side stamps it
226
- * anyway so a future receiver upgrade — once Node lands the API — can
227
- * apply it without a server-side data migration.
228
- *
229
- * Symlinks: skipped on both sides for the same reasons as `hq-mtime`.
230
- */
231
- export const FILE_BTIME_META_KEY = "hq-btime";
232
-
233
- /**
234
- * Encode/decode the symlink wire body. Kept as exported helpers so the
235
- * format is centrally defined and tests can probe both sides without
236
- * duplicating the prefix string.
237
- */
238
- export function encodeSymlinkBody(target: string): Buffer {
239
- return Buffer.from(SYMLINK_BODY_PREFIX + target, "utf-8");
240
- }
241
-
242
- /**
243
- * Staged-download (`.hq-tmp-*`) and replacement-backup (`.hq-backup-*`)
244
- * sentinels are cleaned by try/finally in-process, but a hard kill
245
- * mid-transfer (agent job `timeout -k`, OOM, systemd stop) never reaches the
246
- * finally and nothing else removes them. On a fleet box that leaked 866M of
247
- * orphans in two days and filled /tmp (2026-07-20 incident). Sweep stale
248
- * sentinels in the destination dir before staging a new one there.
249
- *
250
- * The age threshold far exceeds any live transfer, so a concurrent download's
251
- * fresh sentinel is never touched. Only non-directories matching the sentinel
252
- * pattern are candidates. Per-dir memoization bounds the readdir cost to once
253
- * per directory per process run.
254
- */
255
- const STAGED_SENTINEL_PATTERN = /^\.hq-(tmp|backup)-/;
256
- const STAGED_SWEEP_MAX_AGE_MS = 60 * 60 * 1000;
257
- const sweptStagedDirs = new Set<string>();
258
-
259
- export function sweepStaleStagedFiles(
260
- dir: string,
261
- maxAgeMs: number = STAGED_SWEEP_MAX_AGE_MS,
262
- nowMs: number = Date.now(),
263
- ): string[] {
264
- let entries: string[];
265
- try {
266
- entries = fs.readdirSync(dir);
267
- } catch (err) {
268
- // Destination dir may not exist yet (created later by downloadFile) —
269
- // nothing to sweep. Anything else is unexpected but must not fail the
270
- // download that triggered the sweep.
271
- if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
272
- console.warn(`[hq-sync] staged-file sweep: cannot read ${dir}: ${err}`);
273
- }
274
- return [];
275
- }
276
- const removed: string[] = [];
277
- for (const name of entries) {
278
- if (!STAGED_SENTINEL_PATTERN.test(name)) continue;
279
- const full = path.join(dir, name);
280
- try {
281
- const stat = fs.lstatSync(full);
282
- if (stat.isDirectory()) continue;
283
- if (nowMs - stat.mtimeMs < maxAgeMs) continue;
284
- fs.unlinkSync(full);
285
- removed.push(full);
286
- } catch (err) {
287
- // A concurrent process may have removed it first (ENOENT: fine), or the
288
- // file is unremovable (EPERM/EROFS) — either way the download must
289
- // proceed; surface non-ENOENT so the leak stays visible.
290
- if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
291
- console.warn(`[hq-sync] staged-file sweep: cannot remove ${full}: ${err}`);
292
- }
293
- }
294
- }
295
- if (removed.length > 0) {
296
- console.warn(
297
- `[hq-sync] removed ${removed.length} stale staged file(s) in ${dir}`,
298
- );
299
- }
300
- return removed;
301
- }
302
-
303
- function sweepStaleStagedFilesOnce(dir: string): void {
304
- if (sweptStagedDirs.has(dir)) return;
305
- sweptStagedDirs.add(dir);
306
- sweepStaleStagedFiles(dir);
307
- }
308
-
309
- function downloadTempPath(localPath: string): string {
310
- const dir = path.dirname(localPath);
311
- sweepStaleStagedFilesOnce(dir);
312
- const random = crypto.randomBytes(10).toString("hex");
313
- return path.join(dir, `.hq-tmp-${random}`);
314
- }
315
-
316
- function removeTempPath(tempPath: string): void {
317
- try {
318
- fs.unlinkSync(tempPath);
319
- } catch {
320
- // A win32 directory-type symlink is a directory reparse point and
321
- // refuses unlink (EPERM); rmdir is the correct removal call for it.
322
- try {
323
- fs.rmdirSync(tempPath);
324
- } catch {
325
- // Best-effort cleanup; do not mask the transfer/materialization error.
326
- }
327
- }
328
- }
329
-
330
- export interface CreateStagedSymlinkOps {
331
- platform: NodeJS.Platform;
332
- symlink(target: string, linkPath: string, type?: fs.symlink.Type): void;
333
- statIsDirectory(absPath: string): boolean | undefined;
334
- }
335
-
336
- const DEFAULT_CREATE_STAGED_SYMLINK_OPS: CreateStagedSymlinkOps = {
337
- platform: process.platform,
338
- symlink: (target, linkPath, type) =>
339
- fs.symlinkSync(target, linkPath, type),
340
- statIsDirectory: (absPath) => {
341
- try {
342
- return fs.statSync(absPath).isDirectory();
343
- } catch {
344
- return undefined;
345
- }
346
- },
347
- };
348
-
349
- /**
350
- * Create a fully-staged symlink without requiring Windows Developer Mode.
351
- * NTFS symlinks fail with EPERM without SeCreateSymbolicLinkPrivilege, while
352
- * directory junctions need no privilege but require an absolute target. HQ
353
- * vault links are directory overlays, so missing targets default to junctions.
354
- * See Sentry HQ-DESKTOP-3M.
355
- */
356
- export function createStagedSymlink(
357
- target: string,
358
- linkPath: string,
359
- ops: CreateStagedSymlinkOps = DEFAULT_CREATE_STAGED_SYMLINK_OPS,
360
- ): void {
361
- if (ops.platform !== "win32") {
362
- ops.symlink(target, linkPath);
363
- return;
364
- }
365
-
366
- const absTarget = path.isAbsolute(target)
367
- ? target
368
- : path.resolve(path.dirname(linkPath), target);
369
- if (ops.statIsDirectory(absTarget) === false) {
370
- ops.symlink(target, linkPath, "file");
371
- return;
372
- }
373
-
374
- ops.symlink(absTarget, linkPath, "junction");
375
- }
376
-
377
- /**
378
- * downloadFile's `options.win32` seam is the authority for symlink-flavor
379
- * decisions (it is what makes Windows semantics testable on POSIX CI), so the
380
- * staged-link helper must see the SAME platform the idempotence fast path
381
- * reasoned about rather than the raw `process.platform`. createStagedSymlink
382
- * only ever asks "is this win32?", so "linux" is a faithful stand-in for
383
- * "not Windows" on the false branch.
384
- */
385
- function stagedSymlinkOpsFor(win32: boolean): CreateStagedSymlinkOps {
386
- return {
387
- ...DEFAULT_CREATE_STAGED_SYMLINK_OPS,
388
- platform: win32 ? "win32" : "linux",
389
- };
390
- }
391
-
392
- /**
393
- * Would createStagedSymlink() materialize `target` as a DIRECTORY-flavored
394
- * link (a junction) rather than a file symlink? Mirrors that helper's rule
395
- * exactly — only a target confirmed to be a non-directory earns a file
396
- * symlink; directories and unresolvable targets become junctions, which need
397
- * no Windows privilege. Keeping the two in lockstep is what lets the
398
- * idempotence fast path below decide whether an existing link is already the
399
- * link we would mint today.
400
- */
401
- function win32StagedLinkIsDirectoryFlavored(
402
- target: string,
403
- linkPath: string,
404
- ops: CreateStagedSymlinkOps = DEFAULT_CREATE_STAGED_SYMLINK_OPS,
405
- ): boolean {
406
- const absTarget = path.isAbsolute(target)
407
- ? target
408
- : path.resolve(path.dirname(linkPath), target);
409
- return ops.statIsDirectory(absTarget) !== false;
410
- }
411
-
412
- /**
413
- * Compare an on-disk readlink() result against an incoming wire target.
414
- *
415
- * POSIX compares exactly: we write the wire target verbatim, so readlink()
416
- * returns it byte-for-byte, and a backslash is an ordinary filename character
417
- * there. On win32 neither property holds — createStagedSymlink() materializes
418
- * directory overlays as JUNCTIONS, which require an ABSOLUTE substitute path,
419
- * and Node rewrites '/' → '\\' at symlink() time. A raw string compare would
420
- * therefore never match on Windows and the fast path would never fire for the
421
- * .claude/skills link farm (the exact perpetual-churn bug this guards). So
422
- * win32 equality resolves BOTH sides against the link's parent directory and
423
- * compares separator-insensitively, which normalizes the relative wire target
424
- * and the absolute junction target onto the same footing.
425
- */
426
- function symlinkTargetsEqual(
427
- existing: string,
428
- incoming: string,
429
- linkPath: string,
430
- win32: boolean,
431
- ): boolean {
432
- if (!win32) return existing === incoming;
433
- const normalize = (value: string): string =>
434
- path
435
- .resolve(path.dirname(linkPath), value.replace(/\\/g, "/"))
436
- .replace(/\\/g, "/");
437
- return normalize(existing) === normalize(incoming);
438
- }
439
-
440
- export interface ReplaceStagedPathOps {
441
- lstat(path: string): fs.Stats;
442
- rename(from: string, to: string): void;
443
- remove(path: string): void;
444
- }
445
-
446
- const DEFAULT_REPLACE_STAGED_PATH_OPS: ReplaceStagedPathOps = {
447
- lstat: (p) => fs.lstatSync(p),
448
- rename: (from, to) => fs.renameSync(from, to),
449
- remove: (p) => {
450
- try {
451
- fs.unlinkSync(p);
452
- } catch (err) {
453
- // A win32 directory-type symlink or junction is a directory reparse
454
- // point; some runtimes refuse unlink (EPERM/EISDIR) and need rmdir —
455
- // same fallback as removeTempPath. Anything else stays loud so
456
- // replaceStagedPath can roll back.
457
- const code = (err as NodeJS.ErrnoException).code;
458
- if (code !== "EPERM" && code !== "EISDIR") throw err;
459
- fs.rmdirSync(p);
460
- }
461
- },
462
- };
463
-
464
- function replacementBackupPath(localPath: string): string {
465
- const dir = path.dirname(localPath);
466
- const random = crypto.randomBytes(10).toString("hex");
467
- return path.join(dir, `.hq-backup-${random}`);
468
- }
469
-
470
- /**
471
- * Install a fully-staged download over its destination.
472
- *
473
- * A direct rename is atomic and remains the fast path for absent destinations
474
- * and ordinary files. Windows cannot rename over an existing directory link,
475
- * though, because MoveFileEx treats that link as an existing directory. For a
476
- * symlink/junction destination, move the old link to a sibling backup first,
477
- * install the staged entry, then remove the backup. Any failure after the
478
- * backup move restores the exact prior link before rethrowing.
479
- *
480
- * The tiny operations seam keeps the Windows failure/rollback contract
481
- * deterministic in ESM tests without spying on non-configurable fs exports.
482
- */
483
- export function replaceStagedPath(
484
- stagedPath: string,
485
- localPath: string,
486
- ops: ReplaceStagedPathOps = DEFAULT_REPLACE_STAGED_PATH_OPS,
487
- ): void {
488
- let existing: fs.Stats;
489
- try {
490
- existing = ops.lstat(localPath);
491
- } catch (err) {
492
- if ((err as NodeJS.ErrnoException).code === "ENOENT") {
493
- ops.rename(stagedPath, localPath);
494
- return;
495
- }
496
- throw err;
497
- }
498
-
499
- if (!existing.isSymbolicLink()) {
500
- ops.rename(stagedPath, localPath);
501
- return;
502
- }
503
-
504
- const backupPath = replacementBackupPath(localPath);
505
- ops.rename(localPath, backupPath);
506
- let stagedInstalled = false;
507
- try {
508
- ops.rename(stagedPath, localPath);
509
- stagedInstalled = true;
510
- ops.remove(backupPath);
511
- } catch (primaryError) {
512
- const rollbackErrors: unknown[] = [];
513
- if (stagedInstalled) {
514
- try {
515
- ops.rename(localPath, stagedPath);
516
- } catch (rollbackError) {
517
- rollbackErrors.push(rollbackError);
518
- }
519
- }
520
- try {
521
- ops.rename(backupPath, localPath);
522
- } catch (rollbackError) {
523
- rollbackErrors.push(rollbackError);
524
- }
525
- if (rollbackErrors.length > 0) {
526
- throw new AggregateError(
527
- [primaryError, ...rollbackErrors],
528
- "Staged path replacement failed and the previous directory link could not be fully restored",
529
- );
530
- }
531
- throw primaryError;
532
- }
533
- }
534
-
535
- async function getObjectStream(
536
- io: ObjectIO,
537
- key: string,
538
- ): Promise<GetObjectStreamResult> {
539
- if (io.getObjectStream) return io.getObjectStream(key);
540
- const res = await io.getObject(key);
541
- return {
542
- body: (async function* () {
543
- yield res.body;
544
- })(),
545
- metadata: res.metadata,
546
- };
547
- }
548
-
549
- function bufferFromChunk(chunk: Uint8Array): Buffer {
550
- if (Buffer.isBuffer(chunk)) return chunk;
551
- return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
552
- }
553
-
554
- function leadingBytes(chunks: Buffer[], totalLength: number, length: number): Buffer {
555
- const out = Buffer.alloc(Math.min(totalLength, length));
556
- let offset = 0;
557
- for (const chunk of chunks) {
558
- if (offset >= out.length) break;
559
- const take = Math.min(chunk.length, out.length - offset);
560
- chunk.copy(out, offset, 0, take);
561
- offset += take;
562
- }
563
- return out;
564
- }
565
-
566
- async function collectRemainingChunks(
567
- initialChunks: Buffer[],
568
- iterator: AsyncIterator<Uint8Array>,
569
- ): Promise<Buffer> {
570
- const chunks = [...initialChunks];
571
- let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
572
- while (true) {
573
- const next = await iterator.next();
574
- if (next.done) break;
575
- const chunk = bufferFromChunk(next.value);
576
- if (chunk.length === 0) continue;
577
- chunks.push(chunk);
578
- totalLength += chunk.length;
579
- }
580
- return Buffer.concat(chunks, totalLength);
581
- }
582
-
583
- async function* regularDownloadChunks(
584
- initialChunks: Buffer[],
585
- iterator: AsyncIterator<Uint8Array>,
586
- ): AsyncIterable<Buffer> {
587
- for (const chunk of initialChunks) {
588
- if (chunk.length > 0) yield chunk;
589
- }
590
- while (true) {
591
- const next = await iterator.next();
592
- if (next.done) break;
593
- const chunk = bufferFromChunk(next.value);
594
- if (chunk.length > 0) yield chunk;
595
- }
596
- }
597
-
598
- class HashingTransform extends Transform {
599
- private readonly hash = crypto.createHash("sha256");
600
- size = 0;
601
-
602
- _transform(
603
- chunk: Buffer,
604
- _encoding: BufferEncoding,
605
- callback: (error?: Error | null, data?: Buffer) => void,
606
- ): void {
607
- this.hash.update(chunk);
608
- this.size += chunk.length;
609
- callback(null, chunk);
610
- }
611
-
612
- digest(): string {
613
- return this.hash.digest("hex");
614
- }
615
- }
616
-
617
- async function streamRegularFileToTemp(
618
- tempPath: string,
619
- initialChunks: Buffer[],
620
- iterator: AsyncIterator<Uint8Array>,
621
- ): Promise<{ hash: string; size: number }> {
622
- const hashing = new HashingTransform();
623
- await pipeline(
624
- Readable.from(regularDownloadChunks(initialChunks, iterator)),
625
- hashing,
626
- fs.createWriteStream(tempPath, { flags: "wx" }),
627
- );
628
- return { hash: hashing.digest(), size: hashing.size };
629
- }
630
-
631
- /**
632
- * Batch pre-mint transport URLs for `keys` under `op` so the subsequent
633
- * per-file transfer calls (downloadFile/headRemoteFile/…) reuse them instead
634
- * of presigning one key at a time. On the presigned-URL transport this turns
635
- * an N-file leg from N presign requests into ceil(N/100) — the difference
636
- * between completing a bulk pull and 429ing past the 100-req/hr limit. No-op
637
- * on the S3 SDK transport (which has no presign step) and harmless if called
638
- * with an empty list. Best-effort: a prime failure never propagates — the
639
- * per-file path falls back to a single presign.
640
- *
641
- * Call it once, right before a transfer loop, with the full key set the loop
642
- * will touch. The presigned transport memoizes one IO instance per company for
643
- * the run, so the warmed cache is the same one the loop drains.
644
- */
645
- export async function primeObjectTransport(
646
- ctx: EntityContext,
647
- op: "get" | "put" | "delete",
648
- keys: string[],
649
- ): Promise<void> {
650
- if (keys.length === 0) return;
651
- const io = resolveObjectIO(ctx);
652
- if (!io.prime) return;
653
- await io.prime(
654
- op,
655
- keys.map((key) => ({ key })),
656
- );
657
- }
658
-
659
- /**
660
- * Source-side mode + mtime (+ btime when distinct) metadata for a regular
661
- * file, from a single lstat. Symlinks carry none (OS-controlled mode; a link's
662
- * mtime isn't user-meaningful — the wire body is the target string, not file
663
- * content). Shared by uploadFile and the primeUploads pre-pass so the PUT
664
- * metadata they produce is byte-identical. See the FILE_*_META_KEY docs for the
665
- * per-field rationale.
666
- */
667
- function buildModeTimeMetadata(lstat: fs.Stats): Record<string, string> {
668
- const meta: Record<string, string> = {};
669
- if (lstat.isSymbolicLink()) return meta;
670
- meta[FILE_MODE_META_KEY] = (lstat.mode & 0o777).toString(8);
671
- const mtimeFloor = Math.floor(lstat.mtimeMs);
672
- if (Number.isFinite(lstat.mtimeMs)) meta[FILE_MTIME_META_KEY] = String(mtimeFloor);
673
- const btimeFloor = Math.floor(lstat.birthtimeMs);
674
- if (
675
- Number.isFinite(lstat.birthtimeMs) &&
676
- btimeFloor > 0 &&
677
- btimeFloor !== mtimeFloor
678
- ) {
679
- meta[FILE_BTIME_META_KEY] = String(btimeFloor);
680
- }
681
- return meta;
682
- }
683
-
684
- /**
685
- * Resolve the created-at to stamp: the existing object's value (preserved
686
- * across re-uploads so the hq-console NEW-pill window doesn't reset) or now for
687
- * a first upload. HEAD failure / no author → now. Shared by upload* and
688
- * primeUploads so both agree on the value signed into the PUT.
689
- */
690
- async function resolveCreatedAt(
691
- io: ObjectIO,
692
- key: string,
693
- author?: UploadAuthor,
694
- ): Promise<string> {
695
- let createdAt = new Date().toISOString();
696
- if (author) {
697
- try {
698
- const head = await io.headObject(key);
699
- const existing = head?.metadata?.["created-at"];
700
- if (typeof existing === "string" && existing.length > 0) {
701
- createdAt = existing;
702
- }
703
- } catch {
704
- // Object doesn't exist yet, or HEAD failed — keep now (first upload).
705
- }
706
- }
707
- return createdAt;
708
- }
709
-
710
- /**
711
- * One upload's identity for {@link primeUploads}: the vault key, the local
712
- * path (to lstat for mode/mtime), whether it's a symlink, and the author.
713
- */
714
- export interface UploadPrimeItem {
715
- key: string;
716
- localPath: string;
717
- isSymlink: boolean;
718
- author?: UploadAuthor;
719
- }
720
-
721
- /**
722
- * Batch pre-mint PUT URLs (+ the created-at HEADs they depend on) for a set of
723
- * uploads, signing the SAME metadata uploadFile/uploadSymlink would compute so
724
- * the transfer loop can replay the cached headers. Turns an N-file push from
725
- * ~N presign calls (1 per PUT, sometimes 2-3 with HEADs) into ceil(N/1000) GET
726
- * + ceil(N/1000) PUT — the difference between completing a bulk push and 429ing
727
- * past the 100/hr limit. No-op on the S3 SDK transport; best-effort.
728
- *
729
- * The per-item created-at HEADs run over the GET cache primed first, so they
730
- * cost S3 round-trips but NO extra presign calls (not counted against 100/hr).
731
- */
732
- export async function primeUploads(
733
- ctx: EntityContext,
734
- items: UploadPrimeItem[],
735
- ): Promise<void> {
736
- const io = resolveObjectIO(ctx);
737
- if (!io.prime || items.length === 0) return;
738
-
739
- // Prime GET first so each item's created-at HEAD reuses a cached URL.
740
- // Canonicalize to POSIX here (one-canonical-form, matching the uploadFile /
741
- // uploadSymlink boundary): a Windows-origin backslash key must cache under
742
- // the SAME key uploadFile later looks up via hasPrimedPut(toPosixKey(...)),
743
- // or the primed URL silently misses and the upload re-presigns. It also
744
- // keeps the created-at HEAD pointed at the real (POSIX) object.
745
- await io.prime(
746
- "get",
747
- items.map((i) => ({ key: toPosixKey(i.key) })),
748
- );
749
-
750
- // Build per-key PUT metadata with the SAME builders the upload path uses,
751
- // bounded-concurrently (the HEADs are cheap cached-GET fetches).
752
- const putKeys: Array<{
753
- key: string;
754
- contentType: string;
755
- metadata: Record<string, string>;
756
- }> = [];
757
- const CONCURRENCY = 16;
758
- let next = 0;
759
- const worker = async (): Promise<void> => {
760
- while (next < items.length) {
761
- const it = items[next++];
762
- // Same boundary guardrail as uploadFile/uploadSymlink: prime under the
763
- // canonical POSIX key so the cached PUT URL is keyed identically to the
764
- // hasPrimedPut/putObject lookup, and a backslash key can never be primed
765
- // (let alone stored) as a non-POSIX vault key.
766
- const key = toPosixKey(it.key);
767
- const createdAt = await resolveCreatedAt(io, key, it.author);
768
- if (it.isSymlink) {
769
- putKeys.push({
770
- key,
771
- contentType: "application/octet-stream",
772
- metadata: {
773
- [SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
774
- ...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
775
- },
776
- });
777
- } else {
778
- let modeTime: Record<string, string> = {};
779
- try {
780
- modeTime = buildModeTimeMetadata(fs.lstatSync(it.localPath));
781
- } catch {
782
- // raced rm / EPERM — leave stamps off (receiver umask default).
783
- }
784
- putKeys.push({
785
- key,
786
- contentType: getMimeType(key),
787
- metadata: {
788
- ...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
789
- ...modeTime,
790
- },
791
- });
792
- }
793
- }
794
- };
795
- await Promise.all(
796
- Array.from({ length: Math.min(CONCURRENCY, items.length) }, worker),
797
- );
798
-
799
- await io.prime("put", putKeys);
800
- }
801
-
802
- /**
803
- * Normalize an S3 object key to POSIX ("/") separators.
804
- *
805
- * S3 keys are always "/"-separated, and the vault listing reconstructs the
806
- * folder tree by splitting keys on "/". A non-POSIX (Windows) sync client
807
- * builds a key from `path.relative(...)`, whose separator is the native "\\";
808
- * stored verbatim, a nested key like `knowledge\books-eoi.md` contains no "/"
809
- * and the listing renders it flat at the vault root as one oddly-named file.
810
- * Converting every "\\" to "/" makes the stored key POSIX regardless of the
811
- * client OS.
812
- *
813
- * Mirrors the prefix guard in hq-pro files-acl (`FORBIDDEN_CHARS = /[?#\\[\]]/`,
814
- * which already rejects "\\" on browse/grant prefixes). Uploads go direct to
815
- * S3 via vended STS creds, so the server never re-validates the uploaded key —
816
- * the upload primitives (uploadFile / uploadSymlink) are the only shared choke
817
- * point, and they call this so a backslash key can never be stored again.
818
- */
819
- export function toPosixKey(key: string): string {
820
- return key.split("\\").join("/");
821
- }
822
-
823
- /**
824
- * Client-side mirror of hq-pro's server validator
825
- * (`vault-service/handlers/files-acl.ts validateObjectKey`). The direct-S3
826
- * STS transport bypasses server key validation entirely — the upload
827
- * primitives (uploadFile / uploadSymlink) are the only shared choke point —
828
- * so we enforce the SAME rules here BEFORE the PUT. Incident 2026-07-11: a
829
- * client with a stale doubled local tree (companies/{slug}/companies/{slug}/…)
830
- * pushed literal `companies/…` keys into a company-scoped vault bucket via
831
- * direct S3; agent boxes could LIST them but the presign download API
832
- * rejected every one (INVALID_KEY_COMPANIES_SCOPED), erroring the whole
833
- * company sync fleet-wide. A poisoned key must fail loudly at the writer,
834
- * not at every reader.
835
- *
836
- * `scope` selects the companies/-prefix rule: company vault buckets are
837
- * already company-scoped so a `companies/…` key is always wrong there;
838
- * personal vaults legitimately carry `companies/{slug}/…` keys for local
839
- * (non-cloud) companies. Callers derive scope from the EntityContext uid
840
- * (`cmp_*` → "company", otherwise "personal").
841
- *
842
- * Errors carry the same `code` values the server throws so telemetry and
843
- * operator messaging line up across client and server. Pull planning also
844
- * uses the non-throwing classifier below to keep poison keys away from the
845
- * presign transport.
846
- */
847
- // Matching control chars is the whole point — block NUL/0x00–0x1F/0x7F in keys
848
- // (they can smuggle past path checks or corrupt HTTP headers on the signed URL).
849
- // eslint-disable-next-line no-control-regex
850
- const KEY_CONTROL_CHARS = /[\x00-\x1F\x7F]/;
851
-
852
- export type VaultKeyScope = "company" | "personal";
853
-
854
- export interface VaultKeyValidationIssue {
855
- code: string;
856
- message: string;
857
- }
858
-
859
- /**
860
- * Classify a vault key with the same rules as {@link validateVaultUploadKey}
861
- * without throwing. Callers that enumerate remote objects can use this before
862
- * presigning so permanently-invalid legacy keys remain benign skips.
863
- */
864
- export function classifyVaultKey(
865
- key: string,
866
- scope: VaultKeyScope,
867
- ): VaultKeyValidationIssue | null {
868
- if (key.length === 0) {
869
- return {
870
- code: "INVALID_KEY_EMPTY",
871
- message: "Invalid key: must not be empty",
872
- };
873
- }
874
- if (key.startsWith("/")) {
875
- return {
876
- code: "INVALID_KEY_LEADING_SLASH",
877
- message: "Invalid key: leading '/' not allowed",
878
- };
879
- }
880
- if (key.endsWith("/")) {
881
- return {
882
- code: "INVALID_KEY_TRAILING_SLASH",
883
- message: "Invalid key: trailing '/' (folder marker, not an object)",
884
- };
885
- }
886
- if (scope === "company" && key.startsWith("companies/")) {
887
- return {
888
- code: "INVALID_KEY_COMPANIES_SCOPED",
889
- message:
890
- "Invalid key: vault keys are bucket-relative and the vault is already " +
891
- "company-scoped — do not prefix with 'companies/<slug>/'. This " +
892
- "usually means a stale doubled local tree " +
893
- "(companies/<slug>/companies/<slug>/…); remove the inner copy.",
894
- };
895
- }
896
- if (KEY_CONTROL_CHARS.test(key)) {
897
- return {
898
- code: "INVALID_KEY_CONTROL_CHARS",
899
- message: "Invalid key: contains control characters",
900
- };
901
- }
902
- if (key.includes("\\")) {
903
- return {
904
- code: "INVALID_KEY_BACKSLASH",
905
- message:
906
- "Invalid key: backslash separators are not allowed — vault keys are POSIX ('/')",
907
- };
908
- }
909
- if (key.includes("//")) {
910
- return {
911
- code: "INVALID_KEY_DOUBLE_SLASH",
912
- message: "Invalid key: consecutive slashes not allowed",
913
- };
914
- }
915
- for (const segment of key.split("/")) {
916
- if (segment === "." || segment === "..") {
917
- return {
918
- code: "INVALID_KEY_DOT_COMPONENT",
919
- message: "Invalid key: '.' and '..' path components are not allowed",
920
- };
921
- }
922
- }
923
- return null;
924
- }
925
-
926
- export function validateVaultUploadKey(
927
- key: string,
928
- scope: VaultKeyScope,
929
- ): void {
930
- const issue = classifyVaultKey(key, scope);
931
- if (issue !== null) {
932
- throw Object.assign(new Error(issue.message), { code: issue.code });
933
- }
934
- }
935
-
936
- /** Scope for {@link validateVaultUploadKey}, derived from the entity uid. */
937
- function uploadScopeFor(ctx: EntityContext): VaultKeyScope {
938
- return ctx.uid.startsWith("cmp_") ? "company" : "personal";
939
- }
940
-
941
- export async function uploadFile(
942
- ctx: EntityContext,
943
- localPath: string,
944
- key: string,
945
- author?: UploadAuthor,
946
- precondition?: PutPrecondition,
947
- ): Promise<{ etag: string }> {
948
- // Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
949
- // never store a key the server-side validator would reject (see
950
- // validateVaultUploadKey — the direct-S3 STS transport bypasses server
951
- // validation, so this is the last line of defense against key poisoning).
952
- key = toPosixKey(key);
953
- validateVaultUploadKey(key, uploadScopeFor(ctx));
954
- const io = resolveObjectIO(ctx);
955
- const body = fs.readFileSync(localPath);
956
-
957
- // Fast path: a primeUploads() pre-pass already signed this file's metadata
958
- // into a cached PUT URL. Skip the lstat-metadata + created-at HEAD and just
959
- // send the body — putObject replays the cached headers (computed by the SAME
960
- // builders below, so identical). hasPrimedPut only reports true with >60s of
961
- // URL lifetime left, so the cache can't expire before the putObject below.
962
- // Fenced PUTs bypass the fast path: a primed URL was signed WITHOUT the
963
- // conditional header, so replaying it would silently drop the fence.
964
- if (!precondition && io.hasPrimedPut?.(key)) {
965
- const primed = await io.putObject({
966
- key,
967
- body,
968
- contentType: getMimeType(key),
969
- metadata: {},
970
- });
971
- return { etag: primed.etag };
972
- }
973
-
974
- // Source-side mode/mtime/btime (Bug #5 + 5.37.0) and the preserved
975
- // created-at (so the NEW-pill window doesn't reset on re-upload). Both via
976
- // the shared builders that primeUploads uses, so a primed PUT carries the
977
- // identical metadata — see buildModeTimeMetadata / resolveCreatedAt.
978
- let modeTime: Record<string, string> = {};
979
- try {
980
- modeTime = buildModeTimeMetadata(fs.lstatSync(localPath));
981
- } catch {
982
- // raced rm / EPERM — leave stamps off; receiver keeps its umask default.
983
- }
984
- const createdAt = await resolveCreatedAt(io, key, author);
985
- const Metadata: Record<string, string> = {
986
- ...(author ? buildAuthorMetadata(author, createdAt) : {}),
987
- ...modeTime,
988
- };
989
-
990
- const response = await io.putObject({
991
- key,
992
- body,
993
- contentType: getMimeType(key),
994
- metadata: Metadata,
995
- ...(precondition ?? {}),
996
- });
997
-
998
- return { etag: response.etag };
999
- }
1000
-
1001
- /**
1002
- * Upload a symlink as a zero-byte object whose user metadata carries the
1003
- * link's target string. Mirrors uploadFile's signature so callers can pick
1004
- * the right primitive once they've classified the entry as link vs file.
1005
- *
1006
- * The target string is stored verbatim — whatever fs.readlinkSync returned.
1007
- * Relative targets transfer cleanly across machines; absolute targets are
1008
- * preserved as-is and may be broken on a destination that doesn't share
1009
- * the source's $HOME layout. Cross-machine portability of absolute targets
1010
- * is out of scope for this primitive — the policy decision lives in the
1011
- * caller (currently: upload anyway, never silently rewrite).
1012
- */
1013
- export async function uploadSymlink(
1014
- ctx: EntityContext,
1015
- target: string,
1016
- key: string,
1017
- author?: UploadAuthor,
1018
- precondition?: PutPrecondition,
1019
- ): Promise<{ etag: string }> {
1020
- // Boundary guardrail: never store a non-POSIX key (see toPosixKey), and
1021
- // never store a key the server-side validator would reject (see
1022
- // validateVaultUploadKey).
1023
- key = toPosixKey(key);
1024
- validateVaultUploadKey(key, uploadScopeFor(ctx));
1025
- const io = resolveObjectIO(ctx);
1026
- const symlinkBody = encodeSymlinkBody(target);
1027
-
1028
- // Fast path: primeUploads() already signed this symlink's metadata into a
1029
- // cached PUT URL — send the body, replay the cached headers. Fenced PUTs
1030
- // bypass it — the primed URL carries no conditional header (see uploadFile).
1031
- if (!precondition && io.hasPrimedPut?.(key)) {
1032
- const primed = await io.putObject({
1033
- key,
1034
- body: symlinkBody,
1035
- contentType: "application/octet-stream",
1036
- metadata: {},
1037
- });
1038
- return { etag: primed.etag };
1039
- }
1040
-
1041
- // Same created-at preservation as uploadFile (shared resolveCreatedAt) so the
1042
- // NEW-pill window doesn't reset on re-upload, and so a primed PUT matches.
1043
- const createdAt = await resolveCreatedAt(io, key, author);
1044
-
1045
- const Metadata: Record<string, string> = {
1046
- // Marker-only: a constant flag value, not the target. The body
1047
- // is the source of truth for the target (no 2 KiB cap, no
1048
- // header-encoding considerations). See SYMLINK_TARGET_META_KEY
1049
- // doc for the full reasoning.
1050
- [SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
1051
- ...(author ? buildAuthorMetadata(author, createdAt) : {}),
1052
- };
1053
-
1054
- const response = await io.putObject({
1055
- key,
1056
- // Body = SYMLINK_BODY_PREFIX + target (UTF-8). The prefix is what
1057
- // makes a symlink record's ETag distinguishable from a regular
1058
- // file whose contents happen to equal the target string — the
1059
- // LIST-based pull planner can't see per-object metadata, so ETag
1060
- // is its only drift signal across symlink ↔ regular-file
1061
- // transitions. See SYMLINK_BODY_PREFIX doc above.
1062
- body: symlinkBody,
1063
- contentType: "application/octet-stream",
1064
- metadata: Metadata,
1065
- ...(precondition ?? {}),
1066
- });
1067
-
1068
- return { etag: response.etag };
1069
- }
1070
-
1071
- /**
1072
- * Download an object to localPath and return its S3 user-metadata.
1073
- *
1074
- * Materializes regular files and symlink records (the symlink branch
1075
- * reconstructs the link from the body/marker). The GetObject response
1076
- * already carries `response.Metadata` (S3 lowercases keys), so we
1077
- * return it to callers — e.g. the pull loop reads `created-by` to
1078
- * attribute downloaded files to their author with zero extra network.
1079
- */
1080
- export interface DownloadModeWarning {
1081
- /** The mode guardrail that could not be applied exactly. */
1082
- reason: "missing-hq-mode" | "invalid-hq-mode" | "chmod-failed";
1083
- /** Legacy objects can retain a local file's known-good permission bits. */
1084
- fallback?: "preserved-local-mode" | "receiver-default";
1085
- }
1086
-
1087
- /**
1088
- * A download whose parent directory exists as a DANGLING symlink.
1089
- *
1090
- * The vault stores directory symlinks as first-class objects (e.g.
1091
- * `companies/<co>/.obsidian -> ../../.obsidian`) AND stores files beneath them.
1092
- * When the link's target is absent on this machine, the child write fails —
1093
- * and it fails as ENOENT rather than EEXIST, because `mkdir(2)` returns EEXIST
1094
- * for the link and Node's `recursive: true` implementation then stats it, which
1095
- * fails on a dangling link. Raw, that surfaced as a hard error that marked the
1096
- * WHOLE company `errored`/`partial` on every sync cycle (2026-07-24 dogfood box:
1097
- * `.obsidian/hotkeys.json: ENOENT ... mkdir '.../companies/indigo/.obsidian'`).
1098
- *
1099
- * It is a per-object condition, not a company-level failure, so it is typed and
1100
- * the pull loop skips the object loudly instead of failing the run. We do NOT
1101
- * materialize the link target: the target is by definition outside the
1102
- * directory being synced, and creating it would write through a path the
1103
- * caller's containment guard deliberately refuses.
1104
- */
1105
- export class DanglingSymlinkParentError extends Error {
1106
- readonly key: string;
1107
- readonly dir: string;
1108
- constructor(key: string, dir: string) {
1109
- super(
1110
- `download skipped: parent directory is a dangling symlink (${dir}) — ` +
1111
- `the vault stores it as a link whose target is absent locally`,
1112
- );
1113
- this.name = "DanglingSymlinkParentError";
1114
- this.key = key;
1115
- this.dir = dir;
1116
- }
1117
- }
1118
-
1119
- /**
1120
- * Create the parent directory for a download, distinguishing the
1121
- * dangling-symlink case from a genuine mkdir failure.
1122
- *
1123
- * Callers must have already established that `dir` does not resolve
1124
- * (`fs.existsSync(dir) === false`); `existsSync` FOLLOWS symlinks, so a
1125
- * dangling link reports absent here while `lstat` still sees the link.
1126
- */
1127
- function ensureDownloadParentDir(dir: string, key: string): void {
1128
- let link: fs.Stats | null = null;
1129
- try {
1130
- link = fs.lstatSync(dir);
1131
- } catch {
1132
- // Genuinely absent — the ordinary first-write path.
1133
- }
1134
- if (link?.isSymbolicLink()) {
1135
- throw new DanglingSymlinkParentError(key, dir);
1136
- }
1137
- fs.mkdirSync(dir, { recursive: true });
1138
- }
1139
-
1140
- export async function downloadFile(
1141
- ctx: EntityContext,
1142
- key: string,
1143
- localPath: string,
1144
- options: {
1145
- beforeReplace?: () => void;
1146
- /**
1147
- * Platform override so win32 symlink semantics are directly testable
1148
- * on any host (same convention as local-path-codec.ts).
1149
- */
1150
- win32?: boolean;
1151
- } = {},
1152
- ): Promise<{
1153
- metadata?: Record<string, string>;
1154
- contentHash?: string;
1155
- contentSize?: number;
1156
- /** Non-fatal mode guardrail warnings for caller telemetry. */
1157
- modeWarnings?: DownloadModeWarning[];
1158
- }> {
1159
- const io = resolveObjectIO(ctx);
1160
-
1161
- const { body, metadata } = await getObjectStream(io, key);
1162
- const iterator = body[Symbol.asyncIterator]();
1163
- const initialChunks: Buffer[] = [];
1164
- let initialLength = 0;
1165
- while (initialLength < SYMLINK_BODY_PREFIX_BYTES.length) {
1166
- const next = await iterator.next();
1167
- if (next.done) break;
1168
- const chunk = bufferFromChunk(next.value);
1169
- if (chunk.length === 0) continue;
1170
- initialChunks.push(chunk);
1171
- initialLength += chunk.length;
1172
- }
1173
-
1174
- const dir = path.dirname(localPath);
1175
- if (!fs.existsSync(dir)) {
1176
- ensureDownloadParentDir(dir, key);
1177
- }
1178
-
1179
- // Symlink path: presence of SYMLINK_TARGET_META_KEY (any non-empty
1180
- // value) is the discriminator. The TARGET is now sourced from the
1181
- // body — the marker-only metadata convention removes the 2 KiB
1182
- // header limit so long POSIX targets don't fail PutObject.
1183
- //
1184
- // S3 lowercases user-metadata keys on read (and sometimes on
1185
- // write), so the lookup uses the lowercased form. We don't
1186
- // normalize Metadata keys ourselves — the AWS SDK already does it.
1187
- const symlinkMarker = metadata?.[SYMLINK_TARGET_META_KEY];
1188
- // Discriminator: the metadata marker is the primary signal, but the body
1189
- // prefix is a header-loss fallback (S3 cross-region replication of data
1190
- // only, a console copy that drops Metadata, a metadata-stripping transport,
1191
- // or a poisoned regular-file re-upload of a sentinel). Honor BOTH — a
1192
- // marker-less object whose body starts with SYMLINK_BODY_PREFIX still
1193
- // rematerializes as a link instead of being written out as plain
1194
- // `hq-symlink:<target>` text, which would poison the key on the next push.
1195
- // The body is already buffered (tiny for symlink records); reading it here
1196
- // is behavior-preserving for regular files (whose body never starts with
1197
- // the prefix per the SYMLINK_BODY_PREFIX doc). See SYMLINK_BODY_PREFIX.
1198
- const leading = leadingBytes(
1199
- initialChunks,
1200
- initialLength,
1201
- SYMLINK_BODY_PREFIX_BYTES.length,
1202
- );
1203
- const bodyHasSymlinkPrefix =
1204
- leading.length >= SYMLINK_BODY_PREFIX_BYTES.length &&
1205
- leading.subarray(0, SYMLINK_BODY_PREFIX_BYTES.length).equals(
1206
- SYMLINK_BODY_PREFIX_BYTES,
1207
- );
1208
- const isSymlinkRecord =
1209
- (typeof symlinkMarker === "string" && symlinkMarker.length > 0) ||
1210
- bodyHasSymlinkPrefix;
1211
- if (isSymlinkRecord) {
1212
- const objectBody = await collectRemainingChunks(initialChunks, iterator);
1213
- // The target lives in the body (marker-only metadata convention).
1214
- // Symlink record bodies are bounded by target length (typically
1215
- // <300 bytes for relative paths, hard-capped by S3's 5 GB object
1216
- // size); only this branch buffers the body so regular files can stream.
1217
- let symlinkTarget: string;
1218
- if (bodyHasSymlinkPrefix) {
1219
- symlinkTarget = objectBody.toString(
1220
- "utf-8",
1221
- SYMLINK_BODY_PREFIX_BYTES.length,
1222
- );
1223
- } else {
1224
- // Backward-compat fallback: a legacy upload from earlier in
1225
- // this PR's lifetime stored the target in metadata (raw or
1226
- // base64) rather than the body. decodeSymlinkMetadataValue
1227
- // round-trip-validates so a raw value passes through and a
1228
- // base64 value decodes; either way we get the target. This branch
1229
- // is only reachable when the body lacks the prefix, which (given
1230
- // isSymlinkRecord) means the marker is present — the `?? ""` is a
1231
- // type guard for that invariant, and the length-0 check below
1232
- // catches the impossible empty case rather than passing it on.
1233
- symlinkTarget = decodeSymlinkMetadataValue(symlinkMarker ?? "");
1234
- }
1235
-
1236
- if (symlinkTarget.length === 0) {
1237
- throw new Error(
1238
- `Symlink record for ${key} had no target (body: ${objectBody.length} bytes, marker: ${symlinkMarker})`,
1239
- );
1240
- }
1241
-
1242
- // Idempotence fast path: the .claude/skills link farm makes symlink
1243
- // records the most re-pulled object class, and a pull that changes
1244
- // nothing must not touch disk at all (pre-fix, an already-correct
1245
- // Windows dir link still went through temp+rename, EPERM'd every
1246
- // pass, never journaled, and re-downloaded forever). On POSIX a
1247
- // target-string match is sufficient (lstat + readlink — never stat:
1248
- // dangling links are legitimate). On win32 a matching target string is
1249
- // NOT sufficient — links are flavored, and a file-type link at a
1250
- // directory target (minted by an older client, or while the target was
1251
- // absent) is broken despite the matching string. There the fast path
1252
- // additionally requires stat-follow to confirm the link FUNCTIONS as
1253
- // the type win32SymlinkType() would mint now; anything unconfirmable
1254
- // (including a dangling link, whose flavor cannot be probed) falls
1255
- // through to replacement, which is cheap and journals correctly.
1256
- const win32 = options.win32 ?? process.platform === "win32";
1257
- let existingLink: fs.Stats | null = null;
1258
- try {
1259
- existingLink = fs.lstatSync(localPath);
1260
- } catch {
1261
- // Absent (or unreadable) destination: no fast path; the normal
1262
- // materialization below surfaces real filesystem faults loudly.
1263
- }
1264
- if (existingLink?.isSymbolicLink()) {
1265
- let existingTarget: string | null = null;
1266
- try {
1267
- existingTarget = fs.readlinkSync(localPath);
1268
- } catch {
1269
- // Link raced away between lstat and readlink; replace it below.
1270
- }
1271
- if (
1272
- existingTarget !== null &&
1273
- symlinkTargetsEqual(existingTarget, symlinkTarget, localPath, win32)
1274
- ) {
1275
- if (!win32) return { metadata };
1276
- const desiredIsDirectory = win32StagedLinkIsDirectoryFlavored(
1277
- symlinkTarget,
1278
- localPath,
1279
- );
1280
- let followed: fs.Stats | null = null;
1281
- try {
1282
- followed = fs.statSync(localPath);
1283
- } catch {
1284
- // Unresolvable via this link (dangling, or a wrong-flavor link
1285
- // that Windows refuses to traverse): recreate below.
1286
- }
1287
- if (followed !== null && followed.isDirectory() === desiredIsDirectory) {
1288
- return { metadata };
1289
- }
1290
- }
1291
- }
1292
-
1293
- const tempPath = downloadTempPath(localPath);
1294
- try {
1295
- // Creation goes through createStagedSymlink (#219): on win32 a directory
1296
- // overlay becomes an unprivileged JUNCTION rather than a 'dir' NTFS
1297
- // symlink, which is what stops the EPERM that pushed the one-shot runner
1298
- // to PARTIAL_SYNC_EXIT. The staged link lands beside `localPath`, so its
1299
- // flavor decision resolves the target identically to the fast path above.
1300
- createStagedSymlink(symlinkTarget, tempPath, stagedSymlinkOpsFor(win32));
1301
- options.beforeReplace?.();
1302
- replaceStagedPath(tempPath, localPath);
1303
- } catch (err) {
1304
- removeTempPath(tempPath);
1305
- throw err;
1306
- }
1307
- return { metadata };
1308
- }
1309
- const tempPath = downloadTempPath(localPath);
1310
- // A legacy object has no hq-mode. If it is replacing a regular local file,
1311
- // that local mode is the only trustworthy signal we have — retaining it is
1312
- // strictly safer than replacing (for example) a 0755 hook with the receiver
1313
- // umask's usual 0644. Never inspect a symlink or infer intent from content.
1314
- let existingLocalMode: number | undefined;
1315
- try {
1316
- const existing = fs.lstatSync(localPath);
1317
- if (existing.isFile()) existingLocalMode = existing.mode & 0o777;
1318
- } catch {
1319
- // Fresh destination / inaccessible prior path: no local fallback.
1320
- }
1321
- let tempReady = true;
1322
- let streamed: { hash: string; size: number };
1323
- const modeWarnings: DownloadModeWarning[] = [];
1324
- try {
1325
- streamed = await streamRegularFileToTemp(tempPath, initialChunks, iterator);
1326
-
1327
- // Bug #5 — apply source-side mode after the byte write. See
1328
- // FILE_MODE_META_KEY for the metadata contract. The staged path is a
1329
- // regular file, then it is atomically renamed over the destination.
1330
- //
1331
- // Codex P2 (PR #24 round 3): strict octal-only regex BEFORE parseInt.
1332
- // parseInt(modeOctal, 8) accepts partial-prefix garbage — "755junk"
1333
- // parses to 0o755 instead of NaN — so tampered or malformed metadata
1334
- // could still change local permissions unexpectedly. The regex
1335
- // requires 1–4 pure octal digits (`[0-7]{1,4}$`), which matches what
1336
- // the upload side stamps (`(mode & 0o777).toString(8)` → at most
1337
- // three digits, all 0–7) and rejects everything else.
1338
- const modeOctal = metadata?.[FILE_MODE_META_KEY];
1339
- let modeToApply: number | undefined;
1340
- if (typeof modeOctal === "string" && /^[0-7]{1,4}$/.test(modeOctal)) {
1341
- const parsed = parseInt(modeOctal, 8);
1342
- if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 0o777) {
1343
- modeToApply = parsed;
1344
- }
1345
- } else if (modeOctal === undefined) {
1346
- modeToApply = existingLocalMode;
1347
- modeWarnings.push({
1348
- reason: "missing-hq-mode",
1349
- fallback:
1350
- existingLocalMode === undefined
1351
- ? "receiver-default"
1352
- : "preserved-local-mode",
1353
- });
1354
- } else {
1355
- modeWarnings.push({ reason: "invalid-hq-mode", fallback: "receiver-default" });
1356
- }
1357
-
1358
- if (modeToApply !== undefined) {
1359
- try {
1360
- fs.chmodSync(tempPath, modeToApply);
1361
- } catch {
1362
- // chmod failure (read-only FS, EPERM) is non-fatal — the file is
1363
- // materialized, but callers must receive a telemetry signal instead
1364
- // of silently believing the guardrail held.
1365
- modeWarnings.push({ reason: "chmod-failed" });
1366
- }
1367
- }
1368
-
1369
- // 5.37.0 — apply source-side mtime after the byte write (and after the
1370
- // chmod above; ordering between chmod and utimes doesn't matter, but
1371
- // both must run AFTER writeFileSync because writeFileSync resets mtime
1372
- // to wall-clock-now). See FILE_MTIME_META_KEY for the metadata contract.
1373
- //
1374
- // Strict-numeric regex BEFORE parseInt — same Codex P2 lesson as hq-mode.
1375
- // `^-?[0-9]{1,16}$` rejects partial-prefix garbage ("175junk" → 175),
1376
- // empty, double-signed, decimals, whitespace, and oversized strings. A
1377
- // single optional leading `-` is allowed so pre-epoch / reproducible-build
1378
- // timestamps round-trip (Codex PR #27 P2 — `mtimeMs === 0` and negative
1379
- // epoch values are legitimate). 16 digits comfortably covers any plausible
1380
- // epoch-ms value (year ~5138 is 16 digits; we'll cross that bridge later).
1381
- //
1382
- // The staged file is regular; rename preserves these stamps at localPath.
1383
- //
1384
- // Composition with the 5.36.0 lstat fast-path: the journal stamp at the
1385
- // share/sync call sites runs AFTER downloadFile returns, so the lstat
1386
- // it captures sees the post-utimes mtime. Verified in cli/sync.ts
1387
- // (downloadFile → lstatSync → updateEntry) and cli/share.ts (pull path
1388
- // similarly lstats after downloadFile). If a future caller stamps the
1389
- // journal BEFORE downloadFile completes, the fast-path will stale and
1390
- // re-hash every sync forever — keep the call-site invariant intact.
1391
- const mtimeRaw = metadata?.[FILE_MTIME_META_KEY];
1392
- if (typeof mtimeRaw === "string" && /^-?[0-9]{1,16}$/.test(mtimeRaw)) {
1393
- const mtimeMs = parseInt(mtimeRaw, 10);
1394
- if (Number.isFinite(mtimeMs)) {
1395
- try {
1396
- // utimesSync accepts seconds OR Date; use Date(ms) for precision.
1397
- // atime = mtime is fine — many filesystems are mounted noatime
1398
- // and distinguishing "access" vs "modification" time doesn't
1399
- // matter for sync semantics. Setting both keeps the on-disk
1400
- // state deterministic across receivers.
1401
- const mtimeDate = new Date(mtimeMs);
1402
- fs.utimesSync(tempPath, mtimeDate, mtimeDate);
1403
- } catch {
1404
- // EPERM / read-only FS / file just unlinked → non-fatal. The
1405
- // file is materialized at write-time mtime; the source-of-truth
1406
- // can re-sync next pass.
1407
- }
1408
- }
1409
- }
1410
-
1411
- options.beforeReplace?.();
1412
- replaceStagedPath(tempPath, localPath);
1413
- tempReady = false;
1414
- } finally {
1415
- if (tempReady) removeTempPath(tempPath);
1416
- }
1417
-
1418
- // TODO: stamp hq-btime once Node lands lbirthtime (or birthtimeSync).
1419
- // The push side already emits hq-btime when the source FS tracks a
1420
- // distinct creation time, so a future receiver upgrade picks it up
1421
- // automatically without a server-side data migration.
1422
-
1423
- if (modeWarnings.length > 0) {
1424
- for (const warning of modeWarnings) {
1425
- console.warn(
1426
- `[hq-sync] mode warning for ${key}: ${warning.reason}` +
1427
- (warning.fallback ? ` (${warning.fallback})` : ""),
1428
- );
1429
- }
1430
- }
1431
-
1432
- return {
1433
- metadata,
1434
- contentHash: streamed.hash,
1435
- contentSize: streamed.size,
1436
- ...(modeWarnings.length > 0 ? { modeWarnings } : {}),
1437
- };
1438
- }
1439
-
1440
- export interface RemoteFile {
1441
- key: string;
1442
- size: number;
1443
- lastModified: Date;
1444
- etag: string;
1445
- }
1446
-
1447
- export async function listRemoteFiles(
1448
- ctx: EntityContext,
1449
- prefix?: string,
1450
- ): Promise<RemoteFile[]> {
1451
- const io = resolveObjectIO(ctx);
1452
- const files: RemoteFile[] = [];
1453
- let continuationToken: string | undefined;
1454
-
1455
- do {
1456
- const page = await io.listObjects({ prefix, continuationToken });
1457
-
1458
- for (const obj of page.objects) {
1459
- // Drop S3 directory-marker objects: the canonical shape is `0-byte
1460
- // key ending in '/'` (S3 console "Create folder", `aws s3 sync` of
1461
- // empty dirs, sync tools that mirror empty trees). Two downstream
1462
- // sites blow up on them — pull planner (sync.ts: `hashFile` calls
1463
- // `readFileSync` on an existing local dir → EISDIR "read") and the
1464
- // download path (s3.ts: `writeFileSync` on a trailing-slash path
1465
- // → EISDIR "open" after the parent mkdir creates the leaf as a
1466
- // directory). Filtering here eliminates both.
1467
- //
1468
- // Narrow on size===0 (not just trailing-slash) so a hypothetical
1469
- // non-empty object whose key happens to end in '/' is NOT silently
1470
- // hidden — it stays visible and downloadFile surfaces the same
1471
- // EISDIR "open" error pointing at the specific key, which is the
1472
- // signal an operator needs to reconcile the bucket. The vault
1473
- // service doesn't have a code path that produces such an object,
1474
- // but the listing returns whatever lives in the bucket; silent
1475
- // drop would be worse than loud failure for that case.
1476
- //
1477
- // Real 0-byte placeholders like `.gitkeep` never end in `/` and
1478
- // continue to flow through — the 5.13.0 `.gitkeep` regression
1479
- // remains fixed. (The `!key` guard now lives in the ObjectIO layer.)
1480
- if (obj.key.endsWith("/") && obj.size === 0) continue;
1481
-
1482
- files.push({
1483
- key: obj.key,
1484
- size: obj.size,
1485
- lastModified: obj.lastModified,
1486
- etag: obj.etag,
1487
- });
1488
- }
1489
-
1490
- continuationToken = page.nextContinuationToken;
1491
- } while (continuationToken);
1492
-
1493
- return files;
1494
- }
1495
-
1496
- export async function deleteRemoteFile(
1497
- ctx: EntityContext,
1498
- key: string,
1499
- ): Promise<void> {
1500
- await resolveObjectIO(ctx).deleteObject(key);
1501
- }
1502
-
1503
- /**
1504
- * Check if a remote key exists and return its metadata.
1505
- */
1506
- export async function headRemoteFile(
1507
- ctx: EntityContext,
1508
- key: string,
1509
- ): Promise<{ lastModified: Date; etag: string; size: number; metadata?: Record<string, string> } | null> {
1510
- return resolveObjectIO(ctx).headObject(key);
1511
- }
1512
-
1513
- function getMimeType(filePath: string): string {
1514
- const ext = path.extname(filePath).toLowerCase();
1515
- const mimeTypes: Record<string, string> = {
1516
- ".md": "text/markdown",
1517
- ".json": "application/json",
1518
- ".yaml": "text/yaml",
1519
- ".yml": "text/yaml",
1520
- ".ts": "text/typescript",
1521
- ".js": "text/javascript",
1522
- ".txt": "text/plain",
1523
- ".html": "text/html",
1524
- ".css": "text/css",
1525
- ".png": "image/png",
1526
- ".jpg": "image/jpeg",
1527
- ".jpeg": "image/jpeg",
1528
- ".svg": "image/svg+xml",
1529
- ".pdf": "application/pdf",
1530
- };
1531
- return mimeTypes[ext] || "application/octet-stream";
1532
- }