@indigoai-us/hq-cloud 6.16.20 → 6.16.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/bin/sync-runner-installation-id.d.ts +66 -0
  2. package/dist/bin/sync-runner-installation-id.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-installation-id.js +138 -0
  4. package/dist/bin/sync-runner-installation-id.js.map +1 -0
  5. package/dist/bin/sync-runner-manifest.d.ts +120 -0
  6. package/dist/bin/sync-runner-manifest.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-manifest.js +348 -0
  8. package/dist/bin/sync-runner-manifest.js.map +1 -0
  9. package/dist/bin/sync-runner-manifest.test.d.ts +12 -0
  10. package/dist/bin/sync-runner-manifest.test.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-manifest.test.js +510 -0
  12. package/dist/bin/sync-runner-manifest.test.js.map +1 -0
  13. package/dist/bin/sync-runner.d.ts +55 -0
  14. package/dist/bin/sync-runner.d.ts.map +1 -1
  15. package/dist/bin/sync-runner.js +69 -0
  16. package/dist/bin/sync-runner.js.map +1 -1
  17. package/dist/bin/sync-runner.test.js +217 -0
  18. package/dist/bin/sync-runner.test.js.map +1 -1
  19. package/dist/company-slug.d.ts +31 -0
  20. package/dist/company-slug.d.ts.map +1 -0
  21. package/dist/company-slug.js +52 -0
  22. package/dist/company-slug.js.map +1 -0
  23. package/dist/company-slug.test.d.ts +7 -0
  24. package/dist/company-slug.test.d.ts.map +1 -0
  25. package/dist/company-slug.test.js +43 -0
  26. package/dist/company-slug.test.js.map +1 -0
  27. package/dist/index.d.ts +2 -0
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +5 -0
  30. package/dist/index.js.map +1 -1
  31. package/dist/journal-ledger-index.test.d.ts +15 -0
  32. package/dist/journal-ledger-index.test.d.ts.map +1 -0
  33. package/dist/journal-ledger-index.test.js +247 -0
  34. package/dist/journal-ledger-index.test.js.map +1 -0
  35. package/dist/journal.d.ts +110 -0
  36. package/dist/journal.d.ts.map +1 -1
  37. package/dist/journal.js +253 -5
  38. package/dist/journal.js.map +1 -1
  39. package/dist/manifest/US-002.acceptance.test.d.ts +11 -0
  40. package/dist/manifest/US-002.acceptance.test.d.ts.map +1 -0
  41. package/dist/manifest/US-002.acceptance.test.js +160 -0
  42. package/dist/manifest/US-002.acceptance.test.js.map +1 -0
  43. package/dist/manifest/US-004.acceptance.test.d.ts +12 -0
  44. package/dist/manifest/US-004.acceptance.test.d.ts.map +1 -0
  45. package/dist/manifest/US-004.acceptance.test.js +173 -0
  46. package/dist/manifest/US-004.acceptance.test.js.map +1 -0
  47. package/dist/manifest/build-manifest.bench.test.d.ts +22 -0
  48. package/dist/manifest/build-manifest.bench.test.d.ts.map +1 -0
  49. package/dist/manifest/build-manifest.bench.test.js +563 -0
  50. package/dist/manifest/build-manifest.bench.test.js.map +1 -0
  51. package/dist/manifest/build-manifest.d.ts +355 -0
  52. package/dist/manifest/build-manifest.d.ts.map +1 -0
  53. package/dist/manifest/build-manifest.js +1139 -0
  54. package/dist/manifest/build-manifest.js.map +1 -0
  55. package/dist/manifest/build-manifest.test.d.ts +10 -0
  56. package/dist/manifest/build-manifest.test.d.ts.map +1 -0
  57. package/dist/manifest/build-manifest.test.js +1172 -0
  58. package/dist/manifest/build-manifest.test.js.map +1 -0
  59. package/dist/manifest/contract.d.ts +356 -0
  60. package/dist/manifest/contract.d.ts.map +1 -0
  61. package/dist/manifest/contract.js +542 -0
  62. package/dist/manifest/contract.js.map +1 -0
  63. package/dist/manifest/contract.test.d.ts +17 -0
  64. package/dist/manifest/contract.test.d.ts.map +1 -0
  65. package/dist/manifest/contract.test.js +304 -0
  66. package/dist/manifest/contract.test.js.map +1 -0
  67. package/dist/manifest/import-graph.test.d.ts +25 -0
  68. package/dist/manifest/import-graph.test.d.ts.map +1 -0
  69. package/dist/manifest/import-graph.test.js +120 -0
  70. package/dist/manifest/import-graph.test.js.map +1 -0
  71. package/dist/manifest/ledger-shard-isolation.test.d.ts +18 -0
  72. package/dist/manifest/ledger-shard-isolation.test.d.ts.map +1 -0
  73. package/dist/manifest/ledger-shard-isolation.test.js +153 -0
  74. package/dist/manifest/ledger-shard-isolation.test.js.map +1 -0
  75. package/dist/manifest/snapshot-store.d.ts +160 -0
  76. package/dist/manifest/snapshot-store.d.ts.map +1 -0
  77. package/dist/manifest/snapshot-store.js +300 -0
  78. package/dist/manifest/snapshot-store.js.map +1 -0
  79. package/dist/manifest/snapshot-store.test.d.ts +11 -0
  80. package/dist/manifest/snapshot-store.test.d.ts.map +1 -0
  81. package/dist/manifest/snapshot-store.test.js +133 -0
  82. package/dist/manifest/snapshot-store.test.js.map +1 -0
  83. package/dist/manifest/upload-manifest.d.ts +336 -0
  84. package/dist/manifest/upload-manifest.d.ts.map +1 -0
  85. package/dist/manifest/upload-manifest.js +1022 -0
  86. package/dist/manifest/upload-manifest.js.map +1 -0
  87. package/dist/manifest/upload-manifest.test.d.ts +10 -0
  88. package/dist/manifest/upload-manifest.test.d.ts.map +1 -0
  89. package/dist/manifest/upload-manifest.test.js +1116 -0
  90. package/dist/manifest/upload-manifest.test.js.map +1 -0
  91. package/dist/manifest-reconcile.d.ts.map +1 -1
  92. package/dist/manifest-reconcile.js +1 -26
  93. package/dist/manifest-reconcile.js.map +1 -1
  94. package/dist/posix-key.d.ts +36 -0
  95. package/dist/posix-key.d.ts.map +1 -0
  96. package/dist/posix-key.js +38 -0
  97. package/dist/posix-key.js.map +1 -0
  98. package/dist/s3.d.ts +2 -18
  99. package/dist/s3.d.ts.map +1 -1
  100. package/dist/s3.js +6 -20
  101. package/dist/s3.js.map +1 -1
  102. package/dist/sync/journal-ledger-stream.d.ts +56 -0
  103. package/dist/sync/journal-ledger-stream.d.ts.map +1 -0
  104. package/dist/sync/journal-ledger-stream.js +191 -0
  105. package/dist/sync/journal-ledger-stream.js.map +1 -0
  106. package/dist/sync/journal-ledger-stream.test.d.ts +12 -0
  107. package/dist/sync/journal-ledger-stream.test.d.ts.map +1 -0
  108. package/dist/sync/journal-ledger-stream.test.js +134 -0
  109. package/dist/sync/journal-ledger-stream.test.js.map +1 -0
  110. package/package.json +2 -1
@@ -0,0 +1,1022 @@
1
+ /**
2
+ * Client-side manifest UPLOAD pass (sync-reconciliation-audit US-004).
3
+ *
4
+ * WHY THIS SEAM LIVES IN hq-cloud
5
+ * -------------------------------
6
+ * Two clients upload manifests — the desktop sync daemon (hq-sync) after a
7
+ * sync cycle, and `hq sync manifest` in the CLI — and the PRD's decision
8
+ * record settles it as "both, via shared hq-cloud code". The rules that make
9
+ * an upload SAFE are not obvious and must not be re-derived twice:
10
+ *
11
+ * - a snapshot may become the local delta BASE only after the server says
12
+ * `materialised: true` (a `completed` fold that was refused is not a base);
13
+ * - the delta base is the snapshotId the SERVER returns, never the local
14
+ * `snap-*` handle the builder minted (the server derives its own id);
15
+ * - `sequence` must strictly increase per (installation, scope), including
16
+ * across passes that failed before they uploaded anything;
17
+ * - every chunk of one pass must be byte-identical in the fields the server
18
+ * hashes into the snapshotId (`generatedAt`, `sequence`, `chunkCount`,
19
+ * `machineId`, `mode`, `scope`);
20
+ * - a `resend_full` at ANY chunk ends the pass and invalidates the base.
21
+ *
22
+ * Two implementations of those rules would drift, and the failure mode of the
23
+ * drift is silent server-side corruption of a scope's materialised view — the
24
+ * exact thing this audit exists to detect.
25
+ *
26
+ * WHAT THIS DELIBERATELY DOES NOT OWN
27
+ * -----------------------------------
28
+ * - **Auth and the base URL.** The transport is INJECTED. hq-cloud has no
29
+ * business deciding which API host a daemon talks to or how a JWT is
30
+ * minted; the clients already own both, and owning them here would force
31
+ * this module to import the vault client (and with it the AWS SDK) into
32
+ * what is otherwise a lean, hot-path module graph.
33
+ * - **Scheduling.** The caller decides when a pass runs; this function only
34
+ * refuses to run one too often (see the 24h throttle).
35
+ * - **Wiring.** The hq-cli `sync manifest` command, its `hq doctor` check,
36
+ * and the hq-sync daemon hook land SEPARATELY, in those repos, on top of
37
+ * {@link runManifestUploadPass} and {@link readManifestUploadStatus}.
38
+ *
39
+ * FAILURE POSTURE
40
+ * ---------------
41
+ * This never throws at the caller for anything that happens at runtime. The
42
+ * daemon calls it after a sync pass and must continue regardless: an audit
43
+ * that can break sync is strictly worse than no audit. Build errors, transport
44
+ * rejections, aborts and unexpected statuses all come back as
45
+ * `{ status: "failed" }`. Only invalid OPTIONS — a programmer error, not a
46
+ * runtime condition — throw.
47
+ */
48
+ import { buildSyncManifest, } from "./build-manifest.js";
49
+ // A per-scope mutex, not a new dependency: `operation-lock.ts` imports nothing
50
+ // but node builtins, so serialising the pass costs the module graph nothing.
51
+ // (It is not reachable from `build-manifest.ts` either, so the import-graph
52
+ // test's closure is unaffected.)
53
+ import { OperationLockedError, acquireOperationLock, } from "../operation-lock.js";
54
+ import { SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET, SYNC_MANIFEST_TOO_LARGE_REASON_CODE, } from "./contract.js";
55
+ import { MANIFEST_SNAPSHOT_MAX_AGE_MS, isManifestSnapshotBaseUsable, isManifestSnapshotStale, manifestSnapshotScopeKey, manifestSnapshotScopeSlug, readManifestSnapshot, writeManifestSnapshot, } from "./snapshot-store.js";
56
+ // ─── Kill switch / throttle ──────────────────────────────────────────────────
57
+ /**
58
+ * Env var that disables manifest uploads entirely. A user (or a support
59
+ * engineer) must be able to switch this feature off on a machine WITHOUT
60
+ * downgrading hq-cloud, and the switch must short-circuit before the walk —
61
+ * the walk, not the POST, is the expensive half.
62
+ */
63
+ export const MANIFEST_UPLOAD_DISABLED_ENV = "HQ_SYNC_MANIFEST_DISABLED";
64
+ /**
65
+ * Default minimum spacing between upload passes for one scope.
66
+ *
67
+ * The audit is a slow-moving safety net, not telemetry: a daily picture is
68
+ * enough to catch a wedged watcher, and a 70k-file walk per sync cycle is not
69
+ * something a laptop should pay for.
70
+ */
71
+ export const DEFAULT_MANIFEST_UPLOAD_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
72
+ /**
73
+ * Wall-clock budget for the ENTIRE upload phase (all chunks), after which the
74
+ * pass stops sending and reports a non-throwing failure.
75
+ *
76
+ * Why the whole phase and not per chunk: the caller's actual requirement is
77
+ * "the daemon's `await` returns" — the desktop daemon runs this after every
78
+ * sync pass, and a transport that never settles would otherwise wedge the pass
79
+ * (and the sync cycle behind it) forever. A per-chunk budget would not bound
80
+ * that: a 30-chunk manifest would still admit 30 x budget of hanging. One
81
+ * budget for the phase bounds the pass by construction, whatever the chunk
82
+ * count.
83
+ *
84
+ * 60s is generous next to a single small JSON POST and mean next to a sync
85
+ * cycle, so it only ever fires on a genuinely wedged transport — never on a
86
+ * merely slow link, where the pass would be retried next cycle anyway.
87
+ *
88
+ * Note this bounds only the UPLOAD; `maxWallMs` bounds the filesystem walk.
89
+ * They are separate budgets because they fail for unrelated reasons (a slow
90
+ * disk vs. a black-holed socket) and a caller may reasonably tune one alone.
91
+ */
92
+ export const DEFAULT_MANIFEST_UPLOAD_WALL_MS = 60_000;
93
+ /**
94
+ * First retry floor after a pass FAILED without reaching the server.
95
+ *
96
+ * The 24h throttle is armed by `lastUploadAt`, which only a pass that actually
97
+ * reached the server writes. That is correct — a network blip must not
98
+ * suppress the audit for a day — but taken alone it means a permanently broken
99
+ * endpoint (a wrong base URL, an unenrolled installation answering 403, a 5xx
100
+ * outage) buys a FULL tree walk, and under `hashPolicy: "auto"` a full content
101
+ * hash, on EVERY sync cycle. On a large vault that is minutes of IO and
102
+ * gigabytes of RSS per cycle, forever, for an audit that cannot succeed.
103
+ *
104
+ * So a failed pass arms its own, much shorter floor, doubling per consecutive
105
+ * failure up to {@link MANIFEST_FAILURE_BACKOFF_MAX_MS}. One hour keeps a
106
+ * transient outage cheap to recover from (the next hourly cycle retries) while
107
+ * capping a hard failure's cost at ~1 walk/hour instead of 1 walk/cycle.
108
+ */
109
+ export const MANIFEST_FAILURE_BACKOFF_BASE_MS = 60 * 60 * 1000;
110
+ /** Ceiling for the failure backoff — the same daily spacing a healthy scope gets. */
111
+ export const MANIFEST_FAILURE_BACKOFF_MAX_MS = 24 * 60 * 60 * 1000;
112
+ /**
113
+ * Share of `source: "disk"` entries above which a FULL manifest is treated as
114
+ * an implausible picture of a scope that was previously tracked.
115
+ *
116
+ * Set hard against 1.0 rather than at a "suspicious" level like 0.9 on
117
+ * purpose. A legitimate manifest CAN be very disk-heavy — a user who just
118
+ * restored a backup, or a first pass after a journal repair — and refusing
119
+ * those would suppress the audit exactly when it is most useful. The failure
120
+ * this guards is not "mostly disk": it is "the ledger vanished", which is
121
+ * total. 0.995 leaves room for the handful of `ledger`/`both` entries a
122
+ * partially-readable ledger still contributes without admitting a merely
123
+ * lopsided scope.
124
+ */
125
+ export const MANIFEST_IMPLAUSIBLE_DISK_RATIO = 0.995;
126
+ /**
127
+ * Floor the 413 self-heal will not halve below.
128
+ *
129
+ * The halving exists to rescue a scope whose entries are fatter than the
130
+ * default budget assumed; it is not a search for the smallest body the server
131
+ * will take. Below ~256 KiB the chunk count for a large scope climbs past
132
+ * `SYNC_MANIFEST_MAX_CHUNK_COUNT` and the pass would start failing for a
133
+ * second, unrelated reason — and a 413 that survives a budget this small is
134
+ * not a sizing problem at all (a proxy with its own tiny limit, a gateway
135
+ * refusing the route), so halving further would just burn passes hiding it.
136
+ */
137
+ export const MANIFEST_CHUNK_BYTE_BUDGET_FLOOR = 256 * 1024;
138
+ /**
139
+ * The budget to plan the NEXT pass against after this one was refused 413.
140
+ *
141
+ * Halving, not a fixed step, because the client has no information about how
142
+ * far over the line it was: API Gateway's 413 is opaque, and even the
143
+ * server's own rejection reports its ceiling rather than the body's size. A
144
+ * geometric retreat reaches any workable budget in a handful of passes from
145
+ * any starting point, and it converges — a linear step sized for the common
146
+ * case is either too slow for the bad case or overshoots the common one.
147
+ */
148
+ export function nextManifestChunkByteBudget(current) {
149
+ const base = Number.isFinite(current) && current > 0 ? current : SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET;
150
+ return Math.max(MANIFEST_CHUNK_BYTE_BUDGET_FLOOR, Math.floor(base / 2));
151
+ }
152
+ /**
153
+ * Retry floor for a scope that has failed `consecutiveFailures` times in a row.
154
+ *
155
+ * `1 → 1h, 2 → 2h, 3 → 4h … capped at 24h`. Zero (or anything nonsensical)
156
+ * means "no floor": a scope with no failure history is governed by the normal
157
+ * `lastUploadAt` throttle alone.
158
+ */
159
+ export function manifestFailureBackoffMs(consecutiveFailures) {
160
+ if (!Number.isFinite(consecutiveFailures) || consecutiveFailures < 1)
161
+ return 0;
162
+ // Clamped BEFORE the shift: `2 ** 1024` is Infinity, and an unbounded
163
+ // exponent on a corrupt count would produce a floor that never expires.
164
+ const steps = Math.min(Math.floor(consecutiveFailures), 20) - 1;
165
+ return Math.min(MANIFEST_FAILURE_BACKOFF_BASE_MS * 2 ** steps, MANIFEST_FAILURE_BACKOFF_MAX_MS);
166
+ }
167
+ /**
168
+ * Only an explicit, unambiguous value disables. Empty string, `0`, `false` and
169
+ * anything else mean "not set": env vars are routinely exported empty by
170
+ * shell wrappers, and treating that as "off" would silently disable the
171
+ * feature on machines nobody meant to opt out.
172
+ */
173
+ /**
174
+ * The kill switch, readable by a CALLER.
175
+ *
176
+ * `runManifestUploadPass` enforces it itself, but a caller that assembles the
177
+ * pass (resolving identity, minting a transport, choosing scopes) needs to
178
+ * check it BEFORE doing any of that, so an opted-out machine pays literally
179
+ * nothing. Exported rather than duplicated so the two can never disagree about
180
+ * what counts as "disabled".
181
+ */
182
+ export function isManifestUploadDisabled(env) {
183
+ return isUploadDisabled(env);
184
+ }
185
+ function isUploadDisabled(env) {
186
+ const raw = env[MANIFEST_UPLOAD_DISABLED_ENV];
187
+ if (typeof raw !== "string")
188
+ return false;
189
+ const normalised = raw.trim().toLowerCase();
190
+ return normalised === "1" || normalised === "true";
191
+ }
192
+ // ─── Response reading ────────────────────────────────────────────────────────
193
+ function asRecord(body) {
194
+ if (body === null || typeof body !== "object" || Array.isArray(body))
195
+ return {};
196
+ return body;
197
+ }
198
+ /**
199
+ * The server answers `resend_full` as a 200 with `status: "resend_full"` — a
200
+ * SUCCESSFUL HTTP call carrying a protocol-level "start over". It is checked
201
+ * before the 2xx path for exactly that reason; reading only the status code
202
+ * would record a base the server has already thrown away.
203
+ */
204
+ function readsResendFull(status, body) {
205
+ // Only a 2xx carries the protocol answer. A non-2xx is a transport/server
206
+ // FAILURE and must take the failure path: the server's own 409 means
207
+ // VAULT_NOT_PROVISIONED, and reading that as "start over" would report a
208
+ // hard misconfiguration as a scheduled resend AND arm the 24h throttle on
209
+ // it, hiding the fault for a day per attempt.
210
+ if (status < 200 || status >= 300)
211
+ return false;
212
+ const raw = asRecord(body);
213
+ if (raw.status === "resend_full")
214
+ return true;
215
+ return raw.action === "resend_full" || raw.reason === "resend_full" || raw.resendFull === true;
216
+ }
217
+ function readString(body, key) {
218
+ const value = asRecord(body)[key];
219
+ return typeof value === "string" ? value : undefined;
220
+ }
221
+ function readBoolean(body, key) {
222
+ return asRecord(body)[key] === true;
223
+ }
224
+ /** Bare error class name — never the message, which routinely carries a path. */
225
+ function errorClassOf(err) {
226
+ const name = err instanceof Error ? err.constructor?.name ?? err.name : typeof err;
227
+ const cleaned = String(name).replace(/[^A-Za-z0-9_]/g, "");
228
+ if (!cleaned || !/^[A-Za-z]/.test(cleaned))
229
+ return "UnknownError";
230
+ return cleaned.slice(0, 64);
231
+ }
232
+ /**
233
+ * Identity fields the server rejects outright (400
234
+ * CALLER_SELECTED_IDENTITY_REJECTED): the caller does not get to say who it
235
+ * is, the JWT does. The builder never emits these, so this is a cheap
236
+ * belt-and-braces check against a future field being added in the wrong place
237
+ * — it fails the pass locally instead of tripping a server 400 storm.
238
+ */
239
+ const FORBIDDEN_IDENTITY_KEYS = ["personUid", "person_uid", "ownerUid", "callerSub"];
240
+ function findCallerSelectedIdentity(chunk) {
241
+ const raw = chunk;
242
+ return FORBIDDEN_IDENTITY_KEYS.find((key) => raw[key] !== undefined);
243
+ }
244
+ // ─── Snapshot bookkeeping ────────────────────────────────────────────────────
245
+ /**
246
+ * Record that a pass HAPPENED without promoting its snapshot to the delta
247
+ * base.
248
+ *
249
+ * Two invariants ride on this one write:
250
+ *
251
+ * 1. `sequence` must strictly increase per (installation, scope) even when
252
+ * the pass did not produce a base — otherwise a machine that keeps
253
+ * failing to materialise would replay the same sequence forever, and the
254
+ * server derives its snapshotId from it.
255
+ * 2. the 24h throttle must still fire, so a server that answers 404 or 413
256
+ * on every pass cannot turn into a per-cycle full tree walk.
257
+ *
258
+ * The PREVIOUS entries are preserved when they were usable and the server did
259
+ * not ask us to drop them: a refused fold is not evidence that the base the
260
+ * server already holds is wrong.
261
+ */
262
+ function recordUploadAttempt(stateDir, scopeKey, previous, sequence, nowIso, options) {
263
+ const keepBase = !options.invalidateBase && previous !== null && isManifestSnapshotBaseUsable(previous);
264
+ const base = keepBase
265
+ ? { ...previous, sequence, lastUploadAt: nowIso }
266
+ : {
267
+ // A bookkeeping-only record: `baseValid: false` makes it unusable as a
268
+ // delta base while keeping the sequence and the upload time, which a
269
+ // plain delete would throw away.
270
+ snapshotId: previous?.snapshotId ?? "snap-none",
271
+ generatedAt: previous?.generatedAt ?? nowIso,
272
+ sequence,
273
+ entries: {},
274
+ lastUploadAt: nowIso,
275
+ baseValid: false,
276
+ };
277
+ // This function is only ever called on a path where the server ANSWERED
278
+ // (resend_full, 404, 413, an accepted-but-unmaterialised fold). That is
279
+ // proof the endpoint is reachable, so the failure backoff must be cleared —
280
+ // otherwise a scope that failed twice last week would keep a stale retry
281
+ // floor armed against passes that are now succeeding.
282
+ const { lastAttemptAt: _lastAttemptAt, consecutiveFailures: _failures, ...record } = base;
283
+ return writeManifestSnapshot(stateDir, scopeKey, record);
284
+ }
285
+ /**
286
+ * Record a pass that FAILED before the server could answer (or whose answer
287
+ * was itself a failure): burn the sequence, and arm the retry floor.
288
+ *
289
+ * Deliberately NOT `lastUploadAt` — that field means "the server has a picture
290
+ * from this time", and writing it here would claim an upload that never
291
+ * happened, hiding a broken endpoint behind a healthy-looking `hq doctor`.
292
+ */
293
+ function recordFailedAttempt(stateDir, scopeKey, previous, sequence, nowIso, options) {
294
+ const record = previous
295
+ ? { ...previous, sequence }
296
+ : { snapshotId: "snap-none", generatedAt: nowIso, sequence, entries: {}, baseValid: false };
297
+ record.lastAttemptAt = nowIso;
298
+ record.consecutiveFailures = (previous?.consecutiveFailures ?? 0) + 1;
299
+ if (options?.chunkByteBudget !== undefined)
300
+ record.chunkByteBudget = options.chunkByteBudget;
301
+ return writeManifestSnapshot(stateDir, scopeKey, record);
302
+ }
303
+ // ─── The pass ────────────────────────────────────────────────────────────────
304
+ // ─── Per-scope serialisation ─────────────────────────────────────────────────
305
+ /**
306
+ * The `command` recorded in the lock file. It shows up verbatim in
307
+ * `OperationLockedError` messages and in `hq doctor`-style lock dumps, so it
308
+ * names the operation a human would recognise.
309
+ */
310
+ export const MANIFEST_UPLOAD_LOCK_COMMAND = "sync-manifest";
311
+ /**
312
+ * Lock scope for one manifest scope.
313
+ *
314
+ * `sequence = previous + 1` is a read-modify-write across the whole pass: the
315
+ * read happens before the tree walk and the write happens after the last
316
+ * chunk. Nothing else spaces two passes out — `hq sync manifest` sets
317
+ * `ignoreThrottle`, so a human invocation lands squarely inside a daemon
318
+ * pass's window — and two passes that derive the same `sequence` with the same
319
+ * `generatedAt` ms and chunk count make the SERVER derive one snapshotId and
320
+ * fold both passes' chunk sets into it. That is exactly the silent
321
+ * cross-contamination this audit exists to detect, manufactured by the audit.
322
+ *
323
+ * The scope is prefixed so it can never collide with the `operation` /
324
+ * `reindex` gates, and carries the snapshot store's injective slug so two
325
+ * tenants can never share one lock.
326
+ */
327
+ function manifestUploadLockScope(scopeKey) {
328
+ return `manifest-${manifestSnapshotScopeSlug(scopeKey)}`;
329
+ }
330
+ /**
331
+ * Take the per-scope gate, or report that someone else holds it.
332
+ *
333
+ * Three deliberate choices:
334
+ *
335
+ * - **Never wait** (`wait: false`). The daemon calls this after every sync
336
+ * cycle; queueing behind a `hq sync manifest` that is mid-walk would block
337
+ * the sync pass on an audit, which inverts the priority. A contended pass
338
+ * is simply skipped and retried next cycle — the audit is a daily
339
+ * safety net, not a deadline.
340
+ * - **Never throw.** `OperationLockedError` becomes a benign `locked`
341
+ * result; any OTHER failure (an unwritable `~/.hq`, a pathological
342
+ * reclaim race) is logged and the pass proceeds UNSERIALISED. Refusing to
343
+ * upload because a lock file could not be created would let a
344
+ * permissions problem silently disable the audit; proceeding is no worse
345
+ * than the behaviour before this gate existed.
346
+ * - **Stale holders self-heal.** `acquireOperationLock` probes the holder's
347
+ * pid and reclaims the lock when the process is gone, so a machine that
348
+ * was hard-killed mid-pass is not wedged off the audit forever.
349
+ */
350
+ function acquireScopeLock(options, scopeKey) {
351
+ try {
352
+ return acquireOperationLock(options.hqRoot, MANIFEST_UPLOAD_LOCK_COMMAND, {
353
+ scope: manifestUploadLockScope(scopeKey),
354
+ mode: "exclusive",
355
+ wait: false,
356
+ timeoutSec: 0,
357
+ });
358
+ }
359
+ catch (err) {
360
+ if (err instanceof OperationLockedError)
361
+ return "locked";
362
+ options.logger?.warn?.("manifest upload lock unavailable; proceeding unserialised", {
363
+ scopeKey,
364
+ error: errorClassOf(err),
365
+ });
366
+ return null;
367
+ }
368
+ }
369
+ /**
370
+ * Await one transport call, but never unconditionally.
371
+ *
372
+ * The transport is INJECTED, so this module cannot assume it has a timeout —
373
+ * and the one caller that matters (the desktop daemon) `await`s this pass after
374
+ * every sync cycle. A transport that never settles must therefore not be able
375
+ * to wedge the pass: it is raced against the caller's abort signal and against
376
+ * the upload phase's deadline, and whichever answers first wins.
377
+ *
378
+ * Losing the race does NOT cancel the call — this module has no cancellation
379
+ * channel to an injected function — so the abandoned promise is given a no-op
380
+ * rejection handler up front. Without it, a wedged transport that eventually
381
+ * rejects would surface as an unhandled rejection long after the pass returned,
382
+ * which on a daemon is a crash, not a log line.
383
+ *
384
+ * The timer is cleared in a `finally` on every path (and unref'd besides) so a
385
+ * pending deadline can never keep a short-lived CLI process alive.
386
+ */
387
+ async function raceTransportCall(call, signal, deadlineMs) {
388
+ const settled = call.then((response) => ({ kind: "settled", response }));
389
+ settled.catch(() => {
390
+ /* see above: the race may abandon this promise; its rejection is not fatal */
391
+ });
392
+ if (signal?.aborted)
393
+ return { kind: "aborted" };
394
+ const remainingMs = deadlineMs - Date.now();
395
+ if (remainingMs <= 0)
396
+ return { kind: "deadline" };
397
+ let timer;
398
+ let onAbort;
399
+ try {
400
+ return await Promise.race([
401
+ settled,
402
+ new Promise((resolve) => {
403
+ timer = setTimeout(() => resolve({ kind: "deadline" }), remainingMs);
404
+ timer.unref?.();
405
+ }),
406
+ new Promise((resolve) => {
407
+ if (!signal)
408
+ return;
409
+ onAbort = () => resolve({ kind: "aborted" });
410
+ signal.addEventListener("abort", onAbort, { once: true });
411
+ }),
412
+ ]);
413
+ }
414
+ finally {
415
+ if (timer !== undefined)
416
+ clearTimeout(timer);
417
+ if (signal !== undefined && onAbort !== undefined)
418
+ signal.removeEventListener("abort", onAbort);
419
+ }
420
+ }
421
+ function assertOption(condition, message) {
422
+ if (!condition)
423
+ throw new Error(`runManifestUploadPass: ${message}`);
424
+ }
425
+ /**
426
+ * Build and upload one scope's manifest.
427
+ *
428
+ * Chunks go up SEQUENTIALLY in ascending `chunkIndex`: the server folds them
429
+ * into one snapshot keyed by a hash of the shared header fields, and a
430
+ * concurrent send would make a partial-failure state (some chunks in, some
431
+ * not) impossible to reason about from the client side.
432
+ */
433
+ export async function runManifestUploadPass(options) {
434
+ assertOption(typeof options.hqRoot === "string" && options.hqRoot.length > 0, "hqRoot is required");
435
+ assertOption(typeof options.stateDir === "string" && options.stateDir.length > 0, "stateDir is required");
436
+ assertOption(typeof options.installationId === "string" && options.installationId.length > 0, "installationId is required");
437
+ assertOption(typeof options.machineId === "string" && options.machineId.length > 0, "machineId is required");
438
+ assertOption(typeof options.transport === "function", "transport is required");
439
+ // Throws for a malformed scope (a company scope with no companyUid) — that
440
+ // is a programmer error, and it must not be silently swallowed into a
441
+ // "failed" result that looks like a transient server problem.
442
+ const scopeKey = manifestSnapshotScopeKey(options.scope);
443
+ /**
444
+ * A failed snapshot write is NOT cosmetic, so it must never be silent.
445
+ *
446
+ * `writeManifestSnapshot` swallows disk errors and returns `false`. If that
447
+ * happens the two invariants that ride on the file both break at once, and
448
+ * both break INVISIBLY: `lastUploadAt` is never stored, so the 24h throttle
449
+ * stops firing and every sync cycle pays a full tree walk; and `sequence` is
450
+ * never stored, so the next pass REPLAYS the sequence this one already sent
451
+ * to the server. Log it loudly, and let callers of the success path see it
452
+ * via `nextModeFull`.
453
+ */
454
+ const persisted = (ok, what) => {
455
+ if (!ok) {
456
+ options.logger?.error?.("manifest snapshot could not be persisted", { scopeKey, what });
457
+ }
458
+ return ok;
459
+ };
460
+ const env = options.env ?? process.env;
461
+ if (isUploadDisabled(env)) {
462
+ // Deliberately BEFORE the build: the tree walk is the expensive half, so a
463
+ // kill switch that still walked would not be much of a kill switch.
464
+ options.logger?.info?.("manifest upload disabled by env", { scopeKey });
465
+ return { status: "disabled", scopeKey };
466
+ }
467
+ // `dryRun` is not serialised: it uploads nothing, persists nothing and burns
468
+ // no sequence, so it has nothing to race with. Taking the gate for a
469
+ // `--print` would only make a concurrent daemon pass skip a cycle for the
470
+ // sake of a read-only build.
471
+ const lock = options.dryRun ? null : acquireScopeLock(options, scopeKey);
472
+ if (lock === "locked") {
473
+ options.logger?.info?.("manifest upload skipped; another pass holds this scope", { scopeKey });
474
+ return { status: "locked", scopeKey };
475
+ }
476
+ try {
477
+ return await runUploadPassBody(options, scopeKey, persisted);
478
+ }
479
+ finally {
480
+ // Released on EVERY path, including a throw out of the body: a pass that
481
+ // crashed while holding the gate would otherwise lock this scope out of
482
+ // the audit until the process exited.
483
+ lock?.release();
484
+ }
485
+ }
486
+ /**
487
+ * The pass proper, running under the per-scope gate.
488
+ *
489
+ * Split out purely so the gate can be held across the whole read-modify-write
490
+ * of `sequence` — the read is before the walk, the write is after the last
491
+ * chunk — with a single `finally` that cannot be forgotten.
492
+ */
493
+ async function runUploadPassBody(options, scopeKey, persisted) {
494
+ const nowMs = options.now ?? Date.now();
495
+ const nowIso = new Date(nowMs).toISOString();
496
+ const previous = readManifestSnapshot(options.stateDir, scopeKey);
497
+ const minIntervalMs = options.minIntervalMs ?? DEFAULT_MANIFEST_UPLOAD_MIN_INTERVAL_MS;
498
+ // `dryRun` bypasses the throttle because printing is read-only: it uploads
499
+ // nothing and persists nothing, so there is no cost to spread out.
500
+ if (!options.ignoreThrottle && !options.dryRun && previous?.lastUploadAt) {
501
+ const lastMs = Date.parse(previous.lastUploadAt);
502
+ // A timestamp in the FUTURE (clock skew, restored backup) is not trusted
503
+ // to throttle — it would otherwise wedge uploads off for as long as the
504
+ // skew lasts.
505
+ if (Number.isFinite(lastMs) && lastMs <= nowMs && nowMs - lastMs < minIntervalMs) {
506
+ return { status: "throttled", scopeKey };
507
+ }
508
+ }
509
+ // The FAILURE floor, checked separately because it is armed by a different
510
+ // field. Without it, a scope that can never succeed (403 from an unenrolled
511
+ // installation, a 5xx outage, a misconfigured base URL) never writes
512
+ // `lastUploadAt`, so the throttle above never fires and every single sync
513
+ // cycle pays for a full tree walk — the single most expensive thing this
514
+ // module can do — to produce an upload that is guaranteed to fail again.
515
+ if (!options.ignoreThrottle && !options.dryRun && previous?.lastAttemptAt) {
516
+ const floorMs = manifestFailureBackoffMs(previous.consecutiveFailures ?? 0);
517
+ const lastMs = Date.parse(previous.lastAttemptAt);
518
+ // Same future-timestamp guard as above: clock skew must not wedge retries.
519
+ if (floorMs > 0 && Number.isFinite(lastMs) && lastMs <= nowMs && nowMs - lastMs < floorMs) {
520
+ options.logger?.info?.("manifest upload backing off after a failed pass", {
521
+ scopeKey,
522
+ consecutiveFailures: previous.consecutiveFailures ?? 0,
523
+ floorMs,
524
+ });
525
+ return { status: "throttled", scopeKey };
526
+ }
527
+ }
528
+ // A base that is missing, stale (weekly re-baseline) or explicitly
529
+ // invalidated forces a FULL manifest. Computing it here rather than letting
530
+ // the builder decide keeps the "why was this full?" answer in one place, and
531
+ // lets the result report it.
532
+ const baseUnusable = previous === null || !isManifestSnapshotBaseUsable(previous) || isManifestSnapshotStale(previous, nowMs);
533
+ const forceFull = options.forceFull === true || baseUnusable;
534
+ /**
535
+ * What `nextModeFull` must report on any path that leaves the stored base
536
+ * ALONE (transport blip, abort, unexpected status, 404 soft skip).
537
+ *
538
+ * Nothing reached the server's materialised view on those paths, so the base
539
+ * it already holds is still the right one to send a delta against — and
540
+ * hard-coding `true` there would tell the caller to force a full 70k-file
541
+ * re-send on every network hiccup. Conversely, if there was no usable base
542
+ * to begin with, the next pass IS full and must say so.
543
+ */
544
+ const nextModeFullIfBaseUntouched = baseUnusable;
545
+ const sequence = (previous?.sequence ?? 0) + 1;
546
+ /**
547
+ * The byte budget this pass plans chunks against.
548
+ *
549
+ * Precedence is explicit-option → self-healed value on the snapshot →
550
+ * contract default. The middle term is the whole point: a 413 halves the
551
+ * budget and PERSISTS it, so the recovery survives the process exiting. A
552
+ * budget that lived only in memory would re-fail identically every cycle,
553
+ * which is exactly the 6.16.21 behaviour this replaces.
554
+ */
555
+ const chunkByteBudget = options.chunkByteBudget ?? previous?.chunkByteBudget ?? SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET;
556
+ let built;
557
+ try {
558
+ built = await buildSyncManifest({
559
+ scope: options.scope,
560
+ hqRoot: options.hqRoot,
561
+ stateDir: options.stateDir,
562
+ installationId: options.installationId,
563
+ machineId: options.machineId,
564
+ source: options.source,
565
+ sequence,
566
+ now: nowMs,
567
+ forceFull,
568
+ abortSignal: options.abortSignal,
569
+ maxWallMs: options.maxWallMs,
570
+ hashPolicy: options.hashPolicy,
571
+ maxHashedFiles: options.maxHashedFiles,
572
+ maxHashMs: options.maxHashMs,
573
+ listJournalsImpl: options.listJournalsImpl,
574
+ maxEntriesPerChunk: options.maxEntriesPerChunk,
575
+ chunkByteBudget,
576
+ // The builder must NEVER persist the snapshot itself: a snapshot is a
577
+ // valid delta base only once the SERVER has materialised it, and the
578
+ // builder cannot know that. This pass persists it, later, under gate.
579
+ persistSnapshot: false,
580
+ });
581
+ }
582
+ catch (err) {
583
+ options.logger?.warn?.("manifest build failed", { scopeKey, error: errorClassOf(err) });
584
+ // The sequence advances on EVERY outcome, including one that never
585
+ // reached the wire. Keeping the rule unconditional is what makes it
586
+ // auditable: an exception to it is a place where a future edit that DOES
587
+ // upload something can silently reuse a sequence.
588
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
589
+ return {
590
+ status: "failed",
591
+ scopeKey,
592
+ nextModeFull: nextModeFullIfBaseUntouched,
593
+ error: { kind: "build_error", detail: errorClassOf(err) },
594
+ };
595
+ }
596
+ const { mode, snapshot, chunkPlan } = built;
597
+ // Frozen for the whole pass: the server hashes `chunkCount` into the
598
+ // snapshotId, so it must be identical in every chunk header. Reading it once
599
+ // here — rather than `chunks.length` at each use — is also what lets the
600
+ // chunk set stay unmaterialised.
601
+ const chunkCount = chunkPlan.chunkCount;
602
+ // A TRUNCATED walk must never be uploaded. This is not a refinement of the
603
+ // delta-base rule — it is a correctness rule about the wire.
604
+ //
605
+ // The builder already suppresses `removedPaths` and the snapshot write when
606
+ // the walk ran out of wall clock, which was thought to make a truncated pass
607
+ // safe. It does not. On a real 690k-file personal vault the 30s walk budget
608
+ // expired after ~179k files with ZERO entries assembled (the hash phase had
609
+ // not started), and the pass happily POSTed that as a FULL manifest. A full
610
+ // manifest is a complete statement — "this is everything I hold" — so the
611
+ // server's materialised view for the scope becomes empty and every file the
612
+ // client actually holds reads as missing. The delta-base guard cannot help:
613
+ // the damage is done by the upload itself, server-side.
614
+ //
615
+ // So: no POST, no base, no `lastUploadAt`. The sequence is burned (cheap and
616
+ // keeps the monotonic rule unconditional) and the failure backoff is armed
617
+ // so a scope that cannot be walked inside its budget does not re-walk on
618
+ // every cycle. `dryRun` still prints — a truncated manifest is exactly what
619
+ // an operator debugging this needs to see.
620
+ if (built.walkStats.truncated && !options.dryRun) {
621
+ options.logger?.warn?.("manifest walk truncated; refusing to upload a partial scope", {
622
+ scopeKey,
623
+ filesWalked: built.walkStats.filesWalked,
624
+ entries: chunkPlan.entryCount,
625
+ walkMs: built.walkStats.walkMs,
626
+ });
627
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "walk_truncated");
628
+ return {
629
+ status: "walk_truncated",
630
+ scopeKey,
631
+ mode,
632
+ chunkCount,
633
+ uploadedChunks: 0,
634
+ // The stored base (if any) is untouched and still describes a COMPLETE
635
+ // walk, so the next pass may legitimately delta against it.
636
+ nextModeFull: nextModeFullIfBaseUntouched,
637
+ };
638
+ }
639
+ // A FULL manifest of nothing-but-disk, from a scope we have a tracked
640
+ // snapshot for, is not a picture of the machine — it is a picture of a
641
+ // ledger we failed to read. Uploading it is destructive in the same way a
642
+ // truncated walk is: `full` means "this is everything I hold", so the
643
+ // server's materialised view becomes "this client tracks nothing", and every
644
+ // file the user actually has reads as never-synced drift.
645
+ //
646
+ // The prior local snapshot is the evidence. It was written by a pass the
647
+ // SERVER materialised, so "this scope had N tracked entries yesterday" is a
648
+ // fact, not a guess — and a scope does not lose all N in one cycle without
649
+ // something being wrong with the reader rather than the disk.
650
+ //
651
+ // Same posture as `walk_truncated`: no POST, no base, no `lastUploadAt`; the
652
+ // sequence is burned and the failure backoff armed. `dryRun` still prints,
653
+ // because this manifest is precisely what an operator debugging it needs.
654
+ const priorTrackedEntries = previous ? Object.keys(previous.entries).length : 0;
655
+ const diskOnlyShare = chunkPlan.entryCount === 0 ? 0 : chunkPlan.diskEntryCount / chunkPlan.entryCount;
656
+ if (mode === "full" &&
657
+ chunkPlan.entryCount > 0 &&
658
+ diskOnlyShare >= MANIFEST_IMPLAUSIBLE_DISK_RATIO &&
659
+ priorTrackedEntries > 0 &&
660
+ !options.dryRun) {
661
+ options.logger?.error?.("manifest ledger looks unreadable; refusing to upload a disk-only scope", {
662
+ scopeKey,
663
+ entries: chunkPlan.entryCount,
664
+ diskEntries: chunkPlan.diskEntryCount,
665
+ priorTrackedEntries,
666
+ });
667
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "ledger_implausible");
668
+ return {
669
+ status: "ledger_implausible",
670
+ scopeKey,
671
+ mode,
672
+ chunkCount,
673
+ uploadedChunks: 0,
674
+ // The stored base is untouched and still describes a real, tracked
675
+ // scope, so the next pass may legitimately delta against it.
676
+ nextModeFull: nextModeFullIfBaseUntouched,
677
+ };
678
+ }
679
+ if (options.dryRun) {
680
+ return {
681
+ status: "printed",
682
+ scopeKey,
683
+ snapshotId: snapshot.snapshotId,
684
+ mode,
685
+ chunkCount,
686
+ uploadedChunks: 0,
687
+ // The ONE path that legitimately materialises the whole chunk set: an
688
+ // operator asked to SEE it. Nothing is sent, so nothing is streamed.
689
+ chunks: built.chunks,
690
+ };
691
+ }
692
+ // Real clock, deliberately: `nowMs` may be pinned by the caller to make a
693
+ // manifest reproducible, and a pinned clock would make this deadline either
694
+ // never fire or fire instantly.
695
+ const uploadDeadlineMs = Date.now() + Math.max(0, options.uploadWallMs ?? DEFAULT_MANIFEST_UPLOAD_WALL_MS);
696
+ let uploadedChunks = 0;
697
+ let completed = false;
698
+ let materialised = false;
699
+ // The id the SERVER derived for this pass. The client's own
700
+ // `snapshot.snapshotId` is a local random `snap-*` handle that the server has
701
+ // never seen: the server derives its id by hashing the header fields
702
+ // (`installationId`, scope, `machineId`, `mode`, `generatedAt`, `sequence`,
703
+ // `chunkCount`) and returns it on every accepted chunk. Persisting the LOCAL
704
+ // id as the delta base would quote a `baseSnapshotId` the server can never
705
+ // match, so every delta would be answered `resend_full` forever — the audit
706
+ // would still work, but it would pay a full 70k-file re-send on every single
707
+ // pass and never once exercise the delta path.
708
+ let serverSnapshotId;
709
+ // Set when two chunks of one pass come back under DIFFERENT server ids,
710
+ // which means the header fields drifted mid-pass and the snapshot has been
711
+ // scattered across rows that can each only ever stay partial.
712
+ let snapshotIdDrifted = false;
713
+ // Indexed, not `for (const chunk of chunks)`: each chunk is MINTED here,
714
+ // serialised by the transport, and dropped before the next one is asked for,
715
+ // so the pass holds one chunk at a time instead of the whole set. See
716
+ // `ManifestChunkPlan` for why `chunkCount` is safe to fix up front.
717
+ for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) {
718
+ const chunk = chunkPlan.chunk(chunkIndex);
719
+ if (options.abortSignal?.aborted) {
720
+ // Only the sequence is persisted: a half-sent pass leaves the server's
721
+ // partial fold to expire on its own, but the sequence it already burned
722
+ // must never be handed out twice.
723
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
724
+ return {
725
+ status: "failed",
726
+ scopeKey,
727
+ mode,
728
+ chunkCount,
729
+ uploadedChunks,
730
+ nextModeFull: nextModeFullIfBaseUntouched,
731
+ error: { kind: "aborted", chunkIndex: chunk.chunkIndex },
732
+ };
733
+ }
734
+ const forbidden = findCallerSelectedIdentity(chunk);
735
+ if (forbidden) {
736
+ // This can trip on a chunk AFTER earlier chunks were accepted, so the
737
+ // sequence has already been spent server-side; handing it out again
738
+ // would fold a second pass into the same derived snapshotId.
739
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
740
+ return {
741
+ status: "failed",
742
+ scopeKey,
743
+ mode,
744
+ chunkCount,
745
+ uploadedChunks,
746
+ nextModeFull: nextModeFullIfBaseUntouched,
747
+ error: { kind: "rejected", detail: forbidden, chunkIndex: chunk.chunkIndex },
748
+ };
749
+ }
750
+ let outcome;
751
+ try {
752
+ outcome = await raceTransportCall(options.transport(chunk, { attempt: 1 }), options.abortSignal, uploadDeadlineMs);
753
+ }
754
+ catch (err) {
755
+ options.logger?.warn?.("manifest upload transport failed", {
756
+ scopeKey,
757
+ chunkIndex: chunk.chunkIndex,
758
+ error: errorClassOf(err),
759
+ });
760
+ // No `lastUploadAt` write — a network blip must be retried on the next
761
+ // cycle, not throttled away for a day — but the sequence still advances.
762
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
763
+ return {
764
+ status: "failed",
765
+ scopeKey,
766
+ mode,
767
+ chunkCount,
768
+ uploadedChunks,
769
+ nextModeFull: nextModeFullIfBaseUntouched,
770
+ error: { kind: "transport_error", detail: errorClassOf(err), chunkIndex: chunk.chunkIndex },
771
+ };
772
+ }
773
+ if (outcome.kind !== "settled") {
774
+ // Abort and deadline are the same shape of event — "stop sending, the
775
+ // caller is no longer waiting on us" — and both take the same
776
+ // non-throwing exit: burn the sequence (it may already be spent
777
+ // server-side by an earlier chunk of this pass), leave `lastUploadAt`
778
+ // and the delta base alone so the next cycle retries, and return.
779
+ const aborted = outcome.kind === "aborted";
780
+ options.logger?.warn?.(aborted
781
+ ? "manifest upload aborted mid-flight"
782
+ : "manifest upload exceeded its wall clock; stopping", { scopeKey, chunkIndex: chunk.chunkIndex, uploadedChunks });
783
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
784
+ return {
785
+ status: "failed",
786
+ scopeKey,
787
+ mode,
788
+ chunkCount,
789
+ uploadedChunks,
790
+ nextModeFull: nextModeFullIfBaseUntouched,
791
+ error: { kind: aborted ? "aborted" : "upload_timeout", chunkIndex: chunk.chunkIndex },
792
+ };
793
+ }
794
+ const { status, body } = outcome.response;
795
+ if (readsResendFull(status, body)) {
796
+ // The server lost (or refused) this snapshot. Stop IMMEDIATELY — the
797
+ // remaining chunks would fold into a snapshot that no longer exists —
798
+ // and drop the local base so the next pass is full. `snapshot_abandoned`
799
+ // additionally means "do not retry this snapshotId at all", which is
800
+ // exactly what stopping the pass achieves.
801
+ const reason = readString(body, "reason");
802
+ options.logger?.info?.("server requested a full manifest resend", {
803
+ scopeKey,
804
+ chunkIndex: chunk.chunkIndex,
805
+ reason,
806
+ });
807
+ persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "resend_full");
808
+ return {
809
+ status: "resend_full_scheduled",
810
+ scopeKey,
811
+ mode,
812
+ chunkCount,
813
+ uploadedChunks,
814
+ nextModeFull: true,
815
+ error: reason ? { kind: "rejected", status, detail: reason, chunkIndex: chunk.chunkIndex } : undefined,
816
+ };
817
+ }
818
+ if (status === 404) {
819
+ // hq-cloud ships to laptops and updates independently of the server, so
820
+ // "route not deployed here yet" describes a perfectly healthy machine.
821
+ // Info, not error — and no delta base, because nothing was received.
822
+ options.logger?.info?.("manifest upload route not available (soft skip)", { scopeKey });
823
+ persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: false }), "soft_skip");
824
+ return {
825
+ status: "soft_skipped",
826
+ scopeKey,
827
+ mode,
828
+ chunkCount,
829
+ uploadedChunks,
830
+ // `invalidateBase: false` above KEEPS a usable base, so this reports
831
+ // what the next pass will actually do rather than a blanket `true`.
832
+ nextModeFull: nextModeFullIfBaseUntouched,
833
+ };
834
+ }
835
+ if (status === 413) {
836
+ const reason = readString(body, "reason");
837
+ /**
838
+ * TWO DIFFERENT FAULTS SHARE THIS STATUS CODE, AND THEY MUST NOT SHARE
839
+ * A RECOVERY.
840
+ *
841
+ * `manifest_too_large` is a SERVER ANSWER ABOUT THE SCOPE: the fold
842
+ * crossed `SYNC_MANIFEST_MAX_TOTAL_ENTRIES` for this (installation,
843
+ * scope). Re-sending it — at any chunk size — produces the same answer,
844
+ * because the ceiling counts entries, not bytes. Spacing those retries a
845
+ * day apart is right, and that is what `recordUploadAttempt` does.
846
+ *
847
+ * `chunk_too_large` (and a bare 413 from the gateway in front of the
848
+ * handler, which carries no reason at all) is a CLIENT CONFIGURATION
849
+ * FAULT: this body was too big, the SAME manifest cut smaller fits, and
850
+ * nothing about the scope has to change for it to succeed. 6.16.21
851
+ * treated it as the former and armed the 24h throttle on it, so the
852
+ * operator's 199k scope reported a healthy `lastUploadAt`, suppressed
853
+ * itself for a day, and did it again — for a fault that a retry twenty
854
+ * minutes later with half the budget would have cleared.
855
+ *
856
+ * So it takes the FAILURE path (backoff, no `lastUploadAt`, no false
857
+ * healthy signal) and halves the budget for the next pass.
858
+ *
859
+ * NOT RETRIED WITHIN THIS PASS — not as a policy choice but because it
860
+ * cannot be. `chunkCount` is hashed into the server's snapshotId, so
861
+ * re-planning mid-flight would derive a second snapshot for one upload
862
+ * and orphan the chunks already accepted. The retry is the next pass,
863
+ * which the failure backoff schedules an hour out.
864
+ */
865
+ const scopeTooLarge = reason === SYNC_MANIFEST_TOO_LARGE_REASON_CODE;
866
+ if (scopeTooLarge) {
867
+ options.logger?.warn?.("manifest upload rejected: scope exceeds the server's total entry ceiling", {
868
+ scopeKey,
869
+ reason,
870
+ });
871
+ // Non-retryable within this pass, and the server's view of the scope is
872
+ // now incomplete, so the next pass must be full.
873
+ persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "too_large");
874
+ return {
875
+ status: "failed",
876
+ scopeKey,
877
+ mode,
878
+ chunkCount,
879
+ uploadedChunks,
880
+ chunkByteBudget,
881
+ nextModeFull: true,
882
+ error: { kind: "manifest_too_large", status, detail: reason, chunkIndex: chunk.chunkIndex },
883
+ };
884
+ }
885
+ const healedBudget = nextManifestChunkByteBudget(chunkPlan.chunkByteBudget);
886
+ options.logger?.warn?.("manifest chunk rejected as too large; halving the byte budget", {
887
+ scopeKey,
888
+ reason,
889
+ chunkIndex: chunk.chunkIndex,
890
+ chunkByteBudget: chunkPlan.chunkByteBudget,
891
+ maxChunkBytes: chunkPlan.maxChunkBytes,
892
+ nextChunkByteBudget: healedBudget,
893
+ });
894
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, {
895
+ chunkByteBudget: healedBudget,
896
+ }), "chunk_too_large");
897
+ return {
898
+ status: "failed",
899
+ scopeKey,
900
+ mode,
901
+ chunkCount,
902
+ uploadedChunks,
903
+ chunkByteBudget: healedBudget,
904
+ // The stored base was left ALONE by `recordFailedAttempt`, so this
905
+ // reports what the next pass will really do rather than forcing a full
906
+ // re-send of the scope on top of a size fault.
907
+ nextModeFull: nextModeFullIfBaseUntouched,
908
+ error: { kind: "chunk_too_large", status, detail: reason, chunkIndex: chunk.chunkIndex },
909
+ };
910
+ }
911
+ if (status < 200 || status >= 300) {
912
+ options.logger?.warn?.("manifest upload rejected", { scopeKey, status, chunkIndex: chunk.chunkIndex });
913
+ persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
914
+ return {
915
+ status: "failed",
916
+ scopeKey,
917
+ mode,
918
+ chunkCount,
919
+ uploadedChunks,
920
+ nextModeFull: nextModeFullIfBaseUntouched,
921
+ error: {
922
+ kind: "rejected",
923
+ status,
924
+ detail: readString(body, "code") ?? readString(body, "reason"),
925
+ chunkIndex: chunk.chunkIndex,
926
+ },
927
+ };
928
+ }
929
+ uploadedChunks += 1;
930
+ const chunkSnapshotId = readString(body, "snapshotId");
931
+ if (chunkSnapshotId !== undefined) {
932
+ if (serverSnapshotId !== undefined && serverSnapshotId !== chunkSnapshotId)
933
+ snapshotIdDrifted = true;
934
+ serverSnapshotId = chunkSnapshotId;
935
+ }
936
+ completed = readBoolean(body, "completed");
937
+ materialised = readBoolean(body, "materialised");
938
+ }
939
+ if (snapshotIdDrifted) {
940
+ // Belt-and-braces against a builder regression: the chunker stamps the
941
+ // header once, so this can only fire if that stops being true. Refuse the
942
+ // base rather than persist an id that names only part of the snapshot.
943
+ options.logger?.error?.("manifest upload snapshot id drifted mid-pass", { scopeKey });
944
+ materialised = false;
945
+ }
946
+ if (completed && materialised && serverSnapshotId !== undefined) {
947
+ // THE gate. Only now is this snapshot a legitimate delta base: the server
948
+ // has folded every chunk and written the materialised view it will compare
949
+ // the next delta against.
950
+ const stored = persisted(writeManifestSnapshot(options.stateDir, scopeKey, {
951
+ ...snapshot,
952
+ // The SERVER's id, not ours — this value's only job is to be quoted
953
+ // back as the next delta's `baseSnapshotId`.
954
+ snapshotId: serverSnapshotId,
955
+ lastUploadAt: nowIso,
956
+ baseValid: true,
957
+ }), "delta_base");
958
+ return {
959
+ status: "uploaded",
960
+ scopeKey,
961
+ snapshotId: serverSnapshotId,
962
+ mode,
963
+ chunkCount,
964
+ uploadedChunks,
965
+ chunkByteBudget: chunkPlan.chunkByteBudget,
966
+ // If the write failed there is no base on disk, so the next pass cannot
967
+ // be a delta — reporting `false` here would tell the caller a base
968
+ // exists that does not.
969
+ nextModeFull: !stored,
970
+ };
971
+ }
972
+ // Every chunk was accepted but the fold was refused (`materialised: false`)
973
+ // or never completed. The client and the server now disagree about what the
974
+ // server holds, and only a FULL manifest can resolve that.
975
+ // Also covers an accepted pass whose responses carried no `snapshotId`: with
976
+ // no server id there is no base to quote, so the next pass must be full.
977
+ options.logger?.warn?.("manifest upload was not materialised; next pass will be full", {
978
+ scopeKey,
979
+ completed,
980
+ materialised,
981
+ hasServerSnapshotId: serverSnapshotId !== undefined,
982
+ });
983
+ persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "not_materialised");
984
+ return {
985
+ status: "uploaded",
986
+ scopeKey,
987
+ snapshotId: serverSnapshotId ?? snapshot.snapshotId,
988
+ mode,
989
+ chunkCount,
990
+ uploadedChunks,
991
+ chunkByteBudget: chunkPlan.chunkByteBudget,
992
+ nextModeFull: true,
993
+ };
994
+ }
995
+ /**
996
+ * Read per-scope upload state without touching the network or the disk tree.
997
+ *
998
+ * This is the seam `hq doctor`'s `manifest` check consumes: it answers "when
999
+ * did this machine last tell the server what it holds?" for each scope, which
1000
+ * is the one question that distinguishes "no findings" from "no data".
1001
+ * Missing/older snapshots report `lastUploadAt: null` — never uploaded.
1002
+ */
1003
+ export function readManifestUploadStatus(stateDir, scopes) {
1004
+ return scopes.map((scope) => {
1005
+ const scopeKey = manifestSnapshotScopeKey(scope);
1006
+ const snapshot = readManifestSnapshot(stateDir, scopeKey);
1007
+ if (!snapshot) {
1008
+ return { scopeKey, lastUploadAt: null, snapshotId: null, sequence: 0, baseUsable: false };
1009
+ }
1010
+ const baseUsable = isManifestSnapshotBaseUsable(snapshot);
1011
+ return {
1012
+ scopeKey,
1013
+ lastUploadAt: snapshot.lastUploadAt ?? null,
1014
+ snapshotId: baseUsable ? snapshot.snapshotId : null,
1015
+ sequence: snapshot.sequence,
1016
+ baseUsable,
1017
+ };
1018
+ });
1019
+ }
1020
+ /** Re-exported so callers do not need a second import for the staleness rule. */
1021
+ export { MANIFEST_SNAPSHOT_MAX_AGE_MS };
1022
+ //# sourceMappingURL=upload-manifest.js.map