@indigoai-us/hq-cloud 6.16.19 → 6.16.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-installation-id.d.ts +66 -0
- package/dist/bin/sync-runner-installation-id.d.ts.map +1 -0
- package/dist/bin/sync-runner-installation-id.js +138 -0
- package/dist/bin/sync-runner-installation-id.js.map +1 -0
- package/dist/bin/sync-runner-manifest.d.ts +120 -0
- package/dist/bin/sync-runner-manifest.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.js +348 -0
- package/dist/bin/sync-runner-manifest.js.map +1 -0
- package/dist/bin/sync-runner-manifest.test.d.ts +12 -0
- package/dist/bin/sync-runner-manifest.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.test.js +510 -0
- package/dist/bin/sync-runner-manifest.test.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +56 -21
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +86 -61
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +217 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/cli/share-presign-transport.test.d.ts +22 -0
- package/dist/cli/share-presign-transport.test.d.ts.map +1 -0
- package/dist/cli/share-presign-transport.test.js +237 -0
- package/dist/cli/share-presign-transport.test.js.map +1 -0
- package/dist/cli/share.d.ts.map +1 -1
- package/dist/cli/share.js +17 -3
- package/dist/cli/share.js.map +1 -1
- package/dist/cli/sync.d.ts.map +1 -1
- package/dist/cli/sync.js +9 -0
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/sync.test.js +123 -0
- package/dist/cli/sync.test.js.map +1 -1
- package/dist/company-slug.d.ts +31 -0
- package/dist/company-slug.d.ts.map +1 -0
- package/dist/company-slug.js +52 -0
- package/dist/company-slug.js.map +1 -0
- package/dist/company-slug.test.d.ts +7 -0
- package/dist/company-slug.test.d.ts.map +1 -0
- package/dist/company-slug.test.js +43 -0
- package/dist/company-slug.test.js.map +1 -0
- package/dist/company-vault-transport.d.ts +69 -0
- package/dist/company-vault-transport.d.ts.map +1 -0
- package/dist/company-vault-transport.js +89 -0
- package/dist/company-vault-transport.js.map +1 -0
- package/dist/company-vault-transport.test.d.ts +12 -0
- package/dist/company-vault-transport.test.d.ts.map +1 -0
- package/dist/company-vault-transport.test.js +97 -0
- package/dist/company-vault-transport.test.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/journal-ledger-index.test.d.ts +15 -0
- package/dist/journal-ledger-index.test.d.ts.map +1 -0
- package/dist/journal-ledger-index.test.js +247 -0
- package/dist/journal-ledger-index.test.js.map +1 -0
- package/dist/journal.d.ts +110 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +253 -5
- package/dist/journal.js.map +1 -1
- package/dist/manifest/US-002.acceptance.test.d.ts +11 -0
- package/dist/manifest/US-002.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-002.acceptance.test.js +160 -0
- package/dist/manifest/US-002.acceptance.test.js.map +1 -0
- package/dist/manifest/US-004.acceptance.test.d.ts +12 -0
- package/dist/manifest/US-004.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-004.acceptance.test.js +173 -0
- package/dist/manifest/US-004.acceptance.test.js.map +1 -0
- package/dist/manifest/build-manifest.bench.test.d.ts +22 -0
- package/dist/manifest/build-manifest.bench.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.bench.test.js +544 -0
- package/dist/manifest/build-manifest.bench.test.js.map +1 -0
- package/dist/manifest/build-manifest.d.ts +319 -0
- package/dist/manifest/build-manifest.d.ts.map +1 -0
- package/dist/manifest/build-manifest.js +1043 -0
- package/dist/manifest/build-manifest.js.map +1 -0
- package/dist/manifest/build-manifest.test.d.ts +10 -0
- package/dist/manifest/build-manifest.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.test.js +1060 -0
- package/dist/manifest/build-manifest.test.js.map +1 -0
- package/dist/manifest/contract.d.ts +290 -0
- package/dist/manifest/contract.d.ts.map +1 -0
- package/dist/manifest/contract.js +476 -0
- package/dist/manifest/contract.js.map +1 -0
- package/dist/manifest/contract.test.d.ts +17 -0
- package/dist/manifest/contract.test.d.ts.map +1 -0
- package/dist/manifest/contract.test.js +283 -0
- package/dist/manifest/contract.test.js.map +1 -0
- package/dist/manifest/import-graph.test.d.ts +25 -0
- package/dist/manifest/import-graph.test.d.ts.map +1 -0
- package/dist/manifest/import-graph.test.js +120 -0
- package/dist/manifest/import-graph.test.js.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts +18 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.js +153 -0
- package/dist/manifest/ledger-shard-isolation.test.js.map +1 -0
- package/dist/manifest/snapshot-store.d.ts +143 -0
- package/dist/manifest/snapshot-store.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.js +291 -0
- package/dist/manifest/snapshot-store.js.map +1 -0
- package/dist/manifest/snapshot-store.test.d.ts +11 -0
- package/dist/manifest/snapshot-store.test.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.test.js +133 -0
- package/dist/manifest/snapshot-store.test.js.map +1 -0
- package/dist/manifest/upload-manifest.d.ts +301 -0
- package/dist/manifest/upload-manifest.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.js +921 -0
- package/dist/manifest/upload-manifest.js.map +1 -0
- package/dist/manifest/upload-manifest.test.d.ts +10 -0
- package/dist/manifest/upload-manifest.test.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.test.js +1052 -0
- package/dist/manifest/upload-manifest.test.js.map +1 -0
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +1 -26
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/posix-key.d.ts +36 -0
- package/dist/posix-key.d.ts.map +1 -0
- package/dist/posix-key.js +38 -0
- package/dist/posix-key.js.map +1 -0
- package/dist/s3.d.ts +2 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +6 -20
- package/dist/s3.js.map +1 -1
- package/dist/sync/journal-ledger-stream.d.ts +56 -0
- package/dist/sync/journal-ledger-stream.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.js +191 -0
- package/dist/sync/journal-ledger-stream.js.map +1 -0
- package/dist/sync/journal-ledger-stream.test.d.ts +12 -0
- package/dist/sync/journal-ledger-stream.test.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.test.js +134 -0
- package/dist/sync/journal-ledger-stream.test.js.map +1 -0
- package/dist/types.d.ts +18 -7
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,921 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side manifest UPLOAD pass (sync-reconciliation-audit US-004).
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS SEAM LIVES IN hq-cloud
|
|
5
|
+
* -------------------------------
|
|
6
|
+
* Two clients upload manifests — the desktop sync daemon (hq-sync) after a
|
|
7
|
+
* sync cycle, and `hq sync manifest` in the CLI — and the PRD's decision
|
|
8
|
+
* record settles it as "both, via shared hq-cloud code". The rules that make
|
|
9
|
+
* an upload SAFE are not obvious and must not be re-derived twice:
|
|
10
|
+
*
|
|
11
|
+
* - a snapshot may become the local delta BASE only after the server says
|
|
12
|
+
* `materialised: true` (a `completed` fold that was refused is not a base);
|
|
13
|
+
* - the delta base is the snapshotId the SERVER returns, never the local
|
|
14
|
+
* `snap-*` handle the builder minted (the server derives its own id);
|
|
15
|
+
* - `sequence` must strictly increase per (installation, scope), including
|
|
16
|
+
* across passes that failed before they uploaded anything;
|
|
17
|
+
* - every chunk of one pass must be byte-identical in the fields the server
|
|
18
|
+
* hashes into the snapshotId (`generatedAt`, `sequence`, `chunkCount`,
|
|
19
|
+
* `machineId`, `mode`, `scope`);
|
|
20
|
+
* - a `resend_full` at ANY chunk ends the pass and invalidates the base.
|
|
21
|
+
*
|
|
22
|
+
* Two implementations of those rules would drift, and the failure mode of the
|
|
23
|
+
* drift is silent server-side corruption of a scope's materialised view — the
|
|
24
|
+
* exact thing this audit exists to detect.
|
|
25
|
+
*
|
|
26
|
+
* WHAT THIS DELIBERATELY DOES NOT OWN
|
|
27
|
+
* -----------------------------------
|
|
28
|
+
* - **Auth and the base URL.** The transport is INJECTED. hq-cloud has no
|
|
29
|
+
* business deciding which API host a daemon talks to or how a JWT is
|
|
30
|
+
* minted; the clients already own both, and owning them here would force
|
|
31
|
+
* this module to import the vault client (and with it the AWS SDK) into
|
|
32
|
+
* what is otherwise a lean, hot-path module graph.
|
|
33
|
+
* - **Scheduling.** The caller decides when a pass runs; this function only
|
|
34
|
+
* refuses to run one too often (see the 24h throttle).
|
|
35
|
+
* - **Wiring.** The hq-cli `sync manifest` command, its `hq doctor` check,
|
|
36
|
+
* and the hq-sync daemon hook land SEPARATELY, in those repos, on top of
|
|
37
|
+
* {@link runManifestUploadPass} and {@link readManifestUploadStatus}.
|
|
38
|
+
*
|
|
39
|
+
* FAILURE POSTURE
|
|
40
|
+
* ---------------
|
|
41
|
+
* This never throws at the caller for anything that happens at runtime. The
|
|
42
|
+
* daemon calls it after a sync pass and must continue regardless: an audit
|
|
43
|
+
* that can break sync is strictly worse than no audit. Build errors, transport
|
|
44
|
+
* rejections, aborts and unexpected statuses all come back as
|
|
45
|
+
* `{ status: "failed" }`. Only invalid OPTIONS — a programmer error, not a
|
|
46
|
+
* runtime condition — throw.
|
|
47
|
+
*/
|
|
48
|
+
import { buildSyncManifest, } from "./build-manifest.js";
|
|
49
|
+
// A per-scope mutex, not a new dependency: `operation-lock.ts` imports nothing
|
|
50
|
+
// but node builtins, so serialising the pass costs the module graph nothing.
|
|
51
|
+
// (It is not reachable from `build-manifest.ts` either, so the import-graph
|
|
52
|
+
// test's closure is unaffected.)
|
|
53
|
+
import { OperationLockedError, acquireOperationLock, } from "../operation-lock.js";
|
|
54
|
+
import { MANIFEST_SNAPSHOT_MAX_AGE_MS, isManifestSnapshotBaseUsable, isManifestSnapshotStale, manifestSnapshotScopeKey, manifestSnapshotScopeSlug, readManifestSnapshot, writeManifestSnapshot, } from "./snapshot-store.js";
|
|
55
|
+
// ─── Kill switch / throttle ──────────────────────────────────────────────────
|
|
56
|
+
/**
|
|
57
|
+
* Env var that disables manifest uploads entirely. A user (or a support
|
|
58
|
+
* engineer) must be able to switch this feature off on a machine WITHOUT
|
|
59
|
+
* downgrading hq-cloud, and the switch must short-circuit before the walk —
|
|
60
|
+
* the walk, not the POST, is the expensive half.
|
|
61
|
+
*/
|
|
62
|
+
export const MANIFEST_UPLOAD_DISABLED_ENV = "HQ_SYNC_MANIFEST_DISABLED";
|
|
63
|
+
/**
|
|
64
|
+
* Default minimum spacing between upload passes for one scope.
|
|
65
|
+
*
|
|
66
|
+
* The audit is a slow-moving safety net, not telemetry: a daily picture is
|
|
67
|
+
* enough to catch a wedged watcher, and a 70k-file walk per sync cycle is not
|
|
68
|
+
* something a laptop should pay for.
|
|
69
|
+
*/
|
|
70
|
+
export const DEFAULT_MANIFEST_UPLOAD_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
71
|
+
/**
|
|
72
|
+
* Wall-clock budget for the ENTIRE upload phase (all chunks), after which the
|
|
73
|
+
* pass stops sending and reports a non-throwing failure.
|
|
74
|
+
*
|
|
75
|
+
* Why the whole phase and not per chunk: the caller's actual requirement is
|
|
76
|
+
* "the daemon's `await` returns" — the desktop daemon runs this after every
|
|
77
|
+
* sync pass, and a transport that never settles would otherwise wedge the pass
|
|
78
|
+
* (and the sync cycle behind it) forever. A per-chunk budget would not bound
|
|
79
|
+
* that: a 30-chunk manifest would still admit 30 x budget of hanging. One
|
|
80
|
+
* budget for the phase bounds the pass by construction, whatever the chunk
|
|
81
|
+
* count.
|
|
82
|
+
*
|
|
83
|
+
* 60s is generous next to a single small JSON POST and mean next to a sync
|
|
84
|
+
* cycle, so it only ever fires on a genuinely wedged transport — never on a
|
|
85
|
+
* merely slow link, where the pass would be retried next cycle anyway.
|
|
86
|
+
*
|
|
87
|
+
* Note this bounds only the UPLOAD; `maxWallMs` bounds the filesystem walk.
|
|
88
|
+
* They are separate budgets because they fail for unrelated reasons (a slow
|
|
89
|
+
* disk vs. a black-holed socket) and a caller may reasonably tune one alone.
|
|
90
|
+
*/
|
|
91
|
+
export const DEFAULT_MANIFEST_UPLOAD_WALL_MS = 60_000;
|
|
92
|
+
/**
|
|
93
|
+
* First retry floor after a pass FAILED without reaching the server.
|
|
94
|
+
*
|
|
95
|
+
* The 24h throttle is armed by `lastUploadAt`, which only a pass that actually
|
|
96
|
+
* reached the server writes. That is correct — a network blip must not
|
|
97
|
+
* suppress the audit for a day — but taken alone it means a permanently broken
|
|
98
|
+
* endpoint (a wrong base URL, an unenrolled installation answering 403, a 5xx
|
|
99
|
+
* outage) buys a FULL tree walk, and under `hashPolicy: "auto"` a full content
|
|
100
|
+
* hash, on EVERY sync cycle. On a large vault that is minutes of IO and
|
|
101
|
+
* gigabytes of RSS per cycle, forever, for an audit that cannot succeed.
|
|
102
|
+
*
|
|
103
|
+
* So a failed pass arms its own, much shorter floor, doubling per consecutive
|
|
104
|
+
* failure up to {@link MANIFEST_FAILURE_BACKOFF_MAX_MS}. One hour keeps a
|
|
105
|
+
* transient outage cheap to recover from (the next hourly cycle retries) while
|
|
106
|
+
* capping a hard failure's cost at ~1 walk/hour instead of 1 walk/cycle.
|
|
107
|
+
*/
|
|
108
|
+
export const MANIFEST_FAILURE_BACKOFF_BASE_MS = 60 * 60 * 1000;
|
|
109
|
+
/** Ceiling for the failure backoff — the same daily spacing a healthy scope gets. */
|
|
110
|
+
export const MANIFEST_FAILURE_BACKOFF_MAX_MS = 24 * 60 * 60 * 1000;
|
|
111
|
+
/**
|
|
112
|
+
* Share of `source: "disk"` entries above which a FULL manifest is treated as
|
|
113
|
+
* an implausible picture of a scope that was previously tracked.
|
|
114
|
+
*
|
|
115
|
+
* Set hard against 1.0 rather than at a "suspicious" level like 0.9 on
|
|
116
|
+
* purpose. A legitimate manifest CAN be very disk-heavy — a user who just
|
|
117
|
+
* restored a backup, or a first pass after a journal repair — and refusing
|
|
118
|
+
* those would suppress the audit exactly when it is most useful. The failure
|
|
119
|
+
* this guards is not "mostly disk": it is "the ledger vanished", which is
|
|
120
|
+
* total. 0.995 leaves room for the handful of `ledger`/`both` entries a
|
|
121
|
+
* partially-readable ledger still contributes without admitting a merely
|
|
122
|
+
* lopsided scope.
|
|
123
|
+
*/
|
|
124
|
+
export const MANIFEST_IMPLAUSIBLE_DISK_RATIO = 0.995;
|
|
125
|
+
/**
|
|
126
|
+
* Retry floor for a scope that has failed `consecutiveFailures` times in a row.
|
|
127
|
+
*
|
|
128
|
+
* `1 → 1h, 2 → 2h, 3 → 4h … capped at 24h`. Zero (or anything nonsensical)
|
|
129
|
+
* means "no floor": a scope with no failure history is governed by the normal
|
|
130
|
+
* `lastUploadAt` throttle alone.
|
|
131
|
+
*/
|
|
132
|
+
export function manifestFailureBackoffMs(consecutiveFailures) {
|
|
133
|
+
if (!Number.isFinite(consecutiveFailures) || consecutiveFailures < 1)
|
|
134
|
+
return 0;
|
|
135
|
+
// Clamped BEFORE the shift: `2 ** 1024` is Infinity, and an unbounded
|
|
136
|
+
// exponent on a corrupt count would produce a floor that never expires.
|
|
137
|
+
const steps = Math.min(Math.floor(consecutiveFailures), 20) - 1;
|
|
138
|
+
return Math.min(MANIFEST_FAILURE_BACKOFF_BASE_MS * 2 ** steps, MANIFEST_FAILURE_BACKOFF_MAX_MS);
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Only an explicit, unambiguous value disables. Empty string, `0`, `false` and
|
|
142
|
+
* anything else mean "not set": env vars are routinely exported empty by
|
|
143
|
+
* shell wrappers, and treating that as "off" would silently disable the
|
|
144
|
+
* feature on machines nobody meant to opt out.
|
|
145
|
+
*/
|
|
146
|
+
/**
|
|
147
|
+
* The kill switch, readable by a CALLER.
|
|
148
|
+
*
|
|
149
|
+
* `runManifestUploadPass` enforces it itself, but a caller that assembles the
|
|
150
|
+
* pass (resolving identity, minting a transport, choosing scopes) needs to
|
|
151
|
+
* check it BEFORE doing any of that, so an opted-out machine pays literally
|
|
152
|
+
* nothing. Exported rather than duplicated so the two can never disagree about
|
|
153
|
+
* what counts as "disabled".
|
|
154
|
+
*/
|
|
155
|
+
export function isManifestUploadDisabled(env) {
|
|
156
|
+
return isUploadDisabled(env);
|
|
157
|
+
}
|
|
158
|
+
function isUploadDisabled(env) {
|
|
159
|
+
const raw = env[MANIFEST_UPLOAD_DISABLED_ENV];
|
|
160
|
+
if (typeof raw !== "string")
|
|
161
|
+
return false;
|
|
162
|
+
const normalised = raw.trim().toLowerCase();
|
|
163
|
+
return normalised === "1" || normalised === "true";
|
|
164
|
+
}
|
|
165
|
+
// ─── Response reading ────────────────────────────────────────────────────────
|
|
166
|
+
function asRecord(body) {
|
|
167
|
+
if (body === null || typeof body !== "object" || Array.isArray(body))
|
|
168
|
+
return {};
|
|
169
|
+
return body;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* The server answers `resend_full` as a 200 with `status: "resend_full"` — a
|
|
173
|
+
* SUCCESSFUL HTTP call carrying a protocol-level "start over". It is checked
|
|
174
|
+
* before the 2xx path for exactly that reason; reading only the status code
|
|
175
|
+
* would record a base the server has already thrown away.
|
|
176
|
+
*/
|
|
177
|
+
function readsResendFull(status, body) {
|
|
178
|
+
// Only a 2xx carries the protocol answer. A non-2xx is a transport/server
|
|
179
|
+
// FAILURE and must take the failure path: the server's own 409 means
|
|
180
|
+
// VAULT_NOT_PROVISIONED, and reading that as "start over" would report a
|
|
181
|
+
// hard misconfiguration as a scheduled resend AND arm the 24h throttle on
|
|
182
|
+
// it, hiding the fault for a day per attempt.
|
|
183
|
+
if (status < 200 || status >= 300)
|
|
184
|
+
return false;
|
|
185
|
+
const raw = asRecord(body);
|
|
186
|
+
if (raw.status === "resend_full")
|
|
187
|
+
return true;
|
|
188
|
+
return raw.action === "resend_full" || raw.reason === "resend_full" || raw.resendFull === true;
|
|
189
|
+
}
|
|
190
|
+
function readString(body, key) {
|
|
191
|
+
const value = asRecord(body)[key];
|
|
192
|
+
return typeof value === "string" ? value : undefined;
|
|
193
|
+
}
|
|
194
|
+
function readBoolean(body, key) {
|
|
195
|
+
return asRecord(body)[key] === true;
|
|
196
|
+
}
|
|
197
|
+
/** Bare error class name — never the message, which routinely carries a path. */
|
|
198
|
+
function errorClassOf(err) {
|
|
199
|
+
const name = err instanceof Error ? err.constructor?.name ?? err.name : typeof err;
|
|
200
|
+
const cleaned = String(name).replace(/[^A-Za-z0-9_]/g, "");
|
|
201
|
+
if (!cleaned || !/^[A-Za-z]/.test(cleaned))
|
|
202
|
+
return "UnknownError";
|
|
203
|
+
return cleaned.slice(0, 64);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Identity fields the server rejects outright (400
|
|
207
|
+
* CALLER_SELECTED_IDENTITY_REJECTED): the caller does not get to say who it
|
|
208
|
+
* is, the JWT does. The builder never emits these, so this is a cheap
|
|
209
|
+
* belt-and-braces check against a future field being added in the wrong place
|
|
210
|
+
* — it fails the pass locally instead of tripping a server 400 storm.
|
|
211
|
+
*/
|
|
212
|
+
const FORBIDDEN_IDENTITY_KEYS = ["personUid", "person_uid", "ownerUid", "callerSub"];
|
|
213
|
+
function findCallerSelectedIdentity(chunk) {
|
|
214
|
+
const raw = chunk;
|
|
215
|
+
return FORBIDDEN_IDENTITY_KEYS.find((key) => raw[key] !== undefined);
|
|
216
|
+
}
|
|
217
|
+
// ─── Snapshot bookkeeping ────────────────────────────────────────────────────
|
|
218
|
+
/**
|
|
219
|
+
* Record that a pass HAPPENED without promoting its snapshot to the delta
|
|
220
|
+
* base.
|
|
221
|
+
*
|
|
222
|
+
* Two invariants ride on this one write:
|
|
223
|
+
*
|
|
224
|
+
* 1. `sequence` must strictly increase per (installation, scope) even when
|
|
225
|
+
* the pass did not produce a base — otherwise a machine that keeps
|
|
226
|
+
* failing to materialise would replay the same sequence forever, and the
|
|
227
|
+
* server derives its snapshotId from it.
|
|
228
|
+
* 2. the 24h throttle must still fire, so a server that answers 404 or 413
|
|
229
|
+
* on every pass cannot turn into a per-cycle full tree walk.
|
|
230
|
+
*
|
|
231
|
+
* The PREVIOUS entries are preserved when they were usable and the server did
|
|
232
|
+
* not ask us to drop them: a refused fold is not evidence that the base the
|
|
233
|
+
* server already holds is wrong.
|
|
234
|
+
*/
|
|
235
|
+
function recordUploadAttempt(stateDir, scopeKey, previous, sequence, nowIso, options) {
|
|
236
|
+
const keepBase = !options.invalidateBase && previous !== null && isManifestSnapshotBaseUsable(previous);
|
|
237
|
+
const base = keepBase
|
|
238
|
+
? { ...previous, sequence, lastUploadAt: nowIso }
|
|
239
|
+
: {
|
|
240
|
+
// A bookkeeping-only record: `baseValid: false` makes it unusable as a
|
|
241
|
+
// delta base while keeping the sequence and the upload time, which a
|
|
242
|
+
// plain delete would throw away.
|
|
243
|
+
snapshotId: previous?.snapshotId ?? "snap-none",
|
|
244
|
+
generatedAt: previous?.generatedAt ?? nowIso,
|
|
245
|
+
sequence,
|
|
246
|
+
entries: {},
|
|
247
|
+
lastUploadAt: nowIso,
|
|
248
|
+
baseValid: false,
|
|
249
|
+
};
|
|
250
|
+
// This function is only ever called on a path where the server ANSWERED
|
|
251
|
+
// (resend_full, 404, 413, an accepted-but-unmaterialised fold). That is
|
|
252
|
+
// proof the endpoint is reachable, so the failure backoff must be cleared —
|
|
253
|
+
// otherwise a scope that failed twice last week would keep a stale retry
|
|
254
|
+
// floor armed against passes that are now succeeding.
|
|
255
|
+
const { lastAttemptAt: _lastAttemptAt, consecutiveFailures: _failures, ...record } = base;
|
|
256
|
+
return writeManifestSnapshot(stateDir, scopeKey, record);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Record a pass that FAILED before the server could answer (or whose answer
|
|
260
|
+
* was itself a failure): burn the sequence, and arm the retry floor.
|
|
261
|
+
*
|
|
262
|
+
* Deliberately NOT `lastUploadAt` — that field means "the server has a picture
|
|
263
|
+
* from this time", and writing it here would claim an upload that never
|
|
264
|
+
* happened, hiding a broken endpoint behind a healthy-looking `hq doctor`.
|
|
265
|
+
*/
|
|
266
|
+
function recordFailedAttempt(stateDir, scopeKey, previous, sequence, nowIso) {
|
|
267
|
+
const record = previous
|
|
268
|
+
? { ...previous, sequence }
|
|
269
|
+
: { snapshotId: "snap-none", generatedAt: nowIso, sequence, entries: {}, baseValid: false };
|
|
270
|
+
record.lastAttemptAt = nowIso;
|
|
271
|
+
record.consecutiveFailures = (previous?.consecutiveFailures ?? 0) + 1;
|
|
272
|
+
return writeManifestSnapshot(stateDir, scopeKey, record);
|
|
273
|
+
}
|
|
274
|
+
// ─── The pass ────────────────────────────────────────────────────────────────
|
|
275
|
+
// ─── Per-scope serialisation ─────────────────────────────────────────────────
|
|
276
|
+
/**
|
|
277
|
+
* The `command` recorded in the lock file. It shows up verbatim in
|
|
278
|
+
* `OperationLockedError` messages and in `hq doctor`-style lock dumps, so it
|
|
279
|
+
* names the operation a human would recognise.
|
|
280
|
+
*/
|
|
281
|
+
export const MANIFEST_UPLOAD_LOCK_COMMAND = "sync-manifest";
|
|
282
|
+
/**
|
|
283
|
+
* Lock scope for one manifest scope.
|
|
284
|
+
*
|
|
285
|
+
* `sequence = previous + 1` is a read-modify-write across the whole pass: the
|
|
286
|
+
* read happens before the tree walk and the write happens after the last
|
|
287
|
+
* chunk. Nothing else spaces two passes out — `hq sync manifest` sets
|
|
288
|
+
* `ignoreThrottle`, so a human invocation lands squarely inside a daemon
|
|
289
|
+
* pass's window — and two passes that derive the same `sequence` with the same
|
|
290
|
+
* `generatedAt` ms and chunk count make the SERVER derive one snapshotId and
|
|
291
|
+
* fold both passes' chunk sets into it. That is exactly the silent
|
|
292
|
+
* cross-contamination this audit exists to detect, manufactured by the audit.
|
|
293
|
+
*
|
|
294
|
+
* The scope is prefixed so it can never collide with the `operation` /
|
|
295
|
+
* `reindex` gates, and carries the snapshot store's injective slug so two
|
|
296
|
+
* tenants can never share one lock.
|
|
297
|
+
*/
|
|
298
|
+
function manifestUploadLockScope(scopeKey) {
|
|
299
|
+
return `manifest-${manifestSnapshotScopeSlug(scopeKey)}`;
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* Take the per-scope gate, or report that someone else holds it.
|
|
303
|
+
*
|
|
304
|
+
* Three deliberate choices:
|
|
305
|
+
*
|
|
306
|
+
* - **Never wait** (`wait: false`). The daemon calls this after every sync
|
|
307
|
+
* cycle; queueing behind a `hq sync manifest` that is mid-walk would block
|
|
308
|
+
* the sync pass on an audit, which inverts the priority. A contended pass
|
|
309
|
+
* is simply skipped and retried next cycle — the audit is a daily
|
|
310
|
+
* safety net, not a deadline.
|
|
311
|
+
* - **Never throw.** `OperationLockedError` becomes a benign `locked`
|
|
312
|
+
* result; any OTHER failure (an unwritable `~/.hq`, a pathological
|
|
313
|
+
* reclaim race) is logged and the pass proceeds UNSERIALISED. Refusing to
|
|
314
|
+
* upload because a lock file could not be created would let a
|
|
315
|
+
* permissions problem silently disable the audit; proceeding is no worse
|
|
316
|
+
* than the behaviour before this gate existed.
|
|
317
|
+
* - **Stale holders self-heal.** `acquireOperationLock` probes the holder's
|
|
318
|
+
* pid and reclaims the lock when the process is gone, so a machine that
|
|
319
|
+
* was hard-killed mid-pass is not wedged off the audit forever.
|
|
320
|
+
*/
|
|
321
|
+
function acquireScopeLock(options, scopeKey) {
|
|
322
|
+
try {
|
|
323
|
+
return acquireOperationLock(options.hqRoot, MANIFEST_UPLOAD_LOCK_COMMAND, {
|
|
324
|
+
scope: manifestUploadLockScope(scopeKey),
|
|
325
|
+
mode: "exclusive",
|
|
326
|
+
wait: false,
|
|
327
|
+
timeoutSec: 0,
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
catch (err) {
|
|
331
|
+
if (err instanceof OperationLockedError)
|
|
332
|
+
return "locked";
|
|
333
|
+
options.logger?.warn?.("manifest upload lock unavailable; proceeding unserialised", {
|
|
334
|
+
scopeKey,
|
|
335
|
+
error: errorClassOf(err),
|
|
336
|
+
});
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Await one transport call, but never unconditionally.
|
|
342
|
+
*
|
|
343
|
+
* The transport is INJECTED, so this module cannot assume it has a timeout —
|
|
344
|
+
* and the one caller that matters (the desktop daemon) `await`s this pass after
|
|
345
|
+
* every sync cycle. A transport that never settles must therefore not be able
|
|
346
|
+
* to wedge the pass: it is raced against the caller's abort signal and against
|
|
347
|
+
* the upload phase's deadline, and whichever answers first wins.
|
|
348
|
+
*
|
|
349
|
+
* Losing the race does NOT cancel the call — this module has no cancellation
|
|
350
|
+
* channel to an injected function — so the abandoned promise is given a no-op
|
|
351
|
+
* rejection handler up front. Without it, a wedged transport that eventually
|
|
352
|
+
* rejects would surface as an unhandled rejection long after the pass returned,
|
|
353
|
+
* which on a daemon is a crash, not a log line.
|
|
354
|
+
*
|
|
355
|
+
* The timer is cleared in a `finally` on every path (and unref'd besides) so a
|
|
356
|
+
* pending deadline can never keep a short-lived CLI process alive.
|
|
357
|
+
*/
|
|
358
|
+
async function raceTransportCall(call, signal, deadlineMs) {
|
|
359
|
+
const settled = call.then((response) => ({ kind: "settled", response }));
|
|
360
|
+
settled.catch(() => {
|
|
361
|
+
/* see above: the race may abandon this promise; its rejection is not fatal */
|
|
362
|
+
});
|
|
363
|
+
if (signal?.aborted)
|
|
364
|
+
return { kind: "aborted" };
|
|
365
|
+
const remainingMs = deadlineMs - Date.now();
|
|
366
|
+
if (remainingMs <= 0)
|
|
367
|
+
return { kind: "deadline" };
|
|
368
|
+
let timer;
|
|
369
|
+
let onAbort;
|
|
370
|
+
try {
|
|
371
|
+
return await Promise.race([
|
|
372
|
+
settled,
|
|
373
|
+
new Promise((resolve) => {
|
|
374
|
+
timer = setTimeout(() => resolve({ kind: "deadline" }), remainingMs);
|
|
375
|
+
timer.unref?.();
|
|
376
|
+
}),
|
|
377
|
+
new Promise((resolve) => {
|
|
378
|
+
if (!signal)
|
|
379
|
+
return;
|
|
380
|
+
onAbort = () => resolve({ kind: "aborted" });
|
|
381
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
382
|
+
}),
|
|
383
|
+
]);
|
|
384
|
+
}
|
|
385
|
+
finally {
|
|
386
|
+
if (timer !== undefined)
|
|
387
|
+
clearTimeout(timer);
|
|
388
|
+
if (signal !== undefined && onAbort !== undefined)
|
|
389
|
+
signal.removeEventListener("abort", onAbort);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
function assertOption(condition, message) {
|
|
393
|
+
if (!condition)
|
|
394
|
+
throw new Error(`runManifestUploadPass: ${message}`);
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Build and upload one scope's manifest.
|
|
398
|
+
*
|
|
399
|
+
* Chunks go up SEQUENTIALLY in ascending `chunkIndex`: the server folds them
|
|
400
|
+
* into one snapshot keyed by a hash of the shared header fields, and a
|
|
401
|
+
* concurrent send would make a partial-failure state (some chunks in, some
|
|
402
|
+
* not) impossible to reason about from the client side.
|
|
403
|
+
*/
|
|
404
|
+
export async function runManifestUploadPass(options) {
|
|
405
|
+
assertOption(typeof options.hqRoot === "string" && options.hqRoot.length > 0, "hqRoot is required");
|
|
406
|
+
assertOption(typeof options.stateDir === "string" && options.stateDir.length > 0, "stateDir is required");
|
|
407
|
+
assertOption(typeof options.installationId === "string" && options.installationId.length > 0, "installationId is required");
|
|
408
|
+
assertOption(typeof options.machineId === "string" && options.machineId.length > 0, "machineId is required");
|
|
409
|
+
assertOption(typeof options.transport === "function", "transport is required");
|
|
410
|
+
// Throws for a malformed scope (a company scope with no companyUid) — that
|
|
411
|
+
// is a programmer error, and it must not be silently swallowed into a
|
|
412
|
+
// "failed" result that looks like a transient server problem.
|
|
413
|
+
const scopeKey = manifestSnapshotScopeKey(options.scope);
|
|
414
|
+
/**
|
|
415
|
+
* A failed snapshot write is NOT cosmetic, so it must never be silent.
|
|
416
|
+
*
|
|
417
|
+
* `writeManifestSnapshot` swallows disk errors and returns `false`. If that
|
|
418
|
+
* happens the two invariants that ride on the file both break at once, and
|
|
419
|
+
* both break INVISIBLY: `lastUploadAt` is never stored, so the 24h throttle
|
|
420
|
+
* stops firing and every sync cycle pays a full tree walk; and `sequence` is
|
|
421
|
+
* never stored, so the next pass REPLAYS the sequence this one already sent
|
|
422
|
+
* to the server. Log it loudly, and let callers of the success path see it
|
|
423
|
+
* via `nextModeFull`.
|
|
424
|
+
*/
|
|
425
|
+
const persisted = (ok, what) => {
|
|
426
|
+
if (!ok) {
|
|
427
|
+
options.logger?.error?.("manifest snapshot could not be persisted", { scopeKey, what });
|
|
428
|
+
}
|
|
429
|
+
return ok;
|
|
430
|
+
};
|
|
431
|
+
const env = options.env ?? process.env;
|
|
432
|
+
if (isUploadDisabled(env)) {
|
|
433
|
+
// Deliberately BEFORE the build: the tree walk is the expensive half, so a
|
|
434
|
+
// kill switch that still walked would not be much of a kill switch.
|
|
435
|
+
options.logger?.info?.("manifest upload disabled by env", { scopeKey });
|
|
436
|
+
return { status: "disabled", scopeKey };
|
|
437
|
+
}
|
|
438
|
+
// `dryRun` is not serialised: it uploads nothing, persists nothing and burns
|
|
439
|
+
// no sequence, so it has nothing to race with. Taking the gate for a
|
|
440
|
+
// `--print` would only make a concurrent daemon pass skip a cycle for the
|
|
441
|
+
// sake of a read-only build.
|
|
442
|
+
const lock = options.dryRun ? null : acquireScopeLock(options, scopeKey);
|
|
443
|
+
if (lock === "locked") {
|
|
444
|
+
options.logger?.info?.("manifest upload skipped; another pass holds this scope", { scopeKey });
|
|
445
|
+
return { status: "locked", scopeKey };
|
|
446
|
+
}
|
|
447
|
+
try {
|
|
448
|
+
return await runUploadPassBody(options, scopeKey, persisted);
|
|
449
|
+
}
|
|
450
|
+
finally {
|
|
451
|
+
// Released on EVERY path, including a throw out of the body: a pass that
|
|
452
|
+
// crashed while holding the gate would otherwise lock this scope out of
|
|
453
|
+
// the audit until the process exited.
|
|
454
|
+
lock?.release();
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* The pass proper, running under the per-scope gate.
|
|
459
|
+
*
|
|
460
|
+
* Split out purely so the gate can be held across the whole read-modify-write
|
|
461
|
+
* of `sequence` — the read is before the walk, the write is after the last
|
|
462
|
+
* chunk — with a single `finally` that cannot be forgotten.
|
|
463
|
+
*/
|
|
464
|
+
async function runUploadPassBody(options, scopeKey, persisted) {
|
|
465
|
+
const nowMs = options.now ?? Date.now();
|
|
466
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
467
|
+
const previous = readManifestSnapshot(options.stateDir, scopeKey);
|
|
468
|
+
const minIntervalMs = options.minIntervalMs ?? DEFAULT_MANIFEST_UPLOAD_MIN_INTERVAL_MS;
|
|
469
|
+
// `dryRun` bypasses the throttle because printing is read-only: it uploads
|
|
470
|
+
// nothing and persists nothing, so there is no cost to spread out.
|
|
471
|
+
if (!options.ignoreThrottle && !options.dryRun && previous?.lastUploadAt) {
|
|
472
|
+
const lastMs = Date.parse(previous.lastUploadAt);
|
|
473
|
+
// A timestamp in the FUTURE (clock skew, restored backup) is not trusted
|
|
474
|
+
// to throttle — it would otherwise wedge uploads off for as long as the
|
|
475
|
+
// skew lasts.
|
|
476
|
+
if (Number.isFinite(lastMs) && lastMs <= nowMs && nowMs - lastMs < minIntervalMs) {
|
|
477
|
+
return { status: "throttled", scopeKey };
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
// The FAILURE floor, checked separately because it is armed by a different
|
|
481
|
+
// field. Without it, a scope that can never succeed (403 from an unenrolled
|
|
482
|
+
// installation, a 5xx outage, a misconfigured base URL) never writes
|
|
483
|
+
// `lastUploadAt`, so the throttle above never fires and every single sync
|
|
484
|
+
// cycle pays for a full tree walk — the single most expensive thing this
|
|
485
|
+
// module can do — to produce an upload that is guaranteed to fail again.
|
|
486
|
+
if (!options.ignoreThrottle && !options.dryRun && previous?.lastAttemptAt) {
|
|
487
|
+
const floorMs = manifestFailureBackoffMs(previous.consecutiveFailures ?? 0);
|
|
488
|
+
const lastMs = Date.parse(previous.lastAttemptAt);
|
|
489
|
+
// Same future-timestamp guard as above: clock skew must not wedge retries.
|
|
490
|
+
if (floorMs > 0 && Number.isFinite(lastMs) && lastMs <= nowMs && nowMs - lastMs < floorMs) {
|
|
491
|
+
options.logger?.info?.("manifest upload backing off after a failed pass", {
|
|
492
|
+
scopeKey,
|
|
493
|
+
consecutiveFailures: previous.consecutiveFailures ?? 0,
|
|
494
|
+
floorMs,
|
|
495
|
+
});
|
|
496
|
+
return { status: "throttled", scopeKey };
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
// A base that is missing, stale (weekly re-baseline) or explicitly
|
|
500
|
+
// invalidated forces a FULL manifest. Computing it here rather than letting
|
|
501
|
+
// the builder decide keeps the "why was this full?" answer in one place, and
|
|
502
|
+
// lets the result report it.
|
|
503
|
+
const baseUnusable = previous === null || !isManifestSnapshotBaseUsable(previous) || isManifestSnapshotStale(previous, nowMs);
|
|
504
|
+
const forceFull = options.forceFull === true || baseUnusable;
|
|
505
|
+
/**
|
|
506
|
+
* What `nextModeFull` must report on any path that leaves the stored base
|
|
507
|
+
* ALONE (transport blip, abort, unexpected status, 404 soft skip).
|
|
508
|
+
*
|
|
509
|
+
* Nothing reached the server's materialised view on those paths, so the base
|
|
510
|
+
* it already holds is still the right one to send a delta against — and
|
|
511
|
+
* hard-coding `true` there would tell the caller to force a full 70k-file
|
|
512
|
+
* re-send on every network hiccup. Conversely, if there was no usable base
|
|
513
|
+
* to begin with, the next pass IS full and must say so.
|
|
514
|
+
*/
|
|
515
|
+
const nextModeFullIfBaseUntouched = baseUnusable;
|
|
516
|
+
const sequence = (previous?.sequence ?? 0) + 1;
|
|
517
|
+
let built;
|
|
518
|
+
try {
|
|
519
|
+
built = await buildSyncManifest({
|
|
520
|
+
scope: options.scope,
|
|
521
|
+
hqRoot: options.hqRoot,
|
|
522
|
+
stateDir: options.stateDir,
|
|
523
|
+
installationId: options.installationId,
|
|
524
|
+
machineId: options.machineId,
|
|
525
|
+
source: options.source,
|
|
526
|
+
sequence,
|
|
527
|
+
now: nowMs,
|
|
528
|
+
forceFull,
|
|
529
|
+
abortSignal: options.abortSignal,
|
|
530
|
+
maxWallMs: options.maxWallMs,
|
|
531
|
+
hashPolicy: options.hashPolicy,
|
|
532
|
+
maxHashedFiles: options.maxHashedFiles,
|
|
533
|
+
maxHashMs: options.maxHashMs,
|
|
534
|
+
listJournalsImpl: options.listJournalsImpl,
|
|
535
|
+
maxEntriesPerChunk: options.maxEntriesPerChunk,
|
|
536
|
+
// The builder must NEVER persist the snapshot itself: a snapshot is a
|
|
537
|
+
// valid delta base only once the SERVER has materialised it, and the
|
|
538
|
+
// builder cannot know that. This pass persists it, later, under gate.
|
|
539
|
+
persistSnapshot: false,
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
catch (err) {
|
|
543
|
+
options.logger?.warn?.("manifest build failed", { scopeKey, error: errorClassOf(err) });
|
|
544
|
+
// The sequence advances on EVERY outcome, including one that never
|
|
545
|
+
// reached the wire. Keeping the rule unconditional is what makes it
|
|
546
|
+
// auditable: an exception to it is a place where a future edit that DOES
|
|
547
|
+
// upload something can silently reuse a sequence.
|
|
548
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
549
|
+
return {
|
|
550
|
+
status: "failed",
|
|
551
|
+
scopeKey,
|
|
552
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
553
|
+
error: { kind: "build_error", detail: errorClassOf(err) },
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
const { mode, snapshot, chunkPlan } = built;
|
|
557
|
+
// Frozen for the whole pass: the server hashes `chunkCount` into the
|
|
558
|
+
// snapshotId, so it must be identical in every chunk header. Reading it once
|
|
559
|
+
// here — rather than `chunks.length` at each use — is also what lets the
|
|
560
|
+
// chunk set stay unmaterialised.
|
|
561
|
+
const chunkCount = chunkPlan.chunkCount;
|
|
562
|
+
// A TRUNCATED walk must never be uploaded. This is not a refinement of the
|
|
563
|
+
// delta-base rule — it is a correctness rule about the wire.
|
|
564
|
+
//
|
|
565
|
+
// The builder already suppresses `removedPaths` and the snapshot write when
|
|
566
|
+
// the walk ran out of wall clock, which was thought to make a truncated pass
|
|
567
|
+
// safe. It does not. On a real 690k-file personal vault the 30s walk budget
|
|
568
|
+
// expired after ~179k files with ZERO entries assembled (the hash phase had
|
|
569
|
+
// not started), and the pass happily POSTed that as a FULL manifest. A full
|
|
570
|
+
// manifest is a complete statement — "this is everything I hold" — so the
|
|
571
|
+
// server's materialised view for the scope becomes empty and every file the
|
|
572
|
+
// client actually holds reads as missing. The delta-base guard cannot help:
|
|
573
|
+
// the damage is done by the upload itself, server-side.
|
|
574
|
+
//
|
|
575
|
+
// So: no POST, no base, no `lastUploadAt`. The sequence is burned (cheap and
|
|
576
|
+
// keeps the monotonic rule unconditional) and the failure backoff is armed
|
|
577
|
+
// so a scope that cannot be walked inside its budget does not re-walk on
|
|
578
|
+
// every cycle. `dryRun` still prints — a truncated manifest is exactly what
|
|
579
|
+
// an operator debugging this needs to see.
|
|
580
|
+
if (built.walkStats.truncated && !options.dryRun) {
|
|
581
|
+
options.logger?.warn?.("manifest walk truncated; refusing to upload a partial scope", {
|
|
582
|
+
scopeKey,
|
|
583
|
+
filesWalked: built.walkStats.filesWalked,
|
|
584
|
+
entries: chunkPlan.entryCount,
|
|
585
|
+
walkMs: built.walkStats.walkMs,
|
|
586
|
+
});
|
|
587
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "walk_truncated");
|
|
588
|
+
return {
|
|
589
|
+
status: "walk_truncated",
|
|
590
|
+
scopeKey,
|
|
591
|
+
mode,
|
|
592
|
+
chunkCount,
|
|
593
|
+
uploadedChunks: 0,
|
|
594
|
+
// The stored base (if any) is untouched and still describes a COMPLETE
|
|
595
|
+
// walk, so the next pass may legitimately delta against it.
|
|
596
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
597
|
+
};
|
|
598
|
+
}
|
|
599
|
+
// A FULL manifest of nothing-but-disk, from a scope we have a tracked
|
|
600
|
+
// snapshot for, is not a picture of the machine — it is a picture of a
|
|
601
|
+
// ledger we failed to read. Uploading it is destructive in the same way a
|
|
602
|
+
// truncated walk is: `full` means "this is everything I hold", so the
|
|
603
|
+
// server's materialised view becomes "this client tracks nothing", and every
|
|
604
|
+
// file the user actually has reads as never-synced drift.
|
|
605
|
+
//
|
|
606
|
+
// The prior local snapshot is the evidence. It was written by a pass the
|
|
607
|
+
// SERVER materialised, so "this scope had N tracked entries yesterday" is a
|
|
608
|
+
// fact, not a guess — and a scope does not lose all N in one cycle without
|
|
609
|
+
// something being wrong with the reader rather than the disk.
|
|
610
|
+
//
|
|
611
|
+
// Same posture as `walk_truncated`: no POST, no base, no `lastUploadAt`; the
|
|
612
|
+
// sequence is burned and the failure backoff armed. `dryRun` still prints,
|
|
613
|
+
// because this manifest is precisely what an operator debugging it needs.
|
|
614
|
+
const priorTrackedEntries = previous ? Object.keys(previous.entries).length : 0;
|
|
615
|
+
const diskOnlyShare = chunkPlan.entryCount === 0 ? 0 : chunkPlan.diskEntryCount / chunkPlan.entryCount;
|
|
616
|
+
if (mode === "full" &&
|
|
617
|
+
chunkPlan.entryCount > 0 &&
|
|
618
|
+
diskOnlyShare >= MANIFEST_IMPLAUSIBLE_DISK_RATIO &&
|
|
619
|
+
priorTrackedEntries > 0 &&
|
|
620
|
+
!options.dryRun) {
|
|
621
|
+
options.logger?.error?.("manifest ledger looks unreadable; refusing to upload a disk-only scope", {
|
|
622
|
+
scopeKey,
|
|
623
|
+
entries: chunkPlan.entryCount,
|
|
624
|
+
diskEntries: chunkPlan.diskEntryCount,
|
|
625
|
+
priorTrackedEntries,
|
|
626
|
+
});
|
|
627
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "ledger_implausible");
|
|
628
|
+
return {
|
|
629
|
+
status: "ledger_implausible",
|
|
630
|
+
scopeKey,
|
|
631
|
+
mode,
|
|
632
|
+
chunkCount,
|
|
633
|
+
uploadedChunks: 0,
|
|
634
|
+
// The stored base is untouched and still describes a real, tracked
|
|
635
|
+
// scope, so the next pass may legitimately delta against it.
|
|
636
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
if (options.dryRun) {
|
|
640
|
+
return {
|
|
641
|
+
status: "printed",
|
|
642
|
+
scopeKey,
|
|
643
|
+
snapshotId: snapshot.snapshotId,
|
|
644
|
+
mode,
|
|
645
|
+
chunkCount,
|
|
646
|
+
uploadedChunks: 0,
|
|
647
|
+
// The ONE path that legitimately materialises the whole chunk set: an
|
|
648
|
+
// operator asked to SEE it. Nothing is sent, so nothing is streamed.
|
|
649
|
+
chunks: built.chunks,
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
// Real clock, deliberately: `nowMs` may be pinned by the caller to make a
|
|
653
|
+
// manifest reproducible, and a pinned clock would make this deadline either
|
|
654
|
+
// never fire or fire instantly.
|
|
655
|
+
const uploadDeadlineMs = Date.now() + Math.max(0, options.uploadWallMs ?? DEFAULT_MANIFEST_UPLOAD_WALL_MS);
|
|
656
|
+
let uploadedChunks = 0;
|
|
657
|
+
let completed = false;
|
|
658
|
+
let materialised = false;
|
|
659
|
+
// The id the SERVER derived for this pass. The client's own
|
|
660
|
+
// `snapshot.snapshotId` is a local random `snap-*` handle that the server has
|
|
661
|
+
// never seen: the server derives its id by hashing the header fields
|
|
662
|
+
// (`installationId`, scope, `machineId`, `mode`, `generatedAt`, `sequence`,
|
|
663
|
+
// `chunkCount`) and returns it on every accepted chunk. Persisting the LOCAL
|
|
664
|
+
// id as the delta base would quote a `baseSnapshotId` the server can never
|
|
665
|
+
// match, so every delta would be answered `resend_full` forever — the audit
|
|
666
|
+
// would still work, but it would pay a full 70k-file re-send on every single
|
|
667
|
+
// pass and never once exercise the delta path.
|
|
668
|
+
let serverSnapshotId;
|
|
669
|
+
// Set when two chunks of one pass come back under DIFFERENT server ids,
|
|
670
|
+
// which means the header fields drifted mid-pass and the snapshot has been
|
|
671
|
+
// scattered across rows that can each only ever stay partial.
|
|
672
|
+
let snapshotIdDrifted = false;
|
|
673
|
+
// Indexed, not `for (const chunk of chunks)`: each chunk is MINTED here,
|
|
674
|
+
// serialised by the transport, and dropped before the next one is asked for,
|
|
675
|
+
// so the pass holds one chunk at a time instead of the whole set. See
|
|
676
|
+
// `ManifestChunkPlan` for why `chunkCount` is safe to fix up front.
|
|
677
|
+
for (let chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) {
|
|
678
|
+
const chunk = chunkPlan.chunk(chunkIndex);
|
|
679
|
+
if (options.abortSignal?.aborted) {
|
|
680
|
+
// Only the sequence is persisted: a half-sent pass leaves the server's
|
|
681
|
+
// partial fold to expire on its own, but the sequence it already burned
|
|
682
|
+
// must never be handed out twice.
|
|
683
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
684
|
+
return {
|
|
685
|
+
status: "failed",
|
|
686
|
+
scopeKey,
|
|
687
|
+
mode,
|
|
688
|
+
chunkCount,
|
|
689
|
+
uploadedChunks,
|
|
690
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
691
|
+
error: { kind: "aborted", chunkIndex: chunk.chunkIndex },
|
|
692
|
+
};
|
|
693
|
+
}
|
|
694
|
+
const forbidden = findCallerSelectedIdentity(chunk);
|
|
695
|
+
if (forbidden) {
|
|
696
|
+
// This can trip on a chunk AFTER earlier chunks were accepted, so the
|
|
697
|
+
// sequence has already been spent server-side; handing it out again
|
|
698
|
+
// would fold a second pass into the same derived snapshotId.
|
|
699
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
700
|
+
return {
|
|
701
|
+
status: "failed",
|
|
702
|
+
scopeKey,
|
|
703
|
+
mode,
|
|
704
|
+
chunkCount,
|
|
705
|
+
uploadedChunks,
|
|
706
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
707
|
+
error: { kind: "rejected", detail: forbidden, chunkIndex: chunk.chunkIndex },
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
let outcome;
|
|
711
|
+
try {
|
|
712
|
+
outcome = await raceTransportCall(options.transport(chunk, { attempt: 1 }), options.abortSignal, uploadDeadlineMs);
|
|
713
|
+
}
|
|
714
|
+
catch (err) {
|
|
715
|
+
options.logger?.warn?.("manifest upload transport failed", {
|
|
716
|
+
scopeKey,
|
|
717
|
+
chunkIndex: chunk.chunkIndex,
|
|
718
|
+
error: errorClassOf(err),
|
|
719
|
+
});
|
|
720
|
+
// No `lastUploadAt` write — a network blip must be retried on the next
|
|
721
|
+
// cycle, not throttled away for a day — but the sequence still advances.
|
|
722
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
723
|
+
return {
|
|
724
|
+
status: "failed",
|
|
725
|
+
scopeKey,
|
|
726
|
+
mode,
|
|
727
|
+
chunkCount,
|
|
728
|
+
uploadedChunks,
|
|
729
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
730
|
+
error: { kind: "transport_error", detail: errorClassOf(err), chunkIndex: chunk.chunkIndex },
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
if (outcome.kind !== "settled") {
|
|
734
|
+
// Abort and deadline are the same shape of event — "stop sending, the
|
|
735
|
+
// caller is no longer waiting on us" — and both take the same
|
|
736
|
+
// non-throwing exit: burn the sequence (it may already be spent
|
|
737
|
+
// server-side by an earlier chunk of this pass), leave `lastUploadAt`
|
|
738
|
+
// and the delta base alone so the next cycle retries, and return.
|
|
739
|
+
const aborted = outcome.kind === "aborted";
|
|
740
|
+
options.logger?.warn?.(aborted
|
|
741
|
+
? "manifest upload aborted mid-flight"
|
|
742
|
+
: "manifest upload exceeded its wall clock; stopping", { scopeKey, chunkIndex: chunk.chunkIndex, uploadedChunks });
|
|
743
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
744
|
+
return {
|
|
745
|
+
status: "failed",
|
|
746
|
+
scopeKey,
|
|
747
|
+
mode,
|
|
748
|
+
chunkCount,
|
|
749
|
+
uploadedChunks,
|
|
750
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
751
|
+
error: { kind: aborted ? "aborted" : "upload_timeout", chunkIndex: chunk.chunkIndex },
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
const { status, body } = outcome.response;
|
|
755
|
+
if (readsResendFull(status, body)) {
|
|
756
|
+
// The server lost (or refused) this snapshot. Stop IMMEDIATELY — the
|
|
757
|
+
// remaining chunks would fold into a snapshot that no longer exists —
|
|
758
|
+
// and drop the local base so the next pass is full. `snapshot_abandoned`
|
|
759
|
+
// additionally means "do not retry this snapshotId at all", which is
|
|
760
|
+
// exactly what stopping the pass achieves.
|
|
761
|
+
const reason = readString(body, "reason");
|
|
762
|
+
options.logger?.info?.("server requested a full manifest resend", {
|
|
763
|
+
scopeKey,
|
|
764
|
+
chunkIndex: chunk.chunkIndex,
|
|
765
|
+
reason,
|
|
766
|
+
});
|
|
767
|
+
persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "resend_full");
|
|
768
|
+
return {
|
|
769
|
+
status: "resend_full_scheduled",
|
|
770
|
+
scopeKey,
|
|
771
|
+
mode,
|
|
772
|
+
chunkCount,
|
|
773
|
+
uploadedChunks,
|
|
774
|
+
nextModeFull: true,
|
|
775
|
+
error: reason ? { kind: "rejected", status, detail: reason, chunkIndex: chunk.chunkIndex } : undefined,
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
if (status === 404) {
|
|
779
|
+
// hq-cloud ships to laptops and updates independently of the server, so
|
|
780
|
+
// "route not deployed here yet" describes a perfectly healthy machine.
|
|
781
|
+
// Info, not error — and no delta base, because nothing was received.
|
|
782
|
+
options.logger?.info?.("manifest upload route not available (soft skip)", { scopeKey });
|
|
783
|
+
persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: false }), "soft_skip");
|
|
784
|
+
return {
|
|
785
|
+
status: "soft_skipped",
|
|
786
|
+
scopeKey,
|
|
787
|
+
mode,
|
|
788
|
+
chunkCount,
|
|
789
|
+
uploadedChunks,
|
|
790
|
+
// `invalidateBase: false` above KEEPS a usable base, so this reports
|
|
791
|
+
// what the next pass will actually do rather than a blanket `true`.
|
|
792
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
if (status === 413) {
|
|
796
|
+
const reason = readString(body, "reason");
|
|
797
|
+
const kind = reason === "manifest_too_large" ? "manifest_too_large" : "chunk_too_large";
|
|
798
|
+
options.logger?.warn?.("manifest upload rejected as too large", { scopeKey, reason });
|
|
799
|
+
// Non-retryable within this pass, and the server's view of the scope is
|
|
800
|
+
// now incomplete, so the next pass must be full.
|
|
801
|
+
persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "too_large");
|
|
802
|
+
return {
|
|
803
|
+
status: "failed",
|
|
804
|
+
scopeKey,
|
|
805
|
+
mode,
|
|
806
|
+
chunkCount,
|
|
807
|
+
uploadedChunks,
|
|
808
|
+
nextModeFull: true,
|
|
809
|
+
error: { kind, status, detail: reason, chunkIndex: chunk.chunkIndex },
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
if (status < 200 || status >= 300) {
|
|
813
|
+
options.logger?.warn?.("manifest upload rejected", { scopeKey, status, chunkIndex: chunk.chunkIndex });
|
|
814
|
+
persisted(recordFailedAttempt(options.stateDir, scopeKey, previous, sequence, nowIso), "sequence");
|
|
815
|
+
return {
|
|
816
|
+
status: "failed",
|
|
817
|
+
scopeKey,
|
|
818
|
+
mode,
|
|
819
|
+
chunkCount,
|
|
820
|
+
uploadedChunks,
|
|
821
|
+
nextModeFull: nextModeFullIfBaseUntouched,
|
|
822
|
+
error: {
|
|
823
|
+
kind: "rejected",
|
|
824
|
+
status,
|
|
825
|
+
detail: readString(body, "code") ?? readString(body, "reason"),
|
|
826
|
+
chunkIndex: chunk.chunkIndex,
|
|
827
|
+
},
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
uploadedChunks += 1;
|
|
831
|
+
const chunkSnapshotId = readString(body, "snapshotId");
|
|
832
|
+
if (chunkSnapshotId !== undefined) {
|
|
833
|
+
if (serverSnapshotId !== undefined && serverSnapshotId !== chunkSnapshotId)
|
|
834
|
+
snapshotIdDrifted = true;
|
|
835
|
+
serverSnapshotId = chunkSnapshotId;
|
|
836
|
+
}
|
|
837
|
+
completed = readBoolean(body, "completed");
|
|
838
|
+
materialised = readBoolean(body, "materialised");
|
|
839
|
+
}
|
|
840
|
+
if (snapshotIdDrifted) {
|
|
841
|
+
// Belt-and-braces against a builder regression: the chunker stamps the
|
|
842
|
+
// header once, so this can only fire if that stops being true. Refuse the
|
|
843
|
+
// base rather than persist an id that names only part of the snapshot.
|
|
844
|
+
options.logger?.error?.("manifest upload snapshot id drifted mid-pass", { scopeKey });
|
|
845
|
+
materialised = false;
|
|
846
|
+
}
|
|
847
|
+
if (completed && materialised && serverSnapshotId !== undefined) {
|
|
848
|
+
// THE gate. Only now is this snapshot a legitimate delta base: the server
|
|
849
|
+
// has folded every chunk and written the materialised view it will compare
|
|
850
|
+
// the next delta against.
|
|
851
|
+
const stored = persisted(writeManifestSnapshot(options.stateDir, scopeKey, {
|
|
852
|
+
...snapshot,
|
|
853
|
+
// The SERVER's id, not ours — this value's only job is to be quoted
|
|
854
|
+
// back as the next delta's `baseSnapshotId`.
|
|
855
|
+
snapshotId: serverSnapshotId,
|
|
856
|
+
lastUploadAt: nowIso,
|
|
857
|
+
baseValid: true,
|
|
858
|
+
}), "delta_base");
|
|
859
|
+
return {
|
|
860
|
+
status: "uploaded",
|
|
861
|
+
scopeKey,
|
|
862
|
+
snapshotId: serverSnapshotId,
|
|
863
|
+
mode,
|
|
864
|
+
chunkCount,
|
|
865
|
+
uploadedChunks,
|
|
866
|
+
// If the write failed there is no base on disk, so the next pass cannot
|
|
867
|
+
// be a delta — reporting `false` here would tell the caller a base
|
|
868
|
+
// exists that does not.
|
|
869
|
+
nextModeFull: !stored,
|
|
870
|
+
};
|
|
871
|
+
}
|
|
872
|
+
// Every chunk was accepted but the fold was refused (`materialised: false`)
|
|
873
|
+
// or never completed. The client and the server now disagree about what the
|
|
874
|
+
// server holds, and only a FULL manifest can resolve that.
|
|
875
|
+
// Also covers an accepted pass whose responses carried no `snapshotId`: with
|
|
876
|
+
// no server id there is no base to quote, so the next pass must be full.
|
|
877
|
+
options.logger?.warn?.("manifest upload was not materialised; next pass will be full", {
|
|
878
|
+
scopeKey,
|
|
879
|
+
completed,
|
|
880
|
+
materialised,
|
|
881
|
+
hasServerSnapshotId: serverSnapshotId !== undefined,
|
|
882
|
+
});
|
|
883
|
+
persisted(recordUploadAttempt(options.stateDir, scopeKey, previous, sequence, nowIso, { invalidateBase: true }), "not_materialised");
|
|
884
|
+
return {
|
|
885
|
+
status: "uploaded",
|
|
886
|
+
scopeKey,
|
|
887
|
+
snapshotId: serverSnapshotId ?? snapshot.snapshotId,
|
|
888
|
+
mode,
|
|
889
|
+
chunkCount,
|
|
890
|
+
uploadedChunks,
|
|
891
|
+
nextModeFull: true,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Read per-scope upload state without touching the network or the disk tree.
|
|
896
|
+
*
|
|
897
|
+
* This is the seam `hq doctor`'s `manifest` check consumes: it answers "when
|
|
898
|
+
* did this machine last tell the server what it holds?" for each scope, which
|
|
899
|
+
* is the one question that distinguishes "no findings" from "no data".
|
|
900
|
+
* Missing/older snapshots report `lastUploadAt: null` — never uploaded.
|
|
901
|
+
*/
|
|
902
|
+
export function readManifestUploadStatus(stateDir, scopes) {
|
|
903
|
+
return scopes.map((scope) => {
|
|
904
|
+
const scopeKey = manifestSnapshotScopeKey(scope);
|
|
905
|
+
const snapshot = readManifestSnapshot(stateDir, scopeKey);
|
|
906
|
+
if (!snapshot) {
|
|
907
|
+
return { scopeKey, lastUploadAt: null, snapshotId: null, sequence: 0, baseUsable: false };
|
|
908
|
+
}
|
|
909
|
+
const baseUsable = isManifestSnapshotBaseUsable(snapshot);
|
|
910
|
+
return {
|
|
911
|
+
scopeKey,
|
|
912
|
+
lastUploadAt: snapshot.lastUploadAt ?? null,
|
|
913
|
+
snapshotId: baseUsable ? snapshot.snapshotId : null,
|
|
914
|
+
sequence: snapshot.sequence,
|
|
915
|
+
baseUsable,
|
|
916
|
+
};
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
/** Re-exported so callers do not need a second import for the staleness rule. */
|
|
920
|
+
export { MANIFEST_SNAPSHOT_MAX_AGE_MS };
|
|
921
|
+
//# sourceMappingURL=upload-manifest.js.map
|