@indigoai-us/hq-cloud 6.16.20 → 6.16.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-installation-id.d.ts +66 -0
- package/dist/bin/sync-runner-installation-id.d.ts.map +1 -0
- package/dist/bin/sync-runner-installation-id.js +138 -0
- package/dist/bin/sync-runner-installation-id.js.map +1 -0
- package/dist/bin/sync-runner-manifest.d.ts +120 -0
- package/dist/bin/sync-runner-manifest.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.js +348 -0
- package/dist/bin/sync-runner-manifest.js.map +1 -0
- package/dist/bin/sync-runner-manifest.test.d.ts +12 -0
- package/dist/bin/sync-runner-manifest.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.test.js +510 -0
- package/dist/bin/sync-runner-manifest.test.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +55 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +69 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +217 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/company-slug.d.ts +31 -0
- package/dist/company-slug.d.ts.map +1 -0
- package/dist/company-slug.js +52 -0
- package/dist/company-slug.js.map +1 -0
- package/dist/company-slug.test.d.ts +7 -0
- package/dist/company-slug.test.d.ts.map +1 -0
- package/dist/company-slug.test.js +43 -0
- package/dist/company-slug.test.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/journal-ledger-index.test.d.ts +15 -0
- package/dist/journal-ledger-index.test.d.ts.map +1 -0
- package/dist/journal-ledger-index.test.js +247 -0
- package/dist/journal-ledger-index.test.js.map +1 -0
- package/dist/journal.d.ts +110 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +253 -5
- package/dist/journal.js.map +1 -1
- package/dist/manifest/US-002.acceptance.test.d.ts +11 -0
- package/dist/manifest/US-002.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-002.acceptance.test.js +160 -0
- package/dist/manifest/US-002.acceptance.test.js.map +1 -0
- package/dist/manifest/US-004.acceptance.test.d.ts +12 -0
- package/dist/manifest/US-004.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-004.acceptance.test.js +173 -0
- package/dist/manifest/US-004.acceptance.test.js.map +1 -0
- package/dist/manifest/build-manifest.bench.test.d.ts +22 -0
- package/dist/manifest/build-manifest.bench.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.bench.test.js +563 -0
- package/dist/manifest/build-manifest.bench.test.js.map +1 -0
- package/dist/manifest/build-manifest.d.ts +355 -0
- package/dist/manifest/build-manifest.d.ts.map +1 -0
- package/dist/manifest/build-manifest.js +1139 -0
- package/dist/manifest/build-manifest.js.map +1 -0
- package/dist/manifest/build-manifest.test.d.ts +10 -0
- package/dist/manifest/build-manifest.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.test.js +1172 -0
- package/dist/manifest/build-manifest.test.js.map +1 -0
- package/dist/manifest/contract.d.ts +356 -0
- package/dist/manifest/contract.d.ts.map +1 -0
- package/dist/manifest/contract.js +542 -0
- package/dist/manifest/contract.js.map +1 -0
- package/dist/manifest/contract.test.d.ts +17 -0
- package/dist/manifest/contract.test.d.ts.map +1 -0
- package/dist/manifest/contract.test.js +304 -0
- package/dist/manifest/contract.test.js.map +1 -0
- package/dist/manifest/import-graph.test.d.ts +25 -0
- package/dist/manifest/import-graph.test.d.ts.map +1 -0
- package/dist/manifest/import-graph.test.js +120 -0
- package/dist/manifest/import-graph.test.js.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts +18 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.js +153 -0
- package/dist/manifest/ledger-shard-isolation.test.js.map +1 -0
- package/dist/manifest/snapshot-store.d.ts +160 -0
- package/dist/manifest/snapshot-store.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.js +300 -0
- package/dist/manifest/snapshot-store.js.map +1 -0
- package/dist/manifest/snapshot-store.test.d.ts +11 -0
- package/dist/manifest/snapshot-store.test.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.test.js +133 -0
- package/dist/manifest/snapshot-store.test.js.map +1 -0
- package/dist/manifest/upload-manifest.d.ts +336 -0
- package/dist/manifest/upload-manifest.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.js +1022 -0
- package/dist/manifest/upload-manifest.js.map +1 -0
- package/dist/manifest/upload-manifest.test.d.ts +10 -0
- package/dist/manifest/upload-manifest.test.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.test.js +1116 -0
- package/dist/manifest/upload-manifest.test.js.map +1 -0
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +1 -26
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/posix-key.d.ts +36 -0
- package/dist/posix-key.d.ts.map +1 -0
- package/dist/posix-key.js +38 -0
- package/dist/posix-key.js.map +1 -0
- package/dist/s3.d.ts +2 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +6 -20
- package/dist/s3.js.map +1 -1
- package/dist/sync/journal-ledger-stream.d.ts +56 -0
- package/dist/sync/journal-ledger-stream.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.js +191 -0
- package/dist/sync/journal-ledger-stream.js.map +1 -0
- package/dist/sync/journal-ledger-stream.test.d.ts +12 -0
- package/dist/sync/journal-ledger-stream.test.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.test.js +134 -0
- package/dist/sync/journal-ledger-stream.test.js.map +1 -0
- package/package.json +2 -1
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side sync manifest generator (sync-reconciliation-audit US-002).
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS FOR
|
|
5
|
+
* ----------------
|
|
6
|
+
* The reconciliation audit compares three pictures of a scope: what the vault
|
|
7
|
+
* holds, what this machine's journal (the "ledger") believes it synced, and
|
|
8
|
+
* what is actually ON DISK. Only the client can produce the last two, and only
|
|
9
|
+
* together do they distinguish the failure modes that matter:
|
|
10
|
+
*
|
|
11
|
+
* - a file on disk with no ledger row → the daemon stopped tracking it
|
|
12
|
+
* (wedged watcher, crashed push) — invisible to any server-side check;
|
|
13
|
+
* - a ledger row with no file on disk → the local copy vanished;
|
|
14
|
+
* - both present and in agreement → healthy.
|
|
15
|
+
*
|
|
16
|
+
* That is exactly the `disk` / `ledger` / `both` split the wire contract
|
|
17
|
+
* encodes, and it is why this builder unions the two sources instead of
|
|
18
|
+
* walking the disk alone.
|
|
19
|
+
*
|
|
20
|
+
* WHAT THIS DELIBERATELY DOES NOT DO
|
|
21
|
+
* ----------------------------------
|
|
22
|
+
* - It never reads `sync-journal.*.json` by path. The v3 journal's real
|
|
23
|
+
* state lives in a digest-named store; the legacy filename is only a
|
|
24
|
+
* locator. `listJournals()` is the ONLY supported way to enumerate shards.
|
|
25
|
+
* - It never re-implements ignore rules. `createIgnoreFilter()` owns the
|
|
26
|
+
* layered DEFAULT_IGNORES / .hqignore / .hqinclude semantics; a second
|
|
27
|
+
* implementation here would drift and manufacture false findings for every
|
|
28
|
+
* file the two disagree about.
|
|
29
|
+
* - It never reads file CONTENT unless it has to. The walk is stat-only; a
|
|
30
|
+
* hash is computed only when the ledger has no usable one (see
|
|
31
|
+
* {@link canReuseLedgerHash}). On a healthy machine that is ~zero files.
|
|
32
|
+
*
|
|
33
|
+
* COST DISCIPLINE
|
|
34
|
+
* ---------------
|
|
35
|
+
* This runs on the user's laptop, in the background, beside their editor. The
|
|
36
|
+
* walk yields to the event loop every `yieldEvery` files, honours an
|
|
37
|
+
* `AbortSignal`, and hard-stops at `maxWallMs` — marking the manifest
|
|
38
|
+
* `truncated` rather than pretending a partial picture is complete. A
|
|
39
|
+
* truncated manifest never emits `removedPaths` and never overwrites the
|
|
40
|
+
* snapshot, because "I stopped early" and "these files are gone" are the same
|
|
41
|
+
* observation from the walker's point of view and only one of them is true.
|
|
42
|
+
*/
|
|
43
|
+
import { type JournalSummary } from "../journal.js";
|
|
44
|
+
import { type SyncManifestEntry, type SyncManifestMode, type SyncManifestScope, type SyncManifestSource, type SyncManifestUpload, type SyncManifestWalkStats } from "./contract.js";
|
|
45
|
+
import { type ManifestSnapshot } from "./snapshot-store.js";
|
|
46
|
+
/**
|
|
47
|
+
* The contract's {@link SyncManifestWalkStats} has NO `truncated` field, and
|
|
48
|
+
* the contract file is byte-mirrored with hq-pro — adding one here would break
|
|
49
|
+
* a test in BOTH repos.
|
|
50
|
+
*
|
|
51
|
+
* The client still needs to say "this manifest is partial", so the flag rides
|
|
52
|
+
* as an EXTRA field on the walk stats. The contract validator is explicitly
|
|
53
|
+
* tolerant of unknown extra fields (a newer client must not fail an older
|
|
54
|
+
* server), so a chunk carrying it still validates today and the field becomes
|
|
55
|
+
* meaningful to the server the moment the contract adopts it.
|
|
56
|
+
*/
|
|
57
|
+
export interface BuildWalkStats extends SyncManifestWalkStats {
|
|
58
|
+
truncated: boolean;
|
|
59
|
+
/**
|
|
60
|
+
* Directories the walk could not `readdir` (EACCES, a raced unmount, a
|
|
61
|
+
* transient FUSE error). Everything beneath them is INVISIBLE this cycle —
|
|
62
|
+
* which is not the same as absent — so the builder suppresses ledger-only
|
|
63
|
+
* verdicts and removals under those prefixes (see `lostVisibilityPrefixes`).
|
|
64
|
+
* Reported so an operator can tell "clean scope" from "clean scope, plus a
|
|
65
|
+
* subtree I was never allowed to look at".
|
|
66
|
+
*/
|
|
67
|
+
unreadableDirs: number;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Scope plus the local knowledge the wire contract deliberately omits. The
|
|
71
|
+
* contract carries `companyUid` (the server's identifier); the local walk
|
|
72
|
+
* needs the SLUG, because that is what names both the on-disk directory
|
|
73
|
+
* (`companies/{slug}`) and the journal shard.
|
|
74
|
+
*/
|
|
75
|
+
export interface BuildManifestScope extends SyncManifestScope {
|
|
76
|
+
/** Company directory/journal slug. Required for `kind: "company"`. */
|
|
77
|
+
slug?: string;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* How aggressively the builder may open files.
|
|
81
|
+
*
|
|
82
|
+
* - `never` (the DEFAULT, and what the sync-runner tail step uses) — never
|
|
83
|
+
* open a file. A hash is reused from the journal when its (size, mtime)
|
|
84
|
+
* pair still matches; otherwise the entry ships WITHOUT one, meaning
|
|
85
|
+
* "present, hash unknown". This is a pure stat walk: bounded, cheap, and
|
|
86
|
+
* safe to run after every sync cycle.
|
|
87
|
+
* - `auto` — like `never`, but may hash up to `maxHashedFiles` files (and
|
|
88
|
+
* for up to `maxHashMs`) per pass; everything past the cap ships unhashed.
|
|
89
|
+
* Successive passes therefore fill the picture in without any single pass
|
|
90
|
+
* costing minutes.
|
|
91
|
+
* - `full` — hash every file the journal cannot cover, unbounded. Only ever
|
|
92
|
+
* from an explicit human invocation (`hq sync manifest --full-hash`),
|
|
93
|
+
* never from the daemon. This was the previous `auto`, and on a 690k-file
|
|
94
|
+
* vault with a cold journal it measured ~9m40s / 2.9 GB RSS.
|
|
95
|
+
* - `ledger-only` — never open a file AND DROP entries with no reusable
|
|
96
|
+
* hash. Retained for the stat-walk benchmark, which wants to time the walk
|
|
97
|
+
* rather than sha256 throughput. Not for production use: dropping an entry
|
|
98
|
+
* reports a file the client holds as one it does not.
|
|
99
|
+
*/
|
|
100
|
+
export type ManifestHashPolicy = "auto" | "ledger-only" | "full" | "never";
|
|
101
|
+
/**
|
|
102
|
+
* Default per-pass ceiling on files the `auto` policy may READ.
|
|
103
|
+
*
|
|
104
|
+
* Sized so the hash phase stays comparable to the stat walk it follows rather
|
|
105
|
+
* than dwarfing it: a few thousand small files is sub-second, while the
|
|
106
|
+
* unbounded version of this loop measured in minutes on a real vault.
|
|
107
|
+
*/
|
|
108
|
+
export declare const DEFAULT_MANIFEST_MAX_HASHED_FILES = 2000;
|
|
109
|
+
/** Default per-pass wall-clock ceiling on the `auto` policy's hash phase. */
|
|
110
|
+
export declare const DEFAULT_MANIFEST_MAX_HASH_MS = 5000;
|
|
111
|
+
export interface BuildSyncManifestOptions {
|
|
112
|
+
scope: BuildManifestScope;
|
|
113
|
+
hqRoot: string;
|
|
114
|
+
stateDir: string;
|
|
115
|
+
installationId: string;
|
|
116
|
+
machineId: string;
|
|
117
|
+
source: SyncManifestSource;
|
|
118
|
+
sequence: number;
|
|
119
|
+
/** Injected clock (epoch ms) — tests pin it; production omits it. */
|
|
120
|
+
now?: number;
|
|
121
|
+
abortSignal?: AbortSignal;
|
|
122
|
+
/** Server said `resend_full`, or the caller wants a fresh baseline. */
|
|
123
|
+
forceFull?: boolean;
|
|
124
|
+
maxWallMs?: number;
|
|
125
|
+
yieldEvery?: number;
|
|
126
|
+
hashPolicy?: ManifestHashPolicy;
|
|
127
|
+
/** `auto` only: files this pass may read. Defaults to {@link DEFAULT_MANIFEST_MAX_HASHED_FILES}. */
|
|
128
|
+
maxHashedFiles?: number;
|
|
129
|
+
/** `auto` only: wall clock this pass may spend hashing. Defaults to {@link DEFAULT_MANIFEST_MAX_HASH_MS}. */
|
|
130
|
+
maxHashMs?: number;
|
|
131
|
+
/**
|
|
132
|
+
* Test seam: stands in for the real per-slug journal read.
|
|
133
|
+
*
|
|
134
|
+
* Shaped as the whole-machine list for historical reasons — the builder now
|
|
135
|
+
* reads only the ONE shard it needs (see `readLedger`), and when this seam is
|
|
136
|
+
* supplied it simply picks that slug out of the fabricated list.
|
|
137
|
+
*/
|
|
138
|
+
listJournalsImpl?: () => JournalSummary[];
|
|
139
|
+
/** Tuning seam for the ignore filter's path memo (see MANIFEST_IGNORE_MEMO_LIMIT). */
|
|
140
|
+
ignoreMemoLimit?: number;
|
|
141
|
+
/**
|
|
142
|
+
* Cloud-backed slugs, forwarded to `computePersonalVaultPaths()` for the
|
|
143
|
+
* PERSONAL scope only. Same set the runner derives from live membership.
|
|
144
|
+
* Optional: `companies/manifest.yaml`'s `cloud_uid` markers already exclude
|
|
145
|
+
* designated companies without it, so an omitted set is conservative rather
|
|
146
|
+
* than wrong.
|
|
147
|
+
*/
|
|
148
|
+
teamSyncedSlugs?: ReadonlySet<string>;
|
|
149
|
+
/** Skip persisting the new snapshot (used by read-only callers and benches). */
|
|
150
|
+
persistSnapshot?: boolean;
|
|
151
|
+
/**
|
|
152
|
+
* Serialised-byte budget for one chunk. Forwarded to the planner; see
|
|
153
|
+
* {@link ChunkInput.chunkByteBudget}.
|
|
154
|
+
*/
|
|
155
|
+
chunkByteBudget?: number;
|
|
156
|
+
/**
|
|
157
|
+
* Test seam: entries per chunk, clamped to the contract ceiling.
|
|
158
|
+
*
|
|
159
|
+
* Without it the ONLY way to produce a multi-chunk pass is a 50 000-file
|
|
160
|
+
* fixture, so every multi-chunk rule (ascending chunkIndex, a byte-identical
|
|
161
|
+
* header on every chunk, a mid-pass abort) is untestable — and an untestable
|
|
162
|
+
* rule is one a refactor can delete without a red test.
|
|
163
|
+
*/
|
|
164
|
+
maxEntriesPerChunk?: number;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* A chunk set that has been PLANNED but not materialised.
|
|
168
|
+
*
|
|
169
|
+
* WHY THIS IS NOT JUST `SyncManifestUpload[]`
|
|
170
|
+
* ------------------------------------------
|
|
171
|
+
* The chunk set is pure derived data: every chunk is a window onto `entries`
|
|
172
|
+
* plus a header that is identical across the pass. Holding all of it for the
|
|
173
|
+
* duration of the upload buys nothing — the pass sends one chunk at a time —
|
|
174
|
+
* and it grows with the scope, so it is a copy that gets more expensive
|
|
175
|
+
* exactly on the machines that can least afford it.
|
|
176
|
+
*
|
|
177
|
+
* So the plan reports the two things a caller needs UP FRONT — how many chunks
|
|
178
|
+
* there are, and what mode they are in — and materialises a chunk only when
|
|
179
|
+
* one is asked for. The upload loop asks for chunk `i`, serialises it, POSTs
|
|
180
|
+
* it, and drops it before asking for `i + 1`, so the transient cost is one
|
|
181
|
+
* chunk plus its JSON rather than the whole set plus one JSON.
|
|
182
|
+
*
|
|
183
|
+
* BE HONEST ABOUT THE SIZE OF THIS WIN. A chunk's `entries` is a `slice`, so
|
|
184
|
+
* the set that used to be held was 200,000 POINTERS (~1.6 MB), not a second
|
|
185
|
+
* copy of the entries. It is a real bound and it is the RIGHT shape, but it is
|
|
186
|
+
* not what made the operator's 199k scope peak at 642 MB — measured, this
|
|
187
|
+
* changes peak RSS by nothing on a 200k-file fixture. The dominant terms are
|
|
188
|
+
* the walk records, the entry objects and the snapshot map, all O(entries) and
|
|
189
|
+
* all necessarily live at once because the delta decision needs the complete
|
|
190
|
+
* entry set before the first chunk can be sized. Cutting those needs a
|
|
191
|
+
* two-pass build; this keeps the upload half from adding to them.
|
|
192
|
+
*
|
|
193
|
+
* `chunkCount` MUST be known before the first POST and MUST NOT change during
|
|
194
|
+
* it: the server hashes it (with `generatedAt`, `sequence`, `machineId`,
|
|
195
|
+
* `mode` and `scope`) into the snapshotId, so a pass whose chunk count moved
|
|
196
|
+
* mid-flight would derive two different snapshots for one upload. It is fixed
|
|
197
|
+
* here, after the walk and the delta decision, and every chunk this plan mints
|
|
198
|
+
* carries the same frozen header.
|
|
199
|
+
*/
|
|
200
|
+
export interface ManifestChunkPlan {
|
|
201
|
+
/** Frozen for the whole pass; safe to send in every chunk header. */
|
|
202
|
+
chunkCount: number;
|
|
203
|
+
/** Total entries across every chunk. */
|
|
204
|
+
entryCount: number;
|
|
205
|
+
/**
|
|
206
|
+
* The serialised-byte budget this plan was cut against, AFTER clamping.
|
|
207
|
+
*
|
|
208
|
+
* Reported rather than assumed so the upload pass can log the number that
|
|
209
|
+
* actually shaped the chunks (which is not necessarily the number it asked
|
|
210
|
+
* for — see the clamp in {@link planManifestChunks}) and so a 413 can be
|
|
211
|
+
* diagnosed against a real value instead of a default.
|
|
212
|
+
*/
|
|
213
|
+
chunkByteBudget: number;
|
|
214
|
+
/**
|
|
215
|
+
* Largest chunk this plan will mint, in serialised UTF-8 bytes, as measured
|
|
216
|
+
* by the sizing pass.
|
|
217
|
+
*
|
|
218
|
+
* This is the plan's own answer to "will the server take this?", available
|
|
219
|
+
* WITHOUT materialising a single chunk. It can legitimately exceed
|
|
220
|
+
* {@link ManifestChunkPlan.chunkByteBudget} in exactly one case: a single
|
|
221
|
+
* entry larger than the whole budget, which cannot be split further and is
|
|
222
|
+
* sent alone rather than dropped.
|
|
223
|
+
*/
|
|
224
|
+
maxChunkBytes: number;
|
|
225
|
+
/** How many of those entries carry `source: "disk"` (see the upload guard). */
|
|
226
|
+
diskEntryCount: number;
|
|
227
|
+
mode: SyncManifestMode;
|
|
228
|
+
/** Materialise ONE chunk. The caller is expected to drop it before the next. */
|
|
229
|
+
chunk(chunkIndex: number): SyncManifestUpload;
|
|
230
|
+
}
|
|
231
|
+
export interface BuildSyncManifestResult {
|
|
232
|
+
/**
|
|
233
|
+
* Every chunk, materialised on access.
|
|
234
|
+
*
|
|
235
|
+
* A getter, not a field: `dryRun`/print and the tests legitimately want the
|
|
236
|
+
* whole set, and the production upload path legitimately must never build
|
|
237
|
+
* it. Reading this property on a 199k-entry scope allocates the full chunk
|
|
238
|
+
* set — use {@link chunkPlan} on any path that runs on a user's machine.
|
|
239
|
+
*/
|
|
240
|
+
readonly chunks: SyncManifestUpload[];
|
|
241
|
+
/** The streaming surface the upload pass uses. */
|
|
242
|
+
chunkPlan: ManifestChunkPlan;
|
|
243
|
+
mode: SyncManifestMode;
|
|
244
|
+
walkStats: BuildWalkStats;
|
|
245
|
+
/** The snapshot describing this manifest — persisted unless truncated. */
|
|
246
|
+
snapshot: ManifestSnapshot;
|
|
247
|
+
/** Paths that could not be represented on the wire (see `toManifestPath`). */
|
|
248
|
+
unrepresentablePaths: number;
|
|
249
|
+
/** Entries dropped because a `ledger-only` policy left them without a hash. */
|
|
250
|
+
skippedUnhashed: number;
|
|
251
|
+
/** Entries emitted WITHOUT a hash (stat-only, or past this pass's hash cap). */
|
|
252
|
+
unhashedFiles: number;
|
|
253
|
+
/** Entries dropped because the file changed while it was being hashed. */
|
|
254
|
+
droppedStatRaces: number;
|
|
255
|
+
}
|
|
256
|
+
/** Default wall-clock cap for one scope's walk. */
|
|
257
|
+
export declare const DEFAULT_MANIFEST_MAX_WALL_MS = 30000;
|
|
258
|
+
/** Default number of files walked between event-loop yields. */
|
|
259
|
+
export declare const DEFAULT_MANIFEST_YIELD_EVERY = 500;
|
|
260
|
+
/**
|
|
261
|
+
* Ignore-filter memo size for a manifest walk — deliberately far below the
|
|
262
|
+
* filter's own 50k default.
|
|
263
|
+
*
|
|
264
|
+
* That default is tuned for the sync engine, which probes the SAME paths on
|
|
265
|
+
* every cycle and lives off the cache hits. A manifest walk visits each path
|
|
266
|
+
* exactly once, so the memo can never hit; all it does is retain a key string
|
|
267
|
+
* per file (and keep the `ignore` library's internal per-path cache alive
|
|
268
|
+
* beside it) for the whole walk — tens of megabytes of resident memory on a
|
|
269
|
+
* 70k-file root, bought for zero hits. A small window keeps both layers
|
|
270
|
+
* bounded at the cost of a periodic matcher rebuild.
|
|
271
|
+
*/
|
|
272
|
+
export declare const MANIFEST_IGNORE_MEMO_LIMIT = 2000;
|
|
273
|
+
export type ManifestUploadOutcome = "ok" | "soft_skip" | "resend_full" | "error";
|
|
274
|
+
export interface ManifestUploadClassification {
|
|
275
|
+
outcome: ManifestUploadOutcome;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Classify the manifest upload route's response.
|
|
279
|
+
*
|
|
280
|
+
* The one non-obvious rule: **404 is a SOFT SKIP, not an error.** hq-cloud
|
|
281
|
+
* ships to end-user machines and updates independently of the server, so a
|
|
282
|
+
* client that knows about the manifest route will routinely meet a deployment
|
|
283
|
+
* that does not host it yet. Treating that as an error would light up every
|
|
284
|
+
* such user's logs (and their support tickets) with a failure that describes
|
|
285
|
+
* a perfectly healthy machine. It is logged at info and the cycle moves on.
|
|
286
|
+
*
|
|
287
|
+
* A 409 — or any 2xx/4xx body asking for `resend_full` — means the server lost
|
|
288
|
+
* its delta base, so the next manifest must be FULL.
|
|
289
|
+
*/
|
|
290
|
+
export declare function classifyManifestUploadResponse(status: number, body?: unknown): ManifestUploadClassification;
|
|
291
|
+
/**
|
|
292
|
+
* Build the manifest chunks for one scope.
|
|
293
|
+
*
|
|
294
|
+
* Never throws for environmental reasons (unreadable journal, unreadable
|
|
295
|
+
* directory, undecodable filename, snapshot write failure) — those degrade to
|
|
296
|
+
* a narrower but honest manifest. It DOES throw for programming errors:
|
|
297
|
+
* a company scope without a uid or slug, or a personal scope carrying a
|
|
298
|
+
* company uid, are caller bugs that must fail loudly rather than upload a
|
|
299
|
+
* mislabelled tenant's file list.
|
|
300
|
+
*/
|
|
301
|
+
export declare function buildSyncManifest(options: BuildSyncManifestOptions): Promise<BuildSyncManifestResult>;
|
|
302
|
+
export interface ChunkInput {
|
|
303
|
+
entries: SyncManifestEntry[];
|
|
304
|
+
removedPaths: string[];
|
|
305
|
+
walkStats: BuildWalkStats;
|
|
306
|
+
ignoreRules: string[];
|
|
307
|
+
mode: SyncManifestMode;
|
|
308
|
+
baseSnapshotId?: string;
|
|
309
|
+
generatedAt: string;
|
|
310
|
+
installationId: string;
|
|
311
|
+
machineId: string;
|
|
312
|
+
scope: SyncManifestScope;
|
|
313
|
+
source: SyncManifestSource;
|
|
314
|
+
sequence: number;
|
|
315
|
+
ledgerUnavailable?: {
|
|
316
|
+
errorClass: string;
|
|
317
|
+
};
|
|
318
|
+
/** Test seam; clamped to `SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK`. */
|
|
319
|
+
maxEntriesPerChunk?: number;
|
|
320
|
+
/**
|
|
321
|
+
* Serialised-byte budget for ONE chunk, clamped to
|
|
322
|
+
* `SYNC_MANIFEST_MAX_CHUNK_BYTES`. Omitted means
|
|
323
|
+
* `SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET`.
|
|
324
|
+
*
|
|
325
|
+
* Production DOES set this — unlike `maxEntriesPerChunk`, it is not a test
|
|
326
|
+
* seam. The upload pass halves it after a 413 and persists the reduced
|
|
327
|
+
* value, so a scope whose entries are unusually fat self-heals instead of
|
|
328
|
+
* failing the same way every pass.
|
|
329
|
+
*/
|
|
330
|
+
chunkByteBudget?: number;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Split entries into contract-sized chunks. ALWAYS emits at least one chunk:
|
|
334
|
+
* an empty scope is a real, meaningful observation ("this machine has nothing
|
|
335
|
+
* here"), and swallowing it would look identical to "the client never ran".
|
|
336
|
+
*
|
|
337
|
+
* `removedPaths` ride the FIRST chunk only, so the server never has to
|
|
338
|
+
* de-duplicate a removal repeated across chunks.
|
|
339
|
+
*/
|
|
340
|
+
export declare function chunkManifest(input: ChunkInput): SyncManifestUpload[];
|
|
341
|
+
/**
|
|
342
|
+
* Plan the chunk set without building it.
|
|
343
|
+
*
|
|
344
|
+
* Everything a chunk header carries is decided HERE, once — `chunkCount`
|
|
345
|
+
* included — so every chunk the plan later mints is byte-identical in the
|
|
346
|
+
* fields the server hashes into the snapshotId. Only the `entries` window and
|
|
347
|
+
* `chunkIndex` differ between chunks, which is the whole reason a chunk can be
|
|
348
|
+
* built on demand and thrown away instead of being held for the pass.
|
|
349
|
+
*
|
|
350
|
+
* ALWAYS plans at least one chunk: an empty scope is a real, meaningful
|
|
351
|
+
* observation ("this machine has nothing here"), and swallowing it would look
|
|
352
|
+
* identical to "the client never ran".
|
|
353
|
+
*/
|
|
354
|
+
export declare function planManifestChunks(input: ChunkInput): ManifestChunkPlan;
|
|
355
|
+
//# sourceMappingURL=build-manifest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-manifest.d.ts","sourceRoot":"","sources":["../../src/manifest/build-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAUH,OAAO,EAML,KAAK,cAAc,EAGpB,MAAM,eAAe,CAAC;AAQvB,OAAO,EAYL,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,qBAAqB,EAC3B,MAAM,eAAe,CAAC;AACvB,OAAO,EAOL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAI7B;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAe,SAAQ,qBAAqB;IAC3D,SAAS,EAAE,OAAO,CAAC;IACnB;;;;;;;OAOG;IACH,cAAc,EAAE,MAAM,CAAC;CACxB;AAID;;;;;GAKG;AACH,MAAM,WAAW,kBAAmB,SAAQ,iBAAiB;IAC3D,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,MAAM,kBAAkB,GAAG,MAAM,GAAG,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AAE3E;;;;;;GAMG;AACH,eAAO,MAAM,iCAAiC,OAAQ,CAAC;AAEvD,6EAA6E;AAC7E,eAAO,MAAM,4BAA4B,OAAQ,CAAC;AAElD,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,kBAAkB,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,uEAAuE;IACvE,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,kBAAkB,CAAC;IAChC,oGAAoG;IACpG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6GAA6G;IAC7G,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,MAAM,cAAc,EAAE,CAAC;IAC1C,sFAAsF;IACtF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC,gFAAgF;IAChF,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,WAAW,iBAAiB;IAChC,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;;;OASG;IACH,aAAa,EAAE,MAAM,CAAC;IACtB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,gBAAgB,CAAC;IACvB,gFAAgF;IAChF,KAAK,CAAC,UAAU,EAAE,MAAM,GAAG,kBAAkB,CAAC;CAC/C;AAED,MAAM,WAAW,uBAAuB;IACtC;;;;;;;OAOG;IACH,QAAQ,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACtC,kDAAkD;IAClD,SAAS,EAAE,iBAAiB,CAAC;IAC7B,IAAI,EAAE,gBAAgB,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;IAC1B,0EAA0E;IAC1E,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,8EAA8E;IAC9E,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,eAAe,EAAE,MAAM,CAAC;IACxB,gFAAgF;IAChF,aAAa,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,mDAAmD;AACnD,eAAO,MAAM,4BAA4B,QAAS,CAAC;AACnD,gEAAgE;AAChE,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,0BAA0B,OAAQ,CAAC;AAIhD,MAAM,MAAM,qBAAqB,GAAG,IAAI,GAAG,WAAW,GAAG,aAAa,GAAG,OAAO,CAAC;AAEjF,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,qBAAqB,CAAC;CAChC;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,OAAO,GACb,4BAA4B,CAM9B;AAwjBD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,uBAAuB,CAAC,CAkYlC;AAID,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,IAAI,EAAE,gBAAgB,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,iBAAiB,CAAC;IACzB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,mEAAmE;IACnE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;OASG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,kBAAkB,EAAE,CAOrE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,UAAU,GAAG,iBAAiB,CAwIvE"}
|