@indigoai-us/hq-cloud 6.16.20 → 6.16.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-installation-id.d.ts +66 -0
- package/dist/bin/sync-runner-installation-id.d.ts.map +1 -0
- package/dist/bin/sync-runner-installation-id.js +138 -0
- package/dist/bin/sync-runner-installation-id.js.map +1 -0
- package/dist/bin/sync-runner-manifest.d.ts +120 -0
- package/dist/bin/sync-runner-manifest.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.js +348 -0
- package/dist/bin/sync-runner-manifest.js.map +1 -0
- package/dist/bin/sync-runner-manifest.test.d.ts +12 -0
- package/dist/bin/sync-runner-manifest.test.d.ts.map +1 -0
- package/dist/bin/sync-runner-manifest.test.js +510 -0
- package/dist/bin/sync-runner-manifest.test.js.map +1 -0
- package/dist/bin/sync-runner.d.ts +55 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +69 -0
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +217 -0
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/company-slug.d.ts +31 -0
- package/dist/company-slug.d.ts.map +1 -0
- package/dist/company-slug.js +52 -0
- package/dist/company-slug.js.map +1 -0
- package/dist/company-slug.test.d.ts +7 -0
- package/dist/company-slug.test.d.ts.map +1 -0
- package/dist/company-slug.test.js +43 -0
- package/dist/company-slug.test.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/journal-ledger-index.test.d.ts +15 -0
- package/dist/journal-ledger-index.test.d.ts.map +1 -0
- package/dist/journal-ledger-index.test.js +247 -0
- package/dist/journal-ledger-index.test.js.map +1 -0
- package/dist/journal.d.ts +110 -0
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +253 -5
- package/dist/journal.js.map +1 -1
- package/dist/manifest/US-002.acceptance.test.d.ts +11 -0
- package/dist/manifest/US-002.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-002.acceptance.test.js +160 -0
- package/dist/manifest/US-002.acceptance.test.js.map +1 -0
- package/dist/manifest/US-004.acceptance.test.d.ts +12 -0
- package/dist/manifest/US-004.acceptance.test.d.ts.map +1 -0
- package/dist/manifest/US-004.acceptance.test.js +173 -0
- package/dist/manifest/US-004.acceptance.test.js.map +1 -0
- package/dist/manifest/build-manifest.bench.test.d.ts +22 -0
- package/dist/manifest/build-manifest.bench.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.bench.test.js +563 -0
- package/dist/manifest/build-manifest.bench.test.js.map +1 -0
- package/dist/manifest/build-manifest.d.ts +355 -0
- package/dist/manifest/build-manifest.d.ts.map +1 -0
- package/dist/manifest/build-manifest.js +1139 -0
- package/dist/manifest/build-manifest.js.map +1 -0
- package/dist/manifest/build-manifest.test.d.ts +10 -0
- package/dist/manifest/build-manifest.test.d.ts.map +1 -0
- package/dist/manifest/build-manifest.test.js +1172 -0
- package/dist/manifest/build-manifest.test.js.map +1 -0
- package/dist/manifest/contract.d.ts +356 -0
- package/dist/manifest/contract.d.ts.map +1 -0
- package/dist/manifest/contract.js +542 -0
- package/dist/manifest/contract.js.map +1 -0
- package/dist/manifest/contract.test.d.ts +17 -0
- package/dist/manifest/contract.test.d.ts.map +1 -0
- package/dist/manifest/contract.test.js +304 -0
- package/dist/manifest/contract.test.js.map +1 -0
- package/dist/manifest/import-graph.test.d.ts +25 -0
- package/dist/manifest/import-graph.test.d.ts.map +1 -0
- package/dist/manifest/import-graph.test.js +120 -0
- package/dist/manifest/import-graph.test.js.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts +18 -0
- package/dist/manifest/ledger-shard-isolation.test.d.ts.map +1 -0
- package/dist/manifest/ledger-shard-isolation.test.js +153 -0
- package/dist/manifest/ledger-shard-isolation.test.js.map +1 -0
- package/dist/manifest/snapshot-store.d.ts +160 -0
- package/dist/manifest/snapshot-store.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.js +300 -0
- package/dist/manifest/snapshot-store.js.map +1 -0
- package/dist/manifest/snapshot-store.test.d.ts +11 -0
- package/dist/manifest/snapshot-store.test.d.ts.map +1 -0
- package/dist/manifest/snapshot-store.test.js +133 -0
- package/dist/manifest/snapshot-store.test.js.map +1 -0
- package/dist/manifest/upload-manifest.d.ts +336 -0
- package/dist/manifest/upload-manifest.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.js +1022 -0
- package/dist/manifest/upload-manifest.js.map +1 -0
- package/dist/manifest/upload-manifest.test.d.ts +10 -0
- package/dist/manifest/upload-manifest.test.d.ts.map +1 -0
- package/dist/manifest/upload-manifest.test.js +1116 -0
- package/dist/manifest/upload-manifest.test.js.map +1 -0
- package/dist/manifest-reconcile.d.ts.map +1 -1
- package/dist/manifest-reconcile.js +1 -26
- package/dist/manifest-reconcile.js.map +1 -1
- package/dist/posix-key.d.ts +36 -0
- package/dist/posix-key.d.ts.map +1 -0
- package/dist/posix-key.js +38 -0
- package/dist/posix-key.js.map +1 -0
- package/dist/s3.d.ts +2 -18
- package/dist/s3.d.ts.map +1 -1
- package/dist/s3.js +6 -20
- package/dist/s3.js.map +1 -1
- package/dist/sync/journal-ledger-stream.d.ts +56 -0
- package/dist/sync/journal-ledger-stream.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.js +191 -0
- package/dist/sync/journal-ledger-stream.js.map +1 -0
- package/dist/sync/journal-ledger-stream.test.d.ts +12 -0
- package/dist/sync/journal-ledger-stream.test.d.ts.map +1 -0
- package/dist/sync/journal-ledger-stream.test.js +134 -0
- package/dist/sync/journal-ledger-stream.test.js.map +1 -0
- package/package.json +2 -1
|
@@ -0,0 +1,1139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side sync manifest generator (sync-reconciliation-audit US-002).
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS FOR
|
|
5
|
+
* ----------------
|
|
6
|
+
* The reconciliation audit compares three pictures of a scope: what the vault
|
|
7
|
+
* holds, what this machine's journal (the "ledger") believes it synced, and
|
|
8
|
+
* what is actually ON DISK. Only the client can produce the last two, and only
|
|
9
|
+
* together do they distinguish the failure modes that matter:
|
|
10
|
+
*
|
|
11
|
+
* - a file on disk with no ledger row → the daemon stopped tracking it
|
|
12
|
+
* (wedged watcher, crashed push) — invisible to any server-side check;
|
|
13
|
+
* - a ledger row with no file on disk → the local copy vanished;
|
|
14
|
+
* - both present and in agreement → healthy.
|
|
15
|
+
*
|
|
16
|
+
* That is exactly the `disk` / `ledger` / `both` split the wire contract
|
|
17
|
+
* encodes, and it is why this builder unions the two sources instead of
|
|
18
|
+
* walking the disk alone.
|
|
19
|
+
*
|
|
20
|
+
* WHAT THIS DELIBERATELY DOES NOT DO
|
|
21
|
+
* ----------------------------------
|
|
22
|
+
* - It never reads `sync-journal.*.json` by path. The v3 journal's real
|
|
23
|
+
* state lives in a digest-named store; the legacy filename is only a
|
|
24
|
+
* locator. `listJournals()` is the ONLY supported way to enumerate shards.
|
|
25
|
+
* - It never re-implements ignore rules. `createIgnoreFilter()` owns the
|
|
26
|
+
* layered DEFAULT_IGNORES / .hqignore / .hqinclude semantics; a second
|
|
27
|
+
* implementation here would drift and manufacture false findings for every
|
|
28
|
+
* file the two disagree about.
|
|
29
|
+
* - It never reads file CONTENT unless it has to. The walk is stat-only; a
|
|
30
|
+
* hash is computed only when the ledger has no usable one (see
|
|
31
|
+
* {@link canReuseLedgerHash}). On a healthy machine that is ~zero files.
|
|
32
|
+
*
|
|
33
|
+
* COST DISCIPLINE
|
|
34
|
+
* ---------------
|
|
35
|
+
* This runs on the user's laptop, in the background, beside their editor. The
|
|
36
|
+
* walk yields to the event loop every `yieldEvery` files, honours an
|
|
37
|
+
* `AbortSignal`, and hard-stops at `maxWallMs` — marking the manifest
|
|
38
|
+
* `truncated` rather than pretending a partial picture is complete. A
|
|
39
|
+
* truncated manifest never emits `removedPaths` and never overwrites the
|
|
40
|
+
* snapshot, because "I stopped early" and "these files are gone" are the same
|
|
41
|
+
* observation from the walker's point of view and only one of them is true.
|
|
42
|
+
*/
|
|
43
|
+
import * as fs from "node:fs";
|
|
44
|
+
import * as path from "node:path";
|
|
45
|
+
import { DEFAULT_IGNORES, createIgnoreFilter, } from "../ignore.js";
|
|
46
|
+
import { PERSONAL_VAULT_JOURNAL_SLUG, hashFile, hashSymlinkTarget, readJournalLedgerIndexResult, toLedgerIndex, } from "../journal.js";
|
|
47
|
+
import { isSafeCompanySlug } from "../company-slug.js";
|
|
48
|
+
// Leaf module (node builtins + `personal-vault-exclusions.ts`), so importing it
|
|
49
|
+
// keeps `import-graph.test.ts` green. Sharing it is the whole point: the
|
|
50
|
+
// personal manifest must describe the SAME tree the personal push leg ships.
|
|
51
|
+
import { computePersonalVaultPaths } from "../personal-vault.js";
|
|
52
|
+
import { PERSONAL_VAULT_DEFAULT_EXCLUSIONS, wrapFilterWithPersonalVaultDefaults } from "../personal-vault-exclusions.js";
|
|
53
|
+
import { SESSIONS_SCOPE_PREFIX } from "../sync/pull-scope.js";
|
|
54
|
+
import { SYNC_MANIFEST_CONTRACT_VERSION, SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET, SYNC_MANIFEST_MAX_CHUNK_BYTES, SYNC_MANIFEST_MAX_CHUNK_COUNT, SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK, SYNC_MANIFEST_MAX_FILE_SIZE_BYTES, SYNC_MANIFEST_MAX_IGNORE_RULES, SYNC_MANIFEST_MAX_IGNORE_RULE_LENGTH, SYNC_MANIFEST_MAX_MTIME_MS, SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK, assertSyncManifestPath, } from "./contract.js";
|
|
55
|
+
import { isManifestSnapshotBaseUsable, isManifestSnapshotStale, manifestSnapshotScopeKey, newManifestSnapshotId, readManifestSnapshot, writeManifestSnapshot, } from "./snapshot-store.js";
|
|
56
|
+
/**
|
|
57
|
+
* Default per-pass ceiling on files the `auto` policy may READ.
|
|
58
|
+
*
|
|
59
|
+
* Sized so the hash phase stays comparable to the stat walk it follows rather
|
|
60
|
+
* than dwarfing it: a few thousand small files is sub-second, while the
|
|
61
|
+
* unbounded version of this loop measured in minutes on a real vault.
|
|
62
|
+
*/
|
|
63
|
+
export const DEFAULT_MANIFEST_MAX_HASHED_FILES = 2_000;
|
|
64
|
+
/** Default per-pass wall-clock ceiling on the `auto` policy's hash phase. */
|
|
65
|
+
export const DEFAULT_MANIFEST_MAX_HASH_MS = 5_000;
|
|
66
|
+
/** Default wall-clock cap for one scope's walk. */
|
|
67
|
+
export const DEFAULT_MANIFEST_MAX_WALL_MS = 30_000;
|
|
68
|
+
/** Default number of files walked between event-loop yields. */
|
|
69
|
+
export const DEFAULT_MANIFEST_YIELD_EVERY = 500;
|
|
70
|
+
/**
|
|
71
|
+
* Ignore-filter memo size for a manifest walk — deliberately far below the
|
|
72
|
+
* filter's own 50k default.
|
|
73
|
+
*
|
|
74
|
+
* That default is tuned for the sync engine, which probes the SAME paths on
|
|
75
|
+
* every cycle and lives off the cache hits. A manifest walk visits each path
|
|
76
|
+
* exactly once, so the memo can never hit; all it does is retain a key string
|
|
77
|
+
* per file (and keep the `ignore` library's internal per-path cache alive
|
|
78
|
+
* beside it) for the whole walk — tens of megabytes of resident memory on a
|
|
79
|
+
* 70k-file root, bought for zero hits. A small window keeps both layers
|
|
80
|
+
* bounded at the cost of a periodic matcher rebuild.
|
|
81
|
+
*/
|
|
82
|
+
export const MANIFEST_IGNORE_MEMO_LIMIT = 2_000;
|
|
83
|
+
/**
|
|
84
|
+
* Classify the manifest upload route's response.
|
|
85
|
+
*
|
|
86
|
+
* The one non-obvious rule: **404 is a SOFT SKIP, not an error.** hq-cloud
|
|
87
|
+
* ships to end-user machines and updates independently of the server, so a
|
|
88
|
+
* client that knows about the manifest route will routinely meet a deployment
|
|
89
|
+
* that does not host it yet. Treating that as an error would light up every
|
|
90
|
+
* such user's logs (and their support tickets) with a failure that describes
|
|
91
|
+
* a perfectly healthy machine. It is logged at info and the cycle moves on.
|
|
92
|
+
*
|
|
93
|
+
* A 409 — or any 2xx/4xx body asking for `resend_full` — means the server lost
|
|
94
|
+
* its delta base, so the next manifest must be FULL.
|
|
95
|
+
*/
|
|
96
|
+
export function classifyManifestUploadResponse(status, body) {
|
|
97
|
+
if (status === 404)
|
|
98
|
+
return { outcome: "soft_skip" };
|
|
99
|
+
if (status === 409)
|
|
100
|
+
return { outcome: "resend_full" };
|
|
101
|
+
if (bodyRequestsResendFull(body))
|
|
102
|
+
return { outcome: "resend_full" };
|
|
103
|
+
if (status >= 200 && status < 300)
|
|
104
|
+
return { outcome: "ok" };
|
|
105
|
+
return { outcome: "error" };
|
|
106
|
+
}
|
|
107
|
+
function bodyRequestsResendFull(body) {
|
|
108
|
+
if (body === null || typeof body !== "object" || Array.isArray(body))
|
|
109
|
+
return false;
|
|
110
|
+
const raw = body;
|
|
111
|
+
return raw.action === "resend_full" || raw.reason === "resend_full" || raw.resendFull === true;
|
|
112
|
+
}
|
|
113
|
+
// ─── Path representation ─────────────────────────────────────────────────────
|
|
114
|
+
/**
|
|
115
|
+
* Convert an OS-relative path into the contract's wire form, or `null` when it
|
|
116
|
+
* cannot be represented.
|
|
117
|
+
*
|
|
118
|
+
* macOS hands back NFD (`e` + combining acute); S3 stores whatever bytes it is
|
|
119
|
+
* given. Two spellings of one filename would reconcile as a false
|
|
120
|
+
* MISSING_IN_VAULT plus a false ORPHAN_IN_VAULT for the SAME file, so the wire
|
|
121
|
+
* form is pinned to NFC. Anything the contract still rejects afterwards (a
|
|
122
|
+
* control character, a lone surrogate, a bidi override, an over-long path) is
|
|
123
|
+
* SKIPPED and counted rather than thrown: such a path can never become a vault
|
|
124
|
+
* key anyway, so including it would only manufacture a guaranteed per-file
|
|
125
|
+
* failure downstream — and throwing would let one pathological filename
|
|
126
|
+
* suppress the entire machine's manifest.
|
|
127
|
+
*/
|
|
128
|
+
function toManifestPath(relative) {
|
|
129
|
+
if (!relative)
|
|
130
|
+
return null;
|
|
131
|
+
const posix = relative.split(path.sep).join("/").normalize("NFC");
|
|
132
|
+
try {
|
|
133
|
+
return assertSyncManifestPath("path", posix);
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Derive a bare error CLASS name for the wire.
|
|
141
|
+
*
|
|
142
|
+
* The contract accepts a class name and nothing else, on purpose: an error
|
|
143
|
+
* MESSAGE routinely carries an absolute vault path
|
|
144
|
+
* (`ENOENT ... /Users/x/hq/companies/acme/deal.md`), and that path would then
|
|
145
|
+
* travel into server-side audit findings and logs. A class name carries the
|
|
146
|
+
* shape of the failure with none of the tenant data.
|
|
147
|
+
*/
|
|
148
|
+
function errorClassOf(err) {
|
|
149
|
+
const name = err instanceof Error ? err.constructor?.name ?? err.name : typeof err;
|
|
150
|
+
const cleaned = String(name).replace(/[^A-Za-z0-9_]/g, "");
|
|
151
|
+
if (!cleaned || !/^[A-Za-z]/.test(cleaned))
|
|
152
|
+
return "UnknownError";
|
|
153
|
+
return cleaned.slice(0, 64);
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Read this scope's ledger rows out of the v3 journal.
|
|
157
|
+
*
|
|
158
|
+
* `listJournals()` is the only supported enumeration surface (a caller that
|
|
159
|
+
* reconstructs a path sees one shard at best, and a mangled non-slug at
|
|
160
|
+
* worst). A journal that cannot be read is NOT fatal: the audit still wants
|
|
161
|
+
* the disk picture, and `ledgerUnavailable` tells the server to read every
|
|
162
|
+
* `disk` entry as "ledger unknown" rather than "ledger missing" — otherwise a
|
|
163
|
+
* momentarily locked journal manufactures a findings storm.
|
|
164
|
+
*/
|
|
165
|
+
function readLedger(journalSlug, listJournalsImpl) {
|
|
166
|
+
try {
|
|
167
|
+
// ONE shard, read straight into a compact index — never the machine's
|
|
168
|
+
// whole journal set, and never a full `SyncJournal` kept alive beside the
|
|
169
|
+
// map derived from it.
|
|
170
|
+
//
|
|
171
|
+
// `listJournals()` read and parsed every slug on disk, and `readJournal()`
|
|
172
|
+
// deep-cloned each result into the baseline WeakMap, so this line used to
|
|
173
|
+
// cost roughly twice the total size of every journal on the machine and
|
|
174
|
+
// paid it again for every scope in a fanout, to keep exactly one of them.
|
|
175
|
+
// `readJournalLedgerIndex` reads the one shard, streams it when it is a
|
|
176
|
+
// legacy JSON document, and retains only the six fields a reporting caller
|
|
177
|
+
// reads — so the parse it came from is collectable before the walk starts.
|
|
178
|
+
//
|
|
179
|
+
// `listJournalsImpl` stays honored for the tests that pin ledger behavior
|
|
180
|
+
// through a fabricated summary list; it takes the same compact projection,
|
|
181
|
+
// so the seam cannot drift from the real read.
|
|
182
|
+
if (listJournalsImpl) {
|
|
183
|
+
const summary = listJournalsImpl().find((candidate) => candidate.slug === journalSlug);
|
|
184
|
+
// A scope with no shard yet (first sync) is NOT "unavailable" — it is an
|
|
185
|
+
// empty, perfectly readable ledger. Conflating the two would suppress the
|
|
186
|
+
// very finding this feature exists to surface: files on disk that the
|
|
187
|
+
// journal never tracked.
|
|
188
|
+
return { entries: summary ? toLedgerIndex(summary.journal) : new Map() };
|
|
189
|
+
}
|
|
190
|
+
// `readJournalLedgerIndexResult`, NOT `readJournalLedgerIndex`: the latter
|
|
191
|
+
// answers `null` for BOTH "this scope has no ledger yet" and "the ledger
|
|
192
|
+
// exists and cannot be read", and coalescing that `null` to an empty Map
|
|
193
|
+
// is how a machine whose v3 store was corrupt in every generation reported
|
|
194
|
+
// all 188,997 of its files as `source: "disk"` — on-disk-never-tracked —
|
|
195
|
+
// and uploaded that as a complete, authoritative FULL manifest.
|
|
196
|
+
//
|
|
197
|
+
// An unreadable ledger is not an empty one. It takes the `ledgerUnavailable`
|
|
198
|
+
// path, where the builder asserts nothing about ledger agreement and the
|
|
199
|
+
// upload pass refuses to publish an implausible picture.
|
|
200
|
+
const result = readJournalLedgerIndexResult(journalSlug);
|
|
201
|
+
if (result.status === "unreadable") {
|
|
202
|
+
return { entries: new Map(), unavailableErrorClass: result.errorClass ?? "UnknownError" };
|
|
203
|
+
}
|
|
204
|
+
return { entries: result.index };
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
return { entries: new Map(), unavailableErrorClass: errorClassOf(err) };
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// ─── Ignore rules ────────────────────────────────────────────────────────────
|
|
211
|
+
function readIgnoreFileLines(filePath) {
|
|
212
|
+
let text;
|
|
213
|
+
try {
|
|
214
|
+
text = fs.readFileSync(filePath, "utf8");
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
return [];
|
|
218
|
+
}
|
|
219
|
+
return text
|
|
220
|
+
.split(/\r?\n/)
|
|
221
|
+
.map((line) => line.trim())
|
|
222
|
+
.filter((line) => line.length > 0 && !line.startsWith("#"));
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Compile the ignore-rule list the walk ACTUALLY applied, in precedence order.
|
|
226
|
+
*
|
|
227
|
+
* The server needs this to tell "the client never looked at that file" apart
|
|
228
|
+
* from "the client looked and the file is gone" — without it, a user's
|
|
229
|
+
* `.hqignore` line makes every file under it look like drift. The layers
|
|
230
|
+
* mirror `createIgnoreFilterFromSnapshot` exactly: defaults, the layout-aware
|
|
231
|
+
* root `core.yaml` exclusion, `.hqignore` (or its legacy `.hqsyncignore`
|
|
232
|
+
* spelling), then `.hqinclude` allowlist entries — tagged `include:` because
|
|
233
|
+
* their verdict is INVERTED and an untagged list would read as an exclusion.
|
|
234
|
+
* `.gitignore` is deliberately absent: sync has not consulted it since the
|
|
235
|
+
* 2026-08 personal-vault carve-out incident.
|
|
236
|
+
*/
|
|
237
|
+
function compileIgnoreRules(hqRoot, scopeKind) {
|
|
238
|
+
const rules = [...DEFAULT_IGNORES];
|
|
239
|
+
if (fs.existsSync(path.join(hqRoot, "core", "core.yaml")))
|
|
240
|
+
rules.push("/core.yaml");
|
|
241
|
+
const hqignore = readIgnoreFileLines(path.join(hqRoot, ".hqignore"));
|
|
242
|
+
rules.push(...(hqignore.length > 0 ? hqignore : readIgnoreFileLines(path.join(hqRoot, ".hqsyncignore"))));
|
|
243
|
+
for (const line of readIgnoreFileLines(path.join(hqRoot, ".hqinclude"))) {
|
|
244
|
+
rules.push(`include:${line}`);
|
|
245
|
+
}
|
|
246
|
+
// Never-pulled, push-only prefix; the walk skips it, so it belongs in the
|
|
247
|
+
// rule list the server reads.
|
|
248
|
+
rules.push(SESSIONS_SCOPE_PREFIX);
|
|
249
|
+
if (scopeKind === "personal") {
|
|
250
|
+
for (const exclusion of PERSONAL_VAULT_DEFAULT_EXCLUSIONS) {
|
|
251
|
+
for (const pattern of exclusion.pattern.split(",")) {
|
|
252
|
+
const trimmed = pattern.trim();
|
|
253
|
+
if (trimmed)
|
|
254
|
+
rules.push(trimmed);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const bounded = [];
|
|
259
|
+
for (const rule of rules) {
|
|
260
|
+
if (bounded.length >= SYNC_MANIFEST_MAX_IGNORE_RULES)
|
|
261
|
+
break;
|
|
262
|
+
if (rule.length === 0 || rule.length > SYNC_MANIFEST_MAX_IGNORE_RULE_LENGTH)
|
|
263
|
+
continue;
|
|
264
|
+
bounded.push(rule);
|
|
265
|
+
}
|
|
266
|
+
return bounded;
|
|
267
|
+
}
|
|
268
|
+
function yieldToEventLoop() {
|
|
269
|
+
// `setTimeout(…, 0)` rather than `setImmediate` so the same code runs
|
|
270
|
+
// unchanged under any bundler/runtime that only guarantees timers.
|
|
271
|
+
return new Promise((resolve) => setTimeout(resolve, 0));
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* Stat one file the walk reached and fold it into the outcome.
|
|
275
|
+
*
|
|
276
|
+
* Extracted from the dirent loop so the walk's ROOTS can be folded in the same
|
|
277
|
+
* way: a personal-vault walk root may itself be a FILE (`companies/manifest.yaml`,
|
|
278
|
+
* `workspace/threads/handoff.json`), and re-implementing the collision, NFC and
|
|
279
|
+
* `sessions/` rules for that case is exactly how the two paths would drift.
|
|
280
|
+
*
|
|
281
|
+
* `lstat` (never `stat`) so a symlink is observed as itself rather than as its
|
|
282
|
+
* target: following it would both mis-report the entry's size/mtime and let a
|
|
283
|
+
* link out of the scope smuggle foreign content into a tenant's manifest.
|
|
284
|
+
*/
|
|
285
|
+
function recordWalkedFile(outcome, collidedKeys, absPath, relative) {
|
|
286
|
+
const manifestPath = toManifestPath(relative);
|
|
287
|
+
if (manifestPath === null) {
|
|
288
|
+
outcome.unrepresentablePaths += 1;
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
// `sessions/` is push-only by contract and is excluded from every pull
|
|
292
|
+
// scope (`resolvePullScope().excludePrefixes`). Enumerating it here would
|
|
293
|
+
// report every teammate's un-pulled transcripts as missing locally.
|
|
294
|
+
if (manifestPath.startsWith(SESSIONS_SCOPE_PREFIX)) {
|
|
295
|
+
outcome.ignoredFiles += 1;
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
if (outcome.records.has(manifestPath)) {
|
|
299
|
+
// Distinct on-disk names, one wire key. Neither can be reported
|
|
300
|
+
// truthfully, so both are dropped and counted as unrepresentable —
|
|
301
|
+
// the same treatment as any other path the wire cannot express.
|
|
302
|
+
//
|
|
303
|
+
// Checked BEFORE `lstat`, deliberately. The ambiguity is a property of the
|
|
304
|
+
// directory LISTING, not of whether the second spelling happens to resolve:
|
|
305
|
+
// on a normalization-insensitive filesystem (macOS) both names stat to the
|
|
306
|
+
// same file, while on a byte-exact one (Linux) the NFD spelling of a file
|
|
307
|
+
// stored as NFC does not resolve at all. Statting first would make the
|
|
308
|
+
// collision invisible on exactly the platform where the two names really
|
|
309
|
+
// are two different entries, and the manifest would then publish one
|
|
310
|
+
// file's content under the other's name.
|
|
311
|
+
collidedKeys.set(manifestPath, (collidedKeys.get(manifestPath) ?? 0) + 1);
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
let stats;
|
|
315
|
+
try {
|
|
316
|
+
stats = fs.lstatSync(absPath);
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
return; // Raced away between readdir and lstat.
|
|
320
|
+
}
|
|
321
|
+
if (!stats.isFile() && !stats.isSymbolicLink())
|
|
322
|
+
return;
|
|
323
|
+
outcome.filesWalked += 1;
|
|
324
|
+
const record = {
|
|
325
|
+
size: stats.size,
|
|
326
|
+
mtimeMs: Math.floor(stats.mtimeMs),
|
|
327
|
+
isSymlink: stats.isSymbolicLink(),
|
|
328
|
+
};
|
|
329
|
+
// Only pay for the second string when normalization actually changed the
|
|
330
|
+
// path; `relative === manifestPath` for essentially every ASCII name.
|
|
331
|
+
if (relative !== manifestPath)
|
|
332
|
+
record.raw = relative;
|
|
333
|
+
outcome.records.set(manifestPath, record);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Stat-only, cooperative walk of one scope's WALK ROOTS.
|
|
337
|
+
*
|
|
338
|
+
* `scopeRoot` anchors the relative paths that go on the wire; `walkRoots` are
|
|
339
|
+
* the subtrees actually enumerated. For a company scope the two coincide (one
|
|
340
|
+
* root, the company directory). For the PERSONAL scope they emphatically do
|
|
341
|
+
* not: the personal push leg walks `computePersonalVaultPaths()`, which
|
|
342
|
+
* excludes `companies/` (bar the local, non-cloud carve-outs) along with
|
|
343
|
+
* `repos/`, `workspace/` and `.git`. Walking the whole HQ root here instead
|
|
344
|
+
* would put every cloud company's file under the PERSONAL manifest — a
|
|
345
|
+
* tenant-boundary report the personal vault does not and must not hold.
|
|
346
|
+
*
|
|
347
|
+
* A walk root that is a FILE rather than a directory is recorded directly:
|
|
348
|
+
* `computePersonalVaultPaths()` returns single-file carve-outs
|
|
349
|
+
* (`companies/manifest.yaml`, the continuity pointer) alongside directories.
|
|
350
|
+
*/
|
|
351
|
+
async function walkScope(scopeRoot, walkRoots, shouldSync, options) {
|
|
352
|
+
const records = new Map();
|
|
353
|
+
const outcome = {
|
|
354
|
+
records,
|
|
355
|
+
filesWalked: 0,
|
|
356
|
+
ignoredFiles: 0,
|
|
357
|
+
unrepresentablePaths: 0,
|
|
358
|
+
truncated: false,
|
|
359
|
+
lostVisibilityPrefixes: [],
|
|
360
|
+
};
|
|
361
|
+
// Two raw spellings that collapse onto one NFC key are indistinguishable on
|
|
362
|
+
// the wire, and keeping either would silently publish one file's hash under
|
|
363
|
+
// the other's name (last write wins). Both are dropped after the walk.
|
|
364
|
+
// key → how many EXTRA raw spellings landed on it (the first one is in the
|
|
365
|
+
// map already and is removed too).
|
|
366
|
+
const collidedKeys = new Map();
|
|
367
|
+
// The stack carries each directory's SCOPE-RELATIVE prefix alongside its
|
|
368
|
+
// absolute path. Deriving the relative path this way (prefix + "/" + name)
|
|
369
|
+
// rather than calling `path.relative` per file removes a normalize + split +
|
|
370
|
+
// join + array allocation from the hottest loop in the product — worth
|
|
371
|
+
// roughly a third of the walk's resident growth on a 70k-file root.
|
|
372
|
+
const stack = [];
|
|
373
|
+
// Deduped: `computePersonalVaultPaths()` composes several carve-out lists and
|
|
374
|
+
// is not required to return a disjoint set. Walking one root twice would not
|
|
375
|
+
// corrupt the manifest (the collision map would drop the second spelling),
|
|
376
|
+
// but it would count every file underneath as unrepresentable and delete it
|
|
377
|
+
// from the report — a far worse outcome than a `Set`.
|
|
378
|
+
const seenRoots = new Set();
|
|
379
|
+
for (const walkRoot of walkRoots) {
|
|
380
|
+
const resolved = path.resolve(walkRoot);
|
|
381
|
+
if (seenRoots.has(resolved))
|
|
382
|
+
continue;
|
|
383
|
+
seenRoots.add(resolved);
|
|
384
|
+
const relRoot = resolved === path.resolve(scopeRoot) ? "" : path.relative(scopeRoot, resolved);
|
|
385
|
+
// A root outside the scope anchor cannot be given a scope-relative path, so
|
|
386
|
+
// it cannot be reported truthfully. Dropping it is the honest answer.
|
|
387
|
+
if (relRoot.startsWith("..") || path.isAbsolute(relRoot))
|
|
388
|
+
continue;
|
|
389
|
+
const relDir = relRoot.split(path.sep).join("/");
|
|
390
|
+
// The root itself is subject to the ignore filter, exactly like any dirent.
|
|
391
|
+
let rootStats;
|
|
392
|
+
try {
|
|
393
|
+
rootStats = fs.lstatSync(resolved);
|
|
394
|
+
}
|
|
395
|
+
catch {
|
|
396
|
+
// A named root that is not there is a benign race (or a carve-out for a
|
|
397
|
+
// file this machine does not have). Not a lost-visibility prefix: we know
|
|
398
|
+
// it is absent, we did not fail to look.
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
const rootIsDir = rootStats.isDirectory();
|
|
402
|
+
if (relDir !== "" && !shouldSync(resolved, rootIsDir)) {
|
|
403
|
+
if (!rootIsDir)
|
|
404
|
+
outcome.ignoredFiles += 1;
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
if (rootIsDir) {
|
|
408
|
+
stack.push({ absDir: resolved, relDir });
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
if (relDir === "")
|
|
412
|
+
continue; // The scope anchor itself is not a file entry.
|
|
413
|
+
recordWalkedFile(outcome, collidedKeys, resolved, relDir);
|
|
414
|
+
}
|
|
415
|
+
let sinceYield = 0;
|
|
416
|
+
while (stack.length > 0) {
|
|
417
|
+
if (options.abortSignal?.aborted || Date.now() > options.deadlineMs) {
|
|
418
|
+
outcome.truncated = true;
|
|
419
|
+
break;
|
|
420
|
+
}
|
|
421
|
+
const { absDir, relDir } = stack.pop();
|
|
422
|
+
let dirents;
|
|
423
|
+
try {
|
|
424
|
+
dirents = fs.readdirSync(absDir, { withFileTypes: true });
|
|
425
|
+
}
|
|
426
|
+
catch {
|
|
427
|
+
// An unreadable directory (permissions, raced deletion) is a gap in the
|
|
428
|
+
// picture, not a reason to abandon the rest of the tree — so the walk
|
|
429
|
+
// continues. But the gap MUST be recorded: silently skipping it makes
|
|
430
|
+
// every file underneath look like it vanished locally, which then
|
|
431
|
+
// surfaces as a false `source: "ledger"` entry or, worse, a false
|
|
432
|
+
// removal that the server reads as a deletion.
|
|
433
|
+
//
|
|
434
|
+
// We record the PREFIX rather than marking the whole build `truncated`
|
|
435
|
+
// (the other available answer) because it is nearly free here and far
|
|
436
|
+
// more useful: `truncated` would suppress removals, ledger synthesis and
|
|
437
|
+
// the snapshot write for the ENTIRE scope, so one unreadable directory
|
|
438
|
+
// would permanently disable delta mode for the whole root. Scoping the
|
|
439
|
+
// suppression to the affected subtree keeps the rest of the manifest
|
|
440
|
+
// fully usable.
|
|
441
|
+
outcome.lostVisibilityPrefixes.push(relDir === "" ? "" : `${relDir}/`);
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
for (const dirent of dirents) {
|
|
445
|
+
const absPath = path.join(absDir, dirent.name);
|
|
446
|
+
const isDir = dirent.isDirectory();
|
|
447
|
+
if (!shouldSync(absPath, isDir)) {
|
|
448
|
+
if (!isDir)
|
|
449
|
+
outcome.ignoredFiles += 1;
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
const relative = relDir === "" ? dirent.name : `${relDir}/${dirent.name}`;
|
|
453
|
+
if (isDir) {
|
|
454
|
+
stack.push({ absDir: absPath, relDir: relative });
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
recordWalkedFile(outcome, collidedKeys, absPath, relative);
|
|
458
|
+
if (++sinceYield >= options.yieldEvery) {
|
|
459
|
+
sinceYield = 0;
|
|
460
|
+
await yieldToEventLoop();
|
|
461
|
+
if (options.abortSignal?.aborted || Date.now() > options.deadlineMs) {
|
|
462
|
+
outcome.truncated = true;
|
|
463
|
+
return finishWalk(outcome, collidedKeys);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
return finishWalk(outcome, collidedKeys);
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Drop every NFC key that two distinct raw filenames landed on. Done once at
|
|
472
|
+
* the end rather than inline, because the FIRST of a colliding pair is already
|
|
473
|
+
* in the map by the time the second is seen.
|
|
474
|
+
*/
|
|
475
|
+
function finishWalk(outcome, collidedKeys) {
|
|
476
|
+
for (const [key, extras] of collidedKeys) {
|
|
477
|
+
if (outcome.records.delete(key))
|
|
478
|
+
outcome.filesWalked -= 1;
|
|
479
|
+
// Every raw spelling that collided is unrepresentable: the one that made it
|
|
480
|
+
// into the map, plus each duplicate that was turned away at the door.
|
|
481
|
+
outcome.unrepresentablePaths += extras + 1;
|
|
482
|
+
}
|
|
483
|
+
return outcome;
|
|
484
|
+
}
|
|
485
|
+
// ─── Entry assembly ──────────────────────────────────────────────────────────
|
|
486
|
+
/**
|
|
487
|
+
* True when the ledger's hash can stand in for reading the file.
|
|
488
|
+
*
|
|
489
|
+
* Both halves of the stat pair must agree. `size` alone misses a same-length
|
|
490
|
+
* rewrite; `mtimeMs` alone misses a `cp -p` / `touch -r` restore. The journal
|
|
491
|
+
* also records `ctimeMs` for its own push-skip gate, but the manifest is a
|
|
492
|
+
* REPORTING surface, not a transfer decision: an over-eager re-hash here costs
|
|
493
|
+
* one file read, while a wrong skip on the push path costs a lost upload — so
|
|
494
|
+
* this guard deliberately mirrors the contract's two reported fields rather
|
|
495
|
+
* than inventing a third.
|
|
496
|
+
*/
|
|
497
|
+
function canReuseLedgerHash(entry, record) {
|
|
498
|
+
if (!entry || typeof entry.hash !== "string" || entry.hash.length === 0)
|
|
499
|
+
return false;
|
|
500
|
+
if (entry.size !== record.size)
|
|
501
|
+
return false;
|
|
502
|
+
if (typeof entry.mtimeMs !== "number")
|
|
503
|
+
return false;
|
|
504
|
+
return Math.floor(entry.mtimeMs) === record.mtimeMs;
|
|
505
|
+
}
|
|
506
|
+
function withinContractBounds(size, mtimeMs) {
|
|
507
|
+
return (Number.isSafeInteger(size) &&
|
|
508
|
+
size >= 0 &&
|
|
509
|
+
size <= SYNC_MANIFEST_MAX_FILE_SIZE_BYTES &&
|
|
510
|
+
Number.isSafeInteger(mtimeMs) &&
|
|
511
|
+
mtimeMs >= 0 &&
|
|
512
|
+
mtimeMs <= SYNC_MANIFEST_MAX_MTIME_MS);
|
|
513
|
+
}
|
|
514
|
+
function decorateWithLedger(entry, ledgerEntry) {
|
|
515
|
+
if (typeof ledgerEntry.remoteEtag === "string" && ledgerEntry.remoteEtag.length > 0) {
|
|
516
|
+
entry.ledgerRemoteEtag = ledgerEntry.remoteEtag;
|
|
517
|
+
}
|
|
518
|
+
if (typeof ledgerEntry.syncedAt === "string" && ledgerEntry.syncedAt.length > 0) {
|
|
519
|
+
entry.ledgerSyncedAt = ledgerEntry.syncedAt;
|
|
520
|
+
}
|
|
521
|
+
if (ledgerEntry.direction === "up" || ledgerEntry.direction === "down") {
|
|
522
|
+
entry.ledgerDirection = ledgerEntry.direction;
|
|
523
|
+
}
|
|
524
|
+
return entry;
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Resolve where a scope lives on disk and which journal shard describes it.
|
|
528
|
+
*
|
|
529
|
+
* The personal vault's sync root IS the HQ root and its shard is the reserved
|
|
530
|
+
* {@link PERSONAL_VAULT_JOURNAL_SLUG} sentinel — never the literal `"personal"`
|
|
531
|
+
* slug, which belongs to the `companies/personal` COMPANY and whose collision
|
|
532
|
+
* once churned ~190 files per sync.
|
|
533
|
+
*
|
|
534
|
+
* The returned `wireScope` is rebuilt field by field rather than spread from
|
|
535
|
+
* the caller's object: that is what guarantees a personal manifest can never
|
|
536
|
+
* carry a `companyUid` (or a stray local `slug`) onto the wire.
|
|
537
|
+
*/
|
|
538
|
+
function resolveScope(scope, hqRoot, teamSyncedSlugs) {
|
|
539
|
+
if (scope.kind === "company") {
|
|
540
|
+
if (!scope.companyUid)
|
|
541
|
+
throw new Error("buildSyncManifest: company scope requires companyUid");
|
|
542
|
+
if (!scope.slug)
|
|
543
|
+
throw new Error("buildSyncManifest: company scope requires slug");
|
|
544
|
+
// The slug names a DIRECTORY, and `path.join` will happily resolve
|
|
545
|
+
// `../personal` out of `companies/` entirely — which would walk one tenant's
|
|
546
|
+
// tree and upload it under another's `companyUid`. That is the exact
|
|
547
|
+
// boundary this whole feature is supposed to protect, so the slug is held
|
|
548
|
+
// to the same direct-child rule the reconciler already enforces (shared,
|
|
549
|
+
// not re-derived, so the two can never drift). Throwing rather than
|
|
550
|
+
// degrading is deliberate: only a caller bug can produce such a slug, and
|
|
551
|
+
// it belongs in the same class as "company scope without a uid".
|
|
552
|
+
if (!isSafeCompanySlug(path.join(hqRoot, "companies"), scope.slug)) {
|
|
553
|
+
throw new Error("buildSyncManifest: company scope slug must name a direct child of companies/");
|
|
554
|
+
}
|
|
555
|
+
const companyRoot = path.join(hqRoot, "companies", scope.slug);
|
|
556
|
+
return {
|
|
557
|
+
root: companyRoot,
|
|
558
|
+
walkRoots: [companyRoot],
|
|
559
|
+
journalSlug: scope.slug,
|
|
560
|
+
wireScope: { kind: "company", companyUid: scope.companyUid },
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
if (scope.companyUid) {
|
|
564
|
+
throw new Error("buildSyncManifest: personal scope must not carry a companyUid");
|
|
565
|
+
}
|
|
566
|
+
// NOT `[hqRoot]`. The personal manifest must describe exactly what the
|
|
567
|
+
// personal PUSH leg ships (see sync-runner-company.ts, which starts share()
|
|
568
|
+
// at these same paths), and that carve-out excludes `companies/` for every
|
|
569
|
+
// cloud-backed slug. Reporting the whole HQ root here would tell the server
|
|
570
|
+
// the personal vault holds every tenant's files — findings for objects the
|
|
571
|
+
// personal bucket was never meant to contain, on every single pass.
|
|
572
|
+
return {
|
|
573
|
+
root: hqRoot,
|
|
574
|
+
walkRoots: computePersonalVaultPaths(hqRoot, teamSyncedSlugs ? { teamSyncedSlugs } : {}),
|
|
575
|
+
journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
|
|
576
|
+
wireScope: { kind: "personal" },
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
// ─── Builder ─────────────────────────────────────────────────────────────────
|
|
580
|
+
/**
|
|
581
|
+
* Build the manifest chunks for one scope.
|
|
582
|
+
*
|
|
583
|
+
* Never throws for environmental reasons (unreadable journal, unreadable
|
|
584
|
+
* directory, undecodable filename, snapshot write failure) — those degrade to
|
|
585
|
+
* a narrower but honest manifest. It DOES throw for programming errors:
|
|
586
|
+
* a company scope without a uid or slug, or a personal scope carrying a
|
|
587
|
+
* company uid, are caller bugs that must fail loudly rather than upload a
|
|
588
|
+
* mislabelled tenant's file list.
|
|
589
|
+
*/
|
|
590
|
+
export async function buildSyncManifest(options) {
|
|
591
|
+
const nowMs = options.now ?? Date.now();
|
|
592
|
+
const maxWallMs = options.maxWallMs ?? DEFAULT_MANIFEST_MAX_WALL_MS;
|
|
593
|
+
const yieldEvery = Math.max(1, options.yieldEvery ?? DEFAULT_MANIFEST_YIELD_EVERY);
|
|
594
|
+
// `never`, NOT `auto`. The daemon calls this after every sync cycle, and the
|
|
595
|
+
// old default read every file the journal could not cover — which on a vault
|
|
596
|
+
// whose journal predates hashing is every file. Callers that genuinely want
|
|
597
|
+
// content hashes ask for them.
|
|
598
|
+
const hashPolicy = options.hashPolicy ?? "never";
|
|
599
|
+
const maxHashedFiles = Math.max(0, options.maxHashedFiles ?? DEFAULT_MANIFEST_MAX_HASHED_FILES);
|
|
600
|
+
const maxHashMs = Math.max(0, options.maxHashMs ?? DEFAULT_MANIFEST_MAX_HASH_MS);
|
|
601
|
+
const { root, walkRoots, journalSlug, wireScope } = resolveScope(options.scope, options.hqRoot, options.teamSyncedSlugs);
|
|
602
|
+
const ledger = readLedger(journalSlug, options.listJournalsImpl);
|
|
603
|
+
const ignoreRules = compileIgnoreRules(options.hqRoot, wireScope.kind);
|
|
604
|
+
// The ignore filter is always anchored at the HQ ROOT, even for a company
|
|
605
|
+
// scope: its rules are expressed hq-root-relative, and every company root
|
|
606
|
+
// lives underneath. Re-anchoring it at the company dir would silently change
|
|
607
|
+
// which patterns match.
|
|
608
|
+
const baseFilter = createIgnoreFilter(options.hqRoot, {
|
|
609
|
+
memoLimit: options.ignoreMemoLimit ?? MANIFEST_IGNORE_MEMO_LIMIT,
|
|
610
|
+
});
|
|
611
|
+
const shouldSync = wireScope.kind === "personal"
|
|
612
|
+
? wrapFilterWithPersonalVaultDefaults(baseFilter, options.hqRoot, () => {
|
|
613
|
+
// The out-of-policy telemetry event belongs to the sync runner; the
|
|
614
|
+
// manifest only needs the verdict, and the walker counts it as
|
|
615
|
+
// ignored like any other filtered file.
|
|
616
|
+
})
|
|
617
|
+
: baseFilter;
|
|
618
|
+
const walkStartedAt = Date.now();
|
|
619
|
+
// ONE deadline for the whole build, not one per phase: the caller asked for
|
|
620
|
+
// "spend at most `maxWallMs` on this scope", and the hash phase below is as
|
|
621
|
+
// expensive as the walk (it opens files), so a per-phase cap would let the
|
|
622
|
+
// build run for twice the budget.
|
|
623
|
+
const deadlineMs = walkStartedAt + maxWallMs;
|
|
624
|
+
const walk = await walkScope(root, walkRoots, shouldSync, {
|
|
625
|
+
deadlineMs,
|
|
626
|
+
yieldEvery,
|
|
627
|
+
abortSignal: options.abortSignal,
|
|
628
|
+
});
|
|
629
|
+
const walkMs = Date.now() - walkStartedAt;
|
|
630
|
+
const entries = [];
|
|
631
|
+
// `Object.create(null)`, not `{}`: `__proto__` is a perfectly legal filename
|
|
632
|
+
// (and a legal contract path), and on a plain object literal that key would
|
|
633
|
+
// silently fail to become an own property — the file would vanish from the
|
|
634
|
+
// snapshot and then read as REMOVED on the next delta.
|
|
635
|
+
const snapshotEntries = Object.create(null);
|
|
636
|
+
let hashedFiles = 0;
|
|
637
|
+
let skippedUnhashed = 0;
|
|
638
|
+
let unhashedFiles = 0;
|
|
639
|
+
let droppedStatRaces = 0;
|
|
640
|
+
// Wall clock spent READING files, tracked separately from the walk's own
|
|
641
|
+
// deadline so the `auto` cap bounds the expensive half on its own terms.
|
|
642
|
+
const hashPhaseStartedAt = Date.now();
|
|
643
|
+
// Paths the walk SAW on disk that did not make it into `entries` — an
|
|
644
|
+
// unreadable file, or a post-hash re-stat that caught a writer mid-flight.
|
|
645
|
+
// They are absent from this manifest but emphatically NOT deleted, and the
|
|
646
|
+
// delta below must not report them as removed (see `removed`).
|
|
647
|
+
const seenButDropped = new Set();
|
|
648
|
+
// The hash phase obeys the same cooperative contract as the walk. Without
|
|
649
|
+
// this, a first-ever sync — where the ledger has no hashes and every file
|
|
650
|
+
// must be READ — blocks the event loop for the entire run, ignores
|
|
651
|
+
// `abortSignal`, and overruns `maxWallMs` by an unbounded margin.
|
|
652
|
+
let sinceYield = 0;
|
|
653
|
+
for (const [manifestPath, record] of walk.records) {
|
|
654
|
+
if (++sinceYield >= yieldEvery) {
|
|
655
|
+
sinceYield = 0;
|
|
656
|
+
await yieldToEventLoop();
|
|
657
|
+
}
|
|
658
|
+
if (options.abortSignal?.aborted || Date.now() > deadlineMs) {
|
|
659
|
+
// Same honesty rule as the walk: a partial hash phase is a partial
|
|
660
|
+
// picture, so mark it truncated. That suppresses `removedPaths`, the
|
|
661
|
+
// delta base, and the snapshot write further down.
|
|
662
|
+
walk.truncated = true;
|
|
663
|
+
break;
|
|
664
|
+
}
|
|
665
|
+
const ledgerEntry = ledger.entries.get(manifestPath);
|
|
666
|
+
// Filesystem IO uses the RAW spelling (see DiskRecord.raw): the NFC form is
|
|
667
|
+
// the wire key, and on a normalization-sensitive filesystem it may not name
|
|
668
|
+
// any file at all.
|
|
669
|
+
const absPath = path.join(root, record.raw ?? manifestPath);
|
|
670
|
+
let hash;
|
|
671
|
+
// May this pass afford to READ this file? `full` always; `auto` until it
|
|
672
|
+
// has spent its per-pass budget; `never` / `ledger-only` never.
|
|
673
|
+
const mayHash = hashPolicy === "full" ||
|
|
674
|
+
(hashPolicy === "auto" &&
|
|
675
|
+
hashedFiles < maxHashedFiles &&
|
|
676
|
+
Date.now() - hashPhaseStartedAt < maxHashMs);
|
|
677
|
+
if (canReuseLedgerHash(ledgerEntry, record)) {
|
|
678
|
+
hash = ledgerEntry?.hash;
|
|
679
|
+
}
|
|
680
|
+
else if (mayHash) {
|
|
681
|
+
try {
|
|
682
|
+
hash = record.isSymlink
|
|
683
|
+
? hashSymlinkTarget(fs.readlinkSync(absPath))
|
|
684
|
+
: hashFile(absPath);
|
|
685
|
+
hashedFiles += 1;
|
|
686
|
+
}
|
|
687
|
+
catch {
|
|
688
|
+
// Unreadable or raced away; better absent than wrong.
|
|
689
|
+
seenButDropped.add(manifestPath);
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
// Re-stat AFTER hashing. A file written while we read it yields a hash
|
|
693
|
+
// that matches neither the before nor the after content — publishing it
|
|
694
|
+
// would report a phantom divergence against the vault. Drop the entry
|
|
695
|
+
// and let the next cycle, when the writer has finished, report it.
|
|
696
|
+
try {
|
|
697
|
+
const after = fs.lstatSync(absPath);
|
|
698
|
+
if (after.size !== record.size || Math.floor(after.mtimeMs) !== record.mtimeMs) {
|
|
699
|
+
droppedStatRaces += 1;
|
|
700
|
+
seenButDropped.add(manifestPath);
|
|
701
|
+
continue;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
catch {
|
|
705
|
+
droppedStatRaces += 1;
|
|
706
|
+
seenButDropped.add(manifestPath);
|
|
707
|
+
continue;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
else if (hashPolicy === "ledger-only") {
|
|
711
|
+
// The benchmark seam, and the ONLY policy that still drops. It is not
|
|
712
|
+
// reachable from any production caller.
|
|
713
|
+
skippedUnhashed += 1;
|
|
714
|
+
seenButDropped.add(manifestPath);
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
else {
|
|
718
|
+
// Stat-only: the entry ships with no hash at all. This is the important
|
|
719
|
+
// difference from dropping it — the server learns the path is PRESENT
|
|
720
|
+
// (with its size and mtime), it just cannot verify the content this
|
|
721
|
+
// pass. A dropped entry would read as a file the client does not hold,
|
|
722
|
+
// which is precisely the false finding this audit exists to avoid.
|
|
723
|
+
unhashedFiles += 1;
|
|
724
|
+
}
|
|
725
|
+
if (!withinContractBounds(record.size, record.mtimeMs)) {
|
|
726
|
+
skippedUnhashed += 1;
|
|
727
|
+
seenButDropped.add(manifestPath);
|
|
728
|
+
continue;
|
|
729
|
+
}
|
|
730
|
+
const entry = {
|
|
731
|
+
path: manifestPath,
|
|
732
|
+
...(hash ? { hash } : {}),
|
|
733
|
+
size: record.size,
|
|
734
|
+
mtimeMs: record.mtimeMs,
|
|
735
|
+
// `ledgerUnavailable` means we cannot claim the ledger agrees, so every
|
|
736
|
+
// entry reports as disk-only regardless of what a stale map might say.
|
|
737
|
+
source: ledgerEntry && !ledger.unavailableErrorClass ? "both" : "disk",
|
|
738
|
+
};
|
|
739
|
+
if (ledgerEntry && !ledger.unavailableErrorClass) {
|
|
740
|
+
decorateWithLedger(entry, ledgerEntry);
|
|
741
|
+
// Only claimable when we HAVE a hash. With none, "the ledger's hash
|
|
742
|
+
// differs from ours" is not a statement we can make — and asserting
|
|
743
|
+
// `pendingPush` off an absent hash would flag every file on a stat-only
|
|
744
|
+
// pass as owing an upload.
|
|
745
|
+
if (hash !== undefined && ledgerEntry.hash !== hash)
|
|
746
|
+
entry.pendingPush = true;
|
|
747
|
+
}
|
|
748
|
+
entries.push(entry);
|
|
749
|
+
// The delta snapshot needs a hash to compare against. An unhashed entry has
|
|
750
|
+
// nothing to store, and inventing a placeholder would make the next delta
|
|
751
|
+
// report a change that did not happen — so it is simply left out, and the
|
|
752
|
+
// next pass treats it as new (an ADD, never a removal).
|
|
753
|
+
if (hash !== undefined) {
|
|
754
|
+
snapshotEntries[manifestPath] = { hash, size: record.size, mtimeMs: record.mtimeMs };
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
/**
|
|
758
|
+
* True when a path is one the walk could not have observed even if it exists
|
|
759
|
+
* — so its absence from `walk.records` says nothing about the disk.
|
|
760
|
+
*
|
|
761
|
+
* Two causes, both of which previously read as "the local copy vanished":
|
|
762
|
+
*
|
|
763
|
+
* 1. It sits under a directory the walk could not `readdir`. Invisible, not
|
|
764
|
+
* gone.
|
|
765
|
+
* 2. It now matches an ignore rule. A path that became ignored since it was
|
|
766
|
+
* last synced is simply not our business any more; reporting it as
|
|
767
|
+
* missing turns every new `.hqignore` line into a page of false drift.
|
|
768
|
+
*
|
|
769
|
+
* The ignore filter is consulted DIRECTLY rather than inferred from the
|
|
770
|
+
* walk's observations, because the walk never visits an ignored path and so
|
|
771
|
+
* has nothing to report about it. This is off the hot path: it runs once per
|
|
772
|
+
* ledger row that is already absent from disk, never per walked file.
|
|
773
|
+
*/
|
|
774
|
+
// Scope-relative, forward-slash prefixes of the subtrees this walk actually
|
|
775
|
+
// enumerated. `[""]` for a company scope (one root, the scope anchor).
|
|
776
|
+
const walkRootPrefixes = walkRoots.map((walkRoot) => {
|
|
777
|
+
const rel = path.relative(root, path.resolve(walkRoot));
|
|
778
|
+
if (rel === "")
|
|
779
|
+
return "";
|
|
780
|
+
return rel.split(path.sep).join("/");
|
|
781
|
+
});
|
|
782
|
+
const isOutsideOurView = (candidatePath) => {
|
|
783
|
+
for (const prefix of walk.lostVisibilityPrefixes) {
|
|
784
|
+
if (prefix === "" || candidatePath.startsWith(prefix))
|
|
785
|
+
return true;
|
|
786
|
+
}
|
|
787
|
+
// THIRD cause, introduced with the personal-vault carve-out: the path is
|
|
788
|
+
// not under any walk root at all. A ledger row under a cloud-backed
|
|
789
|
+
// company directory is a row the PERSONAL leg no longer ships (that
|
|
790
|
+
// company was designated and moved to its own bucket) — reporting it as a
|
|
791
|
+
// vanished local file would manufacture one finding per file of every
|
|
792
|
+
// designated company, forever.
|
|
793
|
+
const insideAWalkRoot = walkRootPrefixes.some((prefix) => prefix === "" ||
|
|
794
|
+
candidatePath === prefix ||
|
|
795
|
+
candidatePath.startsWith(`${prefix}/`));
|
|
796
|
+
if (!insideAWalkRoot)
|
|
797
|
+
return true;
|
|
798
|
+
return !shouldSync(path.join(root, candidatePath), false);
|
|
799
|
+
};
|
|
800
|
+
// Ledger rows with no file on disk. These are the point of the exercise: a
|
|
801
|
+
// vanished local file is undetectable from the server alone. Skipped when
|
|
802
|
+
// the walk truncated — "not reached" is not "not there".
|
|
803
|
+
if (!ledger.unavailableErrorClass && !walk.truncated) {
|
|
804
|
+
for (const [ledgerPath, ledgerEntry] of ledger.entries) {
|
|
805
|
+
if (walk.records.has(ledgerPath))
|
|
806
|
+
continue;
|
|
807
|
+
if (ledgerPath.startsWith(SESSIONS_SCOPE_PREFIX))
|
|
808
|
+
continue;
|
|
809
|
+
if (isOutsideOurView(ledgerPath))
|
|
810
|
+
continue;
|
|
811
|
+
if (toManifestPath(ledgerPath) === null) {
|
|
812
|
+
walk.unrepresentablePaths += 1;
|
|
813
|
+
continue;
|
|
814
|
+
}
|
|
815
|
+
const mtimeMs = Math.floor(ledgerEntry.mtimeMs ?? 0);
|
|
816
|
+
if (typeof ledgerEntry.hash !== "string" || !withinContractBounds(ledgerEntry.size, mtimeMs)) {
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
const entry = {
|
|
820
|
+
path: ledgerPath,
|
|
821
|
+
hash: ledgerEntry.hash,
|
|
822
|
+
size: ledgerEntry.size,
|
|
823
|
+
mtimeMs,
|
|
824
|
+
source: "ledger",
|
|
825
|
+
};
|
|
826
|
+
decorateWithLedger(entry, ledgerEntry);
|
|
827
|
+
entries.push(entry);
|
|
828
|
+
// `ledgerEntry.hash`, not `entry.hash`: the entry's is optional now, and
|
|
829
|
+
// the guard above has already proved this one is a string.
|
|
830
|
+
snapshotEntries[ledgerPath] = { hash: ledgerEntry.hash, size: entry.size, mtimeMs };
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
const walkStats = {
|
|
834
|
+
filesWalked: walk.filesWalked,
|
|
835
|
+
walkMs,
|
|
836
|
+
hashedFiles,
|
|
837
|
+
ignoredFiles: walk.ignoredFiles,
|
|
838
|
+
unhashedFiles,
|
|
839
|
+
truncated: walk.truncated,
|
|
840
|
+
unreadableDirs: walk.lostVisibilityPrefixes.length,
|
|
841
|
+
};
|
|
842
|
+
const snapshot = {
|
|
843
|
+
snapshotId: newManifestSnapshotId(),
|
|
844
|
+
generatedAt: new Date(nowMs).toISOString(),
|
|
845
|
+
sequence: options.sequence,
|
|
846
|
+
entries: snapshotEntries,
|
|
847
|
+
};
|
|
848
|
+
const scopeKey = manifestSnapshotScopeKey(wireScope);
|
|
849
|
+
const previous = options.forceFull ? null : readManifestSnapshot(options.stateDir, scopeKey);
|
|
850
|
+
const usableBase = previous &&
|
|
851
|
+
isManifestSnapshotBaseUsable(previous) &&
|
|
852
|
+
!isManifestSnapshotStale(previous, nowMs) &&
|
|
853
|
+
!walk.truncated
|
|
854
|
+
? previous
|
|
855
|
+
: null;
|
|
856
|
+
let mode = "full";
|
|
857
|
+
let wireEntries = entries;
|
|
858
|
+
let removedPaths = [];
|
|
859
|
+
if (usableBase) {
|
|
860
|
+
// Carry a hash FORWARD for any entry this pass could not hash but whose
|
|
861
|
+
// stat pair is unchanged since the base. Without this the base ERODES: a
|
|
862
|
+
// stat-only pass would drop every unhashed path from the snapshot it
|
|
863
|
+
// persists, so the pass after it would find no `before` for them and
|
|
864
|
+
// re-send the lot as changed — a delta that is a full manifest wearing a
|
|
865
|
+
// hat. (`snapshotEntries` is the same object `snapshot.entries` holds, so
|
|
866
|
+
// this lands in the persisted record.)
|
|
867
|
+
for (const entry of entries) {
|
|
868
|
+
if (entry.hash !== undefined)
|
|
869
|
+
continue;
|
|
870
|
+
const before = usableBase.entries[entry.path];
|
|
871
|
+
if (!before)
|
|
872
|
+
continue;
|
|
873
|
+
if (before.size !== entry.size || before.mtimeMs !== entry.mtimeMs)
|
|
874
|
+
continue;
|
|
875
|
+
snapshotEntries[entry.path] = before;
|
|
876
|
+
}
|
|
877
|
+
const changed = entries.filter((entry) => {
|
|
878
|
+
const before = usableBase.entries[entry.path];
|
|
879
|
+
if (!before)
|
|
880
|
+
return true;
|
|
881
|
+
if (before.size !== entry.size || before.mtimeMs !== entry.mtimeMs)
|
|
882
|
+
return true;
|
|
883
|
+
// No hash this pass (stat-only): the stat pair is all we have, and it
|
|
884
|
+
// agrees. Comparing `undefined !== before.hash` here would mark EVERY
|
|
885
|
+
// file changed on every stat-only pass, which turns a delta back into a
|
|
886
|
+
// full re-send and defeats the point of having a base at all.
|
|
887
|
+
if (entry.hash === undefined)
|
|
888
|
+
return false;
|
|
889
|
+
return before.hash !== entry.hash;
|
|
890
|
+
});
|
|
891
|
+
// A loop, not `new Set(entries.map(...))`: the `map` materialises a second
|
|
892
|
+
// array of every path string purely to be consumed once and discarded, and
|
|
893
|
+
// on a 199k-entry scope that transient array lands at the pass's peak.
|
|
894
|
+
const present = new Set();
|
|
895
|
+
for (const entry of entries)
|
|
896
|
+
present.add(entry.path);
|
|
897
|
+
const removed = Object.keys(usableBase.entries).filter((candidate) => {
|
|
898
|
+
if (present.has(candidate))
|
|
899
|
+
return false;
|
|
900
|
+
// Walked, but dropped before it could be reported. It exists.
|
|
901
|
+
if (seenButDropped.has(candidate))
|
|
902
|
+
return false;
|
|
903
|
+
// With the ledger unreadable the `ledger`-source loop above never ran,
|
|
904
|
+
// so every ledger-only path in the base is missing from `entries` for a
|
|
905
|
+
// reason that has nothing to do with deletion. We cannot tell the two
|
|
906
|
+
// apart this cycle, so we assert nothing rather than assert wrongly.
|
|
907
|
+
if (ledger.unavailableErrorClass)
|
|
908
|
+
return false;
|
|
909
|
+
// Unreadable subtree or newly-ignored path: absent from `entries` for a
|
|
910
|
+
// reason that is not deletion. Same rule as the ledger loop above.
|
|
911
|
+
if (isOutsideOurView(candidate))
|
|
912
|
+
return false;
|
|
913
|
+
return true;
|
|
914
|
+
});
|
|
915
|
+
// An enormous removal set usually means the tree moved out from under us
|
|
916
|
+
// (an unmounted volume, a relocated HQ root), not that the user deleted
|
|
917
|
+
// 50k files. Re-baselining with a FULL manifest states what we can see
|
|
918
|
+
// instead of asserting a deletion we are not confident about.
|
|
919
|
+
if (removed.length <= SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK) {
|
|
920
|
+
mode = "delta";
|
|
921
|
+
wireEntries = changed;
|
|
922
|
+
removedPaths = removed;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
const chunkInput = {
|
|
926
|
+
entries: wireEntries,
|
|
927
|
+
removedPaths,
|
|
928
|
+
walkStats,
|
|
929
|
+
ignoreRules,
|
|
930
|
+
mode,
|
|
931
|
+
baseSnapshotId: mode === "delta" ? usableBase?.snapshotId : undefined,
|
|
932
|
+
generatedAt: snapshot.generatedAt,
|
|
933
|
+
installationId: options.installationId,
|
|
934
|
+
machineId: options.machineId,
|
|
935
|
+
scope: wireScope,
|
|
936
|
+
source: options.source,
|
|
937
|
+
sequence: options.sequence,
|
|
938
|
+
maxEntriesPerChunk: options.maxEntriesPerChunk,
|
|
939
|
+
chunkByteBudget: options.chunkByteBudget,
|
|
940
|
+
ledgerUnavailable: ledger.unavailableErrorClass
|
|
941
|
+
? { errorClass: ledger.unavailableErrorClass }
|
|
942
|
+
: undefined,
|
|
943
|
+
};
|
|
944
|
+
const chunkPlan = planManifestChunks(chunkInput);
|
|
945
|
+
// A truncated walk saw only part of the tree. Persisting it as the delta
|
|
946
|
+
// base would make every unseen file look REMOVED on the next cycle.
|
|
947
|
+
if (!walk.truncated && options.persistSnapshot !== false) {
|
|
948
|
+
writeManifestSnapshot(options.stateDir, scopeKey, snapshot);
|
|
949
|
+
}
|
|
950
|
+
return {
|
|
951
|
+
// Lazy on purpose — see the doc on `BuildSyncManifestResult.chunks`. The
|
|
952
|
+
// upload pass reads `chunkPlan` and never touches this.
|
|
953
|
+
get chunks() {
|
|
954
|
+
return chunkManifest(chunkInput);
|
|
955
|
+
},
|
|
956
|
+
chunkPlan,
|
|
957
|
+
mode,
|
|
958
|
+
walkStats,
|
|
959
|
+
snapshot,
|
|
960
|
+
unrepresentablePaths: walk.unrepresentablePaths,
|
|
961
|
+
skippedUnhashed,
|
|
962
|
+
unhashedFiles,
|
|
963
|
+
droppedStatRaces,
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
/**
|
|
967
|
+
* Split entries into contract-sized chunks. ALWAYS emits at least one chunk:
|
|
968
|
+
* an empty scope is a real, meaningful observation ("this machine has nothing
|
|
969
|
+
* here"), and swallowing it would look identical to "the client never ran".
|
|
970
|
+
*
|
|
971
|
+
* `removedPaths` ride the FIRST chunk only, so the server never has to
|
|
972
|
+
* de-duplicate a removal repeated across chunks.
|
|
973
|
+
*/
|
|
974
|
+
export function chunkManifest(input) {
|
|
975
|
+
const plan = planManifestChunks(input);
|
|
976
|
+
const uploads = [];
|
|
977
|
+
for (let chunkIndex = 0; chunkIndex < plan.chunkCount; chunkIndex++) {
|
|
978
|
+
uploads.push(plan.chunk(chunkIndex));
|
|
979
|
+
}
|
|
980
|
+
return uploads;
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Plan the chunk set without building it.
|
|
984
|
+
*
|
|
985
|
+
* Everything a chunk header carries is decided HERE, once — `chunkCount`
|
|
986
|
+
* included — so every chunk the plan later mints is byte-identical in the
|
|
987
|
+
* fields the server hashes into the snapshotId. Only the `entries` window and
|
|
988
|
+
* `chunkIndex` differ between chunks, which is the whole reason a chunk can be
|
|
989
|
+
* built on demand and thrown away instead of being held for the pass.
|
|
990
|
+
*
|
|
991
|
+
* ALWAYS plans at least one chunk: an empty scope is a real, meaningful
|
|
992
|
+
* observation ("this machine has nothing here"), and swallowing it would look
|
|
993
|
+
* identical to "the client never ran".
|
|
994
|
+
*/
|
|
995
|
+
export function planManifestChunks(input) {
|
|
996
|
+
// Clamped, never merely trusted: a caller-supplied size above the contract
|
|
997
|
+
// ceiling would build chunks the server rejects, and one below 1 would loop
|
|
998
|
+
// forever.
|
|
999
|
+
const perChunk = Math.min(SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK, Math.max(1, Math.floor(input.maxEntriesPerChunk ?? SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK)));
|
|
1000
|
+
const rawBudget = input.chunkByteBudget ?? SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET;
|
|
1001
|
+
// Same shape of clamp as `perChunk`, and for the same reason: a budget above
|
|
1002
|
+
// the server's own ceiling plans chunks it will refuse, and a non-positive
|
|
1003
|
+
// one would cut a chunk per entry (or never cut at all).
|
|
1004
|
+
const chunkByteBudget = Math.min(SYNC_MANIFEST_MAX_CHUNK_BYTES, Math.max(1, Math.floor(Number.isFinite(rawBudget) ? rawBudget : SYNC_MANIFEST_DEFAULT_CHUNK_BYTE_BUDGET)));
|
|
1005
|
+
// `removedPaths` ride the FIRST chunk only, so the server never has to
|
|
1006
|
+
// de-duplicate a removal repeated across chunks. Bounded once, not per chunk.
|
|
1007
|
+
const firstChunkRemovedPaths = input.removedPaths.slice(0, SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK);
|
|
1008
|
+
/**
|
|
1009
|
+
* Everything in a chunk that is NOT an entry: the frozen header, the
|
|
1010
|
+
* ignore-rule list, and (chunk 0 only) the removal list.
|
|
1011
|
+
*
|
|
1012
|
+
* Measured by serialising a real chunk with an EMPTY `entries` array, so it
|
|
1013
|
+
* cannot drift from what the wire actually carries the way a hand-counted
|
|
1014
|
+
* constant would. Both variants are measured because chunk 0 is the fat one
|
|
1015
|
+
* and budgeting every chunk against chunk 0's overhead would waste the
|
|
1016
|
+
* difference on all the others.
|
|
1017
|
+
*/
|
|
1018
|
+
// `chunkCount` is the value being computed, so it cannot be stamped
|
|
1019
|
+
// truthfully here. Both counters are measured at their WIDEST legal value
|
|
1020
|
+
// instead of their real one, which over-states the overhead by a few bytes
|
|
1021
|
+
// and never under-states it — the only direction that is safe, since an
|
|
1022
|
+
// under-stated overhead silently produces chunks fatter than the budget.
|
|
1023
|
+
const widestCounter = SYNC_MANIFEST_MAX_CHUNK_COUNT;
|
|
1024
|
+
const overheadFor = (chunkIndex) => Buffer.byteLength(JSON.stringify(buildChunk(input, firstChunkRemovedPaths, chunkIndex, widestCounter, [])), "utf8") +
|
|
1025
|
+
// `chunkIndex` is stamped as 0 or 1 by the two probes below but may be up
|
|
1026
|
+
// to `widestCounter` in a real chunk; charge the difference in digits.
|
|
1027
|
+
(String(widestCounter).length - String(chunkIndex).length);
|
|
1028
|
+
const firstChunkOverhead = overheadFor(0);
|
|
1029
|
+
const laterChunkOverhead = input.removedPaths.length > 0 ? overheadFor(1) : firstChunkOverhead;
|
|
1030
|
+
/**
|
|
1031
|
+
* SIZING PASS — the reason `chunkCount` can still be frozen before the first
|
|
1032
|
+
* POST.
|
|
1033
|
+
*
|
|
1034
|
+
* The server hashes `chunkCount` (with `generatedAt`, `sequence`,
|
|
1035
|
+
* `machineId`, `mode` and `scope`) into the snapshotId, so it MUST be known
|
|
1036
|
+
* up front and MUST NOT move mid-pass. With a count-only bound that was
|
|
1037
|
+
* arithmetic — `ceil(entries / perChunk)`. A byte budget makes the
|
|
1038
|
+
* boundaries data-dependent, so they have to be walked for.
|
|
1039
|
+
*
|
|
1040
|
+
* They are walked for ONCE, here, and recorded as start offsets. Each entry
|
|
1041
|
+
* is serialised a single time to learn its length and the string is dropped
|
|
1042
|
+
* immediately, so the pass costs O(entries) time and O(1) live memory on top
|
|
1043
|
+
* of the entries themselves — it never holds a chunk, a chunk's JSON, or a
|
|
1044
|
+
* second copy of anything. `chunk(i)` later slices between two recorded
|
|
1045
|
+
* offsets; nothing is serialised twice to decide where a cut goes.
|
|
1046
|
+
*
|
|
1047
|
+
* The alternative — switching the wire to a "final chunk" flag so boundaries
|
|
1048
|
+
* could be discovered while uploading — was rejected: it is a contract
|
|
1049
|
+
* change, it moves snapshotId derivation, and it buys nothing this pass does
|
|
1050
|
+
* not already get for one linear walk.
|
|
1051
|
+
*/
|
|
1052
|
+
const starts = [0];
|
|
1053
|
+
let maxChunkBytes = 0;
|
|
1054
|
+
{
|
|
1055
|
+
let used = firstChunkOverhead;
|
|
1056
|
+
let countInChunk = 0;
|
|
1057
|
+
for (let i = 0; i < input.entries.length; i++) {
|
|
1058
|
+
// `+ 1` for the `,` that joins this entry to the previous one. Charged
|
|
1059
|
+
// to every entry including the first, where it stands in for nothing —
|
|
1060
|
+
// one byte of over-estimate per chunk is the correct direction to err.
|
|
1061
|
+
const entryBytes = Buffer.byteLength(JSON.stringify(input.entries[i]), "utf8") + 1;
|
|
1062
|
+
const wouldExceedBytes = countInChunk > 0 && used + entryBytes > chunkByteBudget;
|
|
1063
|
+
const wouldExceedCount = countInChunk >= perChunk;
|
|
1064
|
+
if (wouldExceedBytes || wouldExceedCount) {
|
|
1065
|
+
maxChunkBytes = Math.max(maxChunkBytes, used);
|
|
1066
|
+
starts.push(i);
|
|
1067
|
+
used = laterChunkOverhead;
|
|
1068
|
+
countInChunk = 0;
|
|
1069
|
+
}
|
|
1070
|
+
// `countInChunk > 0` above is what keeps a single entry fatter than the
|
|
1071
|
+
// whole budget from wedging this loop: it is placed alone in its own
|
|
1072
|
+
// chunk and `maxChunkBytes` reports the overrun honestly rather than
|
|
1073
|
+
// silently dropping a file the user has.
|
|
1074
|
+
used += entryBytes;
|
|
1075
|
+
countInChunk += 1;
|
|
1076
|
+
}
|
|
1077
|
+
maxChunkBytes = Math.max(maxChunkBytes, used);
|
|
1078
|
+
}
|
|
1079
|
+
// ALWAYS at least one chunk: an empty scope is a real, meaningful
|
|
1080
|
+
// observation ("this machine has nothing here"), and swallowing it would
|
|
1081
|
+
// look identical to "the client never ran". `starts` already begins at 0, so
|
|
1082
|
+
// the empty case falls out without a special branch.
|
|
1083
|
+
const chunkCount = starts.length;
|
|
1084
|
+
// Counted in one pass here rather than by the upload guard, which would
|
|
1085
|
+
// otherwise have to re-walk every chunk it is trying to avoid materialising.
|
|
1086
|
+
let diskEntryCount = 0;
|
|
1087
|
+
for (const entry of input.entries)
|
|
1088
|
+
if (entry.source === "disk")
|
|
1089
|
+
diskEntryCount += 1;
|
|
1090
|
+
return {
|
|
1091
|
+
chunkCount,
|
|
1092
|
+
entryCount: input.entries.length,
|
|
1093
|
+
chunkByteBudget,
|
|
1094
|
+
maxChunkBytes,
|
|
1095
|
+
diskEntryCount,
|
|
1096
|
+
mode: input.mode,
|
|
1097
|
+
chunk(chunkIndex) {
|
|
1098
|
+
if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= chunkCount) {
|
|
1099
|
+
throw new RangeError(`manifest chunk index ${chunkIndex} is outside 0..${chunkCount - 1}`);
|
|
1100
|
+
}
|
|
1101
|
+
const start = starts[chunkIndex];
|
|
1102
|
+
const stop = chunkIndex + 1 < chunkCount ? starts[chunkIndex + 1] : input.entries.length;
|
|
1103
|
+
return buildChunk(input, firstChunkRemovedPaths, chunkIndex, chunkCount, input.entries.slice(start, stop));
|
|
1104
|
+
},
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Mint one chunk from the frozen header plus a window of entries.
|
|
1109
|
+
*
|
|
1110
|
+
* Extracted so the sizing pass measures header overhead by serialising the
|
|
1111
|
+
* REAL chunk shape (with an empty window) rather than a hand-maintained
|
|
1112
|
+
* replica of it. A replica is exactly the thing that drifts when a field is
|
|
1113
|
+
* added, and it would drift SILENTLY — under-counting overhead just makes
|
|
1114
|
+
* chunks quietly bigger than the budget claims.
|
|
1115
|
+
*/
|
|
1116
|
+
function buildChunk(input, firstChunkRemovedPaths, chunkIndex, chunkCount, entries) {
|
|
1117
|
+
const upload = {
|
|
1118
|
+
contractVersion: SYNC_MANIFEST_CONTRACT_VERSION,
|
|
1119
|
+
installationId: input.installationId,
|
|
1120
|
+
scope: input.scope,
|
|
1121
|
+
machineId: input.machineId,
|
|
1122
|
+
source: input.source,
|
|
1123
|
+
mode: input.mode,
|
|
1124
|
+
generatedAt: input.generatedAt,
|
|
1125
|
+
sequence: input.sequence,
|
|
1126
|
+
chunkIndex,
|
|
1127
|
+
chunkCount,
|
|
1128
|
+
entries,
|
|
1129
|
+
removedPaths: chunkIndex === 0 ? firstChunkRemovedPaths : [],
|
|
1130
|
+
walkStats: input.walkStats,
|
|
1131
|
+
ignoreRules: input.ignoreRules,
|
|
1132
|
+
};
|
|
1133
|
+
if (input.baseSnapshotId)
|
|
1134
|
+
upload.baseSnapshotId = input.baseSnapshotId;
|
|
1135
|
+
if (input.ledgerUnavailable)
|
|
1136
|
+
upload.ledgerUnavailable = input.ledgerUnavailable;
|
|
1137
|
+
return upload;
|
|
1138
|
+
}
|
|
1139
|
+
//# sourceMappingURL=build-manifest.js.map
|