@indigoai-us/hq-cloud 6.16.19 → 6.16.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/bin/sync-runner-installation-id.d.ts +66 -0
  2. package/dist/bin/sync-runner-installation-id.d.ts.map +1 -0
  3. package/dist/bin/sync-runner-installation-id.js +138 -0
  4. package/dist/bin/sync-runner-installation-id.js.map +1 -0
  5. package/dist/bin/sync-runner-manifest.d.ts +120 -0
  6. package/dist/bin/sync-runner-manifest.d.ts.map +1 -0
  7. package/dist/bin/sync-runner-manifest.js +348 -0
  8. package/dist/bin/sync-runner-manifest.js.map +1 -0
  9. package/dist/bin/sync-runner-manifest.test.d.ts +12 -0
  10. package/dist/bin/sync-runner-manifest.test.d.ts.map +1 -0
  11. package/dist/bin/sync-runner-manifest.test.js +510 -0
  12. package/dist/bin/sync-runner-manifest.test.js.map +1 -0
  13. package/dist/bin/sync-runner.d.ts +56 -21
  14. package/dist/bin/sync-runner.d.ts.map +1 -1
  15. package/dist/bin/sync-runner.js +86 -61
  16. package/dist/bin/sync-runner.js.map +1 -1
  17. package/dist/bin/sync-runner.test.js +217 -0
  18. package/dist/bin/sync-runner.test.js.map +1 -1
  19. package/dist/cli/share-presign-transport.test.d.ts +22 -0
  20. package/dist/cli/share-presign-transport.test.d.ts.map +1 -0
  21. package/dist/cli/share-presign-transport.test.js +237 -0
  22. package/dist/cli/share-presign-transport.test.js.map +1 -0
  23. package/dist/cli/share.d.ts.map +1 -1
  24. package/dist/cli/share.js +17 -3
  25. package/dist/cli/share.js.map +1 -1
  26. package/dist/cli/sync.d.ts.map +1 -1
  27. package/dist/cli/sync.js +9 -0
  28. package/dist/cli/sync.js.map +1 -1
  29. package/dist/cli/sync.test.js +123 -0
  30. package/dist/cli/sync.test.js.map +1 -1
  31. package/dist/company-slug.d.ts +31 -0
  32. package/dist/company-slug.d.ts.map +1 -0
  33. package/dist/company-slug.js +52 -0
  34. package/dist/company-slug.js.map +1 -0
  35. package/dist/company-slug.test.d.ts +7 -0
  36. package/dist/company-slug.test.d.ts.map +1 -0
  37. package/dist/company-slug.test.js +43 -0
  38. package/dist/company-slug.test.js.map +1 -0
  39. package/dist/company-vault-transport.d.ts +69 -0
  40. package/dist/company-vault-transport.d.ts.map +1 -0
  41. package/dist/company-vault-transport.js +89 -0
  42. package/dist/company-vault-transport.js.map +1 -0
  43. package/dist/company-vault-transport.test.d.ts +12 -0
  44. package/dist/company-vault-transport.test.d.ts.map +1 -0
  45. package/dist/company-vault-transport.test.js +97 -0
  46. package/dist/company-vault-transport.test.js.map +1 -0
  47. package/dist/index.d.ts +2 -0
  48. package/dist/index.d.ts.map +1 -1
  49. package/dist/index.js +5 -0
  50. package/dist/index.js.map +1 -1
  51. package/dist/journal-ledger-index.test.d.ts +15 -0
  52. package/dist/journal-ledger-index.test.d.ts.map +1 -0
  53. package/dist/journal-ledger-index.test.js +247 -0
  54. package/dist/journal-ledger-index.test.js.map +1 -0
  55. package/dist/journal.d.ts +110 -0
  56. package/dist/journal.d.ts.map +1 -1
  57. package/dist/journal.js +253 -5
  58. package/dist/journal.js.map +1 -1
  59. package/dist/manifest/US-002.acceptance.test.d.ts +11 -0
  60. package/dist/manifest/US-002.acceptance.test.d.ts.map +1 -0
  61. package/dist/manifest/US-002.acceptance.test.js +160 -0
  62. package/dist/manifest/US-002.acceptance.test.js.map +1 -0
  63. package/dist/manifest/US-004.acceptance.test.d.ts +12 -0
  64. package/dist/manifest/US-004.acceptance.test.d.ts.map +1 -0
  65. package/dist/manifest/US-004.acceptance.test.js +173 -0
  66. package/dist/manifest/US-004.acceptance.test.js.map +1 -0
  67. package/dist/manifest/build-manifest.bench.test.d.ts +22 -0
  68. package/dist/manifest/build-manifest.bench.test.d.ts.map +1 -0
  69. package/dist/manifest/build-manifest.bench.test.js +544 -0
  70. package/dist/manifest/build-manifest.bench.test.js.map +1 -0
  71. package/dist/manifest/build-manifest.d.ts +319 -0
  72. package/dist/manifest/build-manifest.d.ts.map +1 -0
  73. package/dist/manifest/build-manifest.js +1043 -0
  74. package/dist/manifest/build-manifest.js.map +1 -0
  75. package/dist/manifest/build-manifest.test.d.ts +10 -0
  76. package/dist/manifest/build-manifest.test.d.ts.map +1 -0
  77. package/dist/manifest/build-manifest.test.js +1060 -0
  78. package/dist/manifest/build-manifest.test.js.map +1 -0
  79. package/dist/manifest/contract.d.ts +290 -0
  80. package/dist/manifest/contract.d.ts.map +1 -0
  81. package/dist/manifest/contract.js +476 -0
  82. package/dist/manifest/contract.js.map +1 -0
  83. package/dist/manifest/contract.test.d.ts +17 -0
  84. package/dist/manifest/contract.test.d.ts.map +1 -0
  85. package/dist/manifest/contract.test.js +283 -0
  86. package/dist/manifest/contract.test.js.map +1 -0
  87. package/dist/manifest/import-graph.test.d.ts +25 -0
  88. package/dist/manifest/import-graph.test.d.ts.map +1 -0
  89. package/dist/manifest/import-graph.test.js +120 -0
  90. package/dist/manifest/import-graph.test.js.map +1 -0
  91. package/dist/manifest/ledger-shard-isolation.test.d.ts +18 -0
  92. package/dist/manifest/ledger-shard-isolation.test.d.ts.map +1 -0
  93. package/dist/manifest/ledger-shard-isolation.test.js +153 -0
  94. package/dist/manifest/ledger-shard-isolation.test.js.map +1 -0
  95. package/dist/manifest/snapshot-store.d.ts +143 -0
  96. package/dist/manifest/snapshot-store.d.ts.map +1 -0
  97. package/dist/manifest/snapshot-store.js +291 -0
  98. package/dist/manifest/snapshot-store.js.map +1 -0
  99. package/dist/manifest/snapshot-store.test.d.ts +11 -0
  100. package/dist/manifest/snapshot-store.test.d.ts.map +1 -0
  101. package/dist/manifest/snapshot-store.test.js +133 -0
  102. package/dist/manifest/snapshot-store.test.js.map +1 -0
  103. package/dist/manifest/upload-manifest.d.ts +301 -0
  104. package/dist/manifest/upload-manifest.d.ts.map +1 -0
  105. package/dist/manifest/upload-manifest.js +921 -0
  106. package/dist/manifest/upload-manifest.js.map +1 -0
  107. package/dist/manifest/upload-manifest.test.d.ts +10 -0
  108. package/dist/manifest/upload-manifest.test.d.ts.map +1 -0
  109. package/dist/manifest/upload-manifest.test.js +1052 -0
  110. package/dist/manifest/upload-manifest.test.js.map +1 -0
  111. package/dist/manifest-reconcile.d.ts.map +1 -1
  112. package/dist/manifest-reconcile.js +1 -26
  113. package/dist/manifest-reconcile.js.map +1 -1
  114. package/dist/posix-key.d.ts +36 -0
  115. package/dist/posix-key.d.ts.map +1 -0
  116. package/dist/posix-key.js +38 -0
  117. package/dist/posix-key.js.map +1 -0
  118. package/dist/s3.d.ts +2 -18
  119. package/dist/s3.d.ts.map +1 -1
  120. package/dist/s3.js +6 -20
  121. package/dist/s3.js.map +1 -1
  122. package/dist/sync/journal-ledger-stream.d.ts +56 -0
  123. package/dist/sync/journal-ledger-stream.d.ts.map +1 -0
  124. package/dist/sync/journal-ledger-stream.js +191 -0
  125. package/dist/sync/journal-ledger-stream.js.map +1 -0
  126. package/dist/sync/journal-ledger-stream.test.d.ts +12 -0
  127. package/dist/sync/journal-ledger-stream.test.d.ts.map +1 -0
  128. package/dist/sync/journal-ledger-stream.test.js +134 -0
  129. package/dist/sync/journal-ledger-stream.test.js.map +1 -0
  130. package/dist/types.d.ts +18 -7
  131. package/dist/types.d.ts.map +1 -1
  132. package/package.json +2 -1
@@ -0,0 +1,1043 @@
1
+ /**
2
+ * Client-side sync manifest generator (sync-reconciliation-audit US-002).
3
+ *
4
+ * WHAT THIS IS FOR
5
+ * ----------------
6
+ * The reconciliation audit compares three pictures of a scope: what the vault
7
+ * holds, what this machine's journal (the "ledger") believes it synced, and
8
+ * what is actually ON DISK. Only the client can produce the last two, and only
9
+ * together do they distinguish the failure modes that matter:
10
+ *
11
+ * - a file on disk with no ledger row → the daemon stopped tracking it
12
+ * (wedged watcher, crashed push) — invisible to any server-side check;
13
+ * - a ledger row with no file on disk → the local copy vanished;
14
+ * - both present and in agreement → healthy.
15
+ *
16
+ * That is exactly the `disk` / `ledger` / `both` split the wire contract
17
+ * encodes, and it is why this builder unions the two sources instead of
18
+ * walking the disk alone.
19
+ *
20
+ * WHAT THIS DELIBERATELY DOES NOT DO
21
+ * ----------------------------------
22
+ * - It never reads `sync-journal.*.json` by path. The v3 journal's real
23
+ * state lives in a digest-named store; the legacy filename is only a
24
+ * locator. `listJournals()` is the ONLY supported way to enumerate shards.
25
+ * - It never re-implements ignore rules. `createIgnoreFilter()` owns the
26
+ * layered DEFAULT_IGNORES / .hqignore / .hqinclude semantics; a second
27
+ * implementation here would drift and manufacture false findings for every
28
+ * file the two disagree about.
29
+ * - It never reads file CONTENT unless it has to. The walk is stat-only; a
30
+ * hash is computed only when the ledger has no usable one (see
31
+ * {@link canReuseLedgerHash}). On a healthy machine that is ~zero files.
32
+ *
33
+ * COST DISCIPLINE
34
+ * ---------------
35
+ * This runs on the user's laptop, in the background, beside their editor. The
36
+ * walk yields to the event loop every `yieldEvery` files, honours an
37
+ * `AbortSignal`, and hard-stops at `maxWallMs` — marking the manifest
38
+ * `truncated` rather than pretending a partial picture is complete. A
39
+ * truncated manifest never emits `removedPaths` and never overwrites the
40
+ * snapshot, because "I stopped early" and "these files are gone" are the same
41
+ * observation from the walker's point of view and only one of them is true.
42
+ */
43
+ import * as fs from "node:fs";
44
+ import * as path from "node:path";
45
+ import { DEFAULT_IGNORES, createIgnoreFilter, } from "../ignore.js";
46
+ import { PERSONAL_VAULT_JOURNAL_SLUG, hashFile, hashSymlinkTarget, readJournalLedgerIndexResult, toLedgerIndex, } from "../journal.js";
47
+ import { isSafeCompanySlug } from "../company-slug.js";
48
+ // Leaf module (node builtins + `personal-vault-exclusions.ts`), so importing it
49
+ // keeps `import-graph.test.ts` green. Sharing it is the whole point: the
50
+ // personal manifest must describe the SAME tree the personal push leg ships.
51
+ import { computePersonalVaultPaths } from "../personal-vault.js";
52
+ import { PERSONAL_VAULT_DEFAULT_EXCLUSIONS, wrapFilterWithPersonalVaultDefaults } from "../personal-vault-exclusions.js";
53
+ import { SESSIONS_SCOPE_PREFIX } from "../sync/pull-scope.js";
54
+ import { SYNC_MANIFEST_CONTRACT_VERSION, SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK, SYNC_MANIFEST_MAX_FILE_SIZE_BYTES, SYNC_MANIFEST_MAX_IGNORE_RULES, SYNC_MANIFEST_MAX_IGNORE_RULE_LENGTH, SYNC_MANIFEST_MAX_MTIME_MS, SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK, assertSyncManifestPath, } from "./contract.js";
55
+ import { isManifestSnapshotBaseUsable, isManifestSnapshotStale, manifestSnapshotScopeKey, newManifestSnapshotId, readManifestSnapshot, writeManifestSnapshot, } from "./snapshot-store.js";
56
+ /**
57
+ * Default per-pass ceiling on files the `auto` policy may READ.
58
+ *
59
+ * Sized so the hash phase stays comparable to the stat walk it follows rather
60
+ * than dwarfing it: a few thousand small files is sub-second, while the
61
+ * unbounded version of this loop measured in minutes on a real vault.
62
+ */
63
+ export const DEFAULT_MANIFEST_MAX_HASHED_FILES = 2_000;
64
+ /** Default per-pass wall-clock ceiling on the `auto` policy's hash phase. */
65
+ export const DEFAULT_MANIFEST_MAX_HASH_MS = 5_000;
66
+ /** Default wall-clock cap for one scope's walk. */
67
+ export const DEFAULT_MANIFEST_MAX_WALL_MS = 30_000;
68
+ /** Default number of files walked between event-loop yields. */
69
+ export const DEFAULT_MANIFEST_YIELD_EVERY = 500;
70
+ /**
71
+ * Ignore-filter memo size for a manifest walk — deliberately far below the
72
+ * filter's own 50k default.
73
+ *
74
+ * That default is tuned for the sync engine, which probes the SAME paths on
75
+ * every cycle and lives off the cache hits. A manifest walk visits each path
76
+ * exactly once, so the memo can never hit; all it does is retain a key string
77
+ * per file (and keep the `ignore` library's internal per-path cache alive
78
+ * beside it) for the whole walk — tens of megabytes of resident memory on a
79
+ * 70k-file root, bought for zero hits. A small window keeps both layers
80
+ * bounded at the cost of a periodic matcher rebuild.
81
+ */
82
+ export const MANIFEST_IGNORE_MEMO_LIMIT = 2_000;
83
+ /**
84
+ * Classify the manifest upload route's response.
85
+ *
86
+ * The one non-obvious rule: **404 is a SOFT SKIP, not an error.** hq-cloud
87
+ * ships to end-user machines and updates independently of the server, so a
88
+ * client that knows about the manifest route will routinely meet a deployment
89
+ * that does not host it yet. Treating that as an error would light up every
90
+ * such user's logs (and their support tickets) with a failure that describes
91
+ * a perfectly healthy machine. It is logged at info and the cycle moves on.
92
+ *
93
+ * A 409 — or any 2xx/4xx body asking for `resend_full` — means the server lost
94
+ * its delta base, so the next manifest must be FULL.
95
+ */
96
+ export function classifyManifestUploadResponse(status, body) {
97
+ if (status === 404)
98
+ return { outcome: "soft_skip" };
99
+ if (status === 409)
100
+ return { outcome: "resend_full" };
101
+ if (bodyRequestsResendFull(body))
102
+ return { outcome: "resend_full" };
103
+ if (status >= 200 && status < 300)
104
+ return { outcome: "ok" };
105
+ return { outcome: "error" };
106
+ }
107
+ function bodyRequestsResendFull(body) {
108
+ if (body === null || typeof body !== "object" || Array.isArray(body))
109
+ return false;
110
+ const raw = body;
111
+ return raw.action === "resend_full" || raw.reason === "resend_full" || raw.resendFull === true;
112
+ }
113
+ // ─── Path representation ─────────────────────────────────────────────────────
114
+ /**
115
+ * Convert an OS-relative path into the contract's wire form, or `null` when it
116
+ * cannot be represented.
117
+ *
118
+ * macOS hands back NFD (`e` + combining acute); S3 stores whatever bytes it is
119
+ * given. Two spellings of one filename would reconcile as a false
120
+ * MISSING_IN_VAULT plus a false ORPHAN_IN_VAULT for the SAME file, so the wire
121
+ * form is pinned to NFC. Anything the contract still rejects afterwards (a
122
+ * control character, a lone surrogate, a bidi override, an over-long path) is
123
+ * SKIPPED and counted rather than thrown: such a path can never become a vault
124
+ * key anyway, so including it would only manufacture a guaranteed per-file
125
+ * failure downstream — and throwing would let one pathological filename
126
+ * suppress the entire machine's manifest.
127
+ */
128
+ function toManifestPath(relative) {
129
+ if (!relative)
130
+ return null;
131
+ const posix = relative.split(path.sep).join("/").normalize("NFC");
132
+ try {
133
+ return assertSyncManifestPath("path", posix);
134
+ }
135
+ catch {
136
+ return null;
137
+ }
138
+ }
139
+ /**
140
+ * Derive a bare error CLASS name for the wire.
141
+ *
142
+ * The contract accepts a class name and nothing else, on purpose: an error
143
+ * MESSAGE routinely carries an absolute vault path
144
+ * (`ENOENT ... /Users/x/hq/companies/acme/deal.md`), and that path would then
145
+ * travel into server-side audit findings and logs. A class name carries the
146
+ * shape of the failure with none of the tenant data.
147
+ */
148
+ function errorClassOf(err) {
149
+ const name = err instanceof Error ? err.constructor?.name ?? err.name : typeof err;
150
+ const cleaned = String(name).replace(/[^A-Za-z0-9_]/g, "");
151
+ if (!cleaned || !/^[A-Za-z]/.test(cleaned))
152
+ return "UnknownError";
153
+ return cleaned.slice(0, 64);
154
+ }
155
+ /**
156
+ * Read this scope's ledger rows out of the v3 journal.
157
+ *
158
+ * `listJournals()` is the only supported enumeration surface (a caller that
159
+ * reconstructs a path sees one shard at best, and a mangled non-slug at
160
+ * worst). A journal that cannot be read is NOT fatal: the audit still wants
161
+ * the disk picture, and `ledgerUnavailable` tells the server to read every
162
+ * `disk` entry as "ledger unknown" rather than "ledger missing" — otherwise a
163
+ * momentarily locked journal manufactures a findings storm.
164
+ */
165
+ function readLedger(journalSlug, listJournalsImpl) {
166
+ try {
167
+ // ONE shard, read straight into a compact index — never the machine's
168
+ // whole journal set, and never a full `SyncJournal` kept alive beside the
169
+ // map derived from it.
170
+ //
171
+ // `listJournals()` read and parsed every slug on disk, and `readJournal()`
172
+ // deep-cloned each result into the baseline WeakMap, so this line used to
173
+ // cost roughly twice the total size of every journal on the machine and
174
+ // paid it again for every scope in a fanout, to keep exactly one of them.
175
+ // `readJournalLedgerIndex` reads the one shard, streams it when it is a
176
+ // legacy JSON document, and retains only the six fields a reporting caller
177
+ // reads — so the parse it came from is collectable before the walk starts.
178
+ //
179
+ // `listJournalsImpl` stays honored for the tests that pin ledger behavior
180
+ // through a fabricated summary list; it takes the same compact projection,
181
+ // so the seam cannot drift from the real read.
182
+ if (listJournalsImpl) {
183
+ const summary = listJournalsImpl().find((candidate) => candidate.slug === journalSlug);
184
+ // A scope with no shard yet (first sync) is NOT "unavailable" — it is an
185
+ // empty, perfectly readable ledger. Conflating the two would suppress the
186
+ // very finding this feature exists to surface: files on disk that the
187
+ // journal never tracked.
188
+ return { entries: summary ? toLedgerIndex(summary.journal) : new Map() };
189
+ }
190
+ // `readJournalLedgerIndexResult`, NOT `readJournalLedgerIndex`: the latter
191
+ // answers `null` for BOTH "this scope has no ledger yet" and "the ledger
192
+ // exists and cannot be read", and coalescing that `null` to an empty Map
193
+ // is how a machine whose v3 store was corrupt in every generation reported
194
+ // all 188,997 of its files as `source: "disk"` — on-disk-never-tracked —
195
+ // and uploaded that as a complete, authoritative FULL manifest.
196
+ //
197
+ // An unreadable ledger is not an empty one. It takes the `ledgerUnavailable`
198
+ // path, where the builder asserts nothing about ledger agreement and the
199
+ // upload pass refuses to publish an implausible picture.
200
+ const result = readJournalLedgerIndexResult(journalSlug);
201
+ if (result.status === "unreadable") {
202
+ return { entries: new Map(), unavailableErrorClass: result.errorClass ?? "UnknownError" };
203
+ }
204
+ return { entries: result.index };
205
+ }
206
+ catch (err) {
207
+ return { entries: new Map(), unavailableErrorClass: errorClassOf(err) };
208
+ }
209
+ }
210
+ // ─── Ignore rules ────────────────────────────────────────────────────────────
211
+ function readIgnoreFileLines(filePath) {
212
+ let text;
213
+ try {
214
+ text = fs.readFileSync(filePath, "utf8");
215
+ }
216
+ catch {
217
+ return [];
218
+ }
219
+ return text
220
+ .split(/\r?\n/)
221
+ .map((line) => line.trim())
222
+ .filter((line) => line.length > 0 && !line.startsWith("#"));
223
+ }
224
+ /**
225
+ * Compile the ignore-rule list the walk ACTUALLY applied, in precedence order.
226
+ *
227
+ * The server needs this to tell "the client never looked at that file" apart
228
+ * from "the client looked and the file is gone" — without it, a user's
229
+ * `.hqignore` line makes every file under it look like drift. The layers
230
+ * mirror `createIgnoreFilterFromSnapshot` exactly: defaults, the layout-aware
231
+ * root `core.yaml` exclusion, `.hqignore` (or its legacy `.hqsyncignore`
232
+ * spelling), then `.hqinclude` allowlist entries — tagged `include:` because
233
+ * their verdict is INVERTED and an untagged list would read as an exclusion.
234
+ * `.gitignore` is deliberately absent: sync has not consulted it since the
235
+ * 2026-08 personal-vault carve-out incident.
236
+ */
237
+ function compileIgnoreRules(hqRoot, scopeKind) {
238
+ const rules = [...DEFAULT_IGNORES];
239
+ if (fs.existsSync(path.join(hqRoot, "core", "core.yaml")))
240
+ rules.push("/core.yaml");
241
+ const hqignore = readIgnoreFileLines(path.join(hqRoot, ".hqignore"));
242
+ rules.push(...(hqignore.length > 0 ? hqignore : readIgnoreFileLines(path.join(hqRoot, ".hqsyncignore"))));
243
+ for (const line of readIgnoreFileLines(path.join(hqRoot, ".hqinclude"))) {
244
+ rules.push(`include:${line}`);
245
+ }
246
+ // Never-pulled, push-only prefix; the walk skips it, so it belongs in the
247
+ // rule list the server reads.
248
+ rules.push(SESSIONS_SCOPE_PREFIX);
249
+ if (scopeKind === "personal") {
250
+ for (const exclusion of PERSONAL_VAULT_DEFAULT_EXCLUSIONS) {
251
+ for (const pattern of exclusion.pattern.split(",")) {
252
+ const trimmed = pattern.trim();
253
+ if (trimmed)
254
+ rules.push(trimmed);
255
+ }
256
+ }
257
+ }
258
+ const bounded = [];
259
+ for (const rule of rules) {
260
+ if (bounded.length >= SYNC_MANIFEST_MAX_IGNORE_RULES)
261
+ break;
262
+ if (rule.length === 0 || rule.length > SYNC_MANIFEST_MAX_IGNORE_RULE_LENGTH)
263
+ continue;
264
+ bounded.push(rule);
265
+ }
266
+ return bounded;
267
+ }
268
+ function yieldToEventLoop() {
269
+ // `setTimeout(…, 0)` rather than `setImmediate` so the same code runs
270
+ // unchanged under any bundler/runtime that only guarantees timers.
271
+ return new Promise((resolve) => setTimeout(resolve, 0));
272
+ }
273
+ /**
274
+ * Stat one file the walk reached and fold it into the outcome.
275
+ *
276
+ * Extracted from the dirent loop so the walk's ROOTS can be folded in the same
277
+ * way: a personal-vault walk root may itself be a FILE (`companies/manifest.yaml`,
278
+ * `workspace/threads/handoff.json`), and re-implementing the collision, NFC and
279
+ * `sessions/` rules for that case is exactly how the two paths would drift.
280
+ *
281
+ * `lstat` (never `stat`) so a symlink is observed as itself rather than as its
282
+ * target: following it would both mis-report the entry's size/mtime and let a
283
+ * link out of the scope smuggle foreign content into a tenant's manifest.
284
+ */
285
+ function recordWalkedFile(outcome, collidedKeys, absPath, relative) {
286
+ const manifestPath = toManifestPath(relative);
287
+ if (manifestPath === null) {
288
+ outcome.unrepresentablePaths += 1;
289
+ return;
290
+ }
291
+ // `sessions/` is push-only by contract and is excluded from every pull
292
+ // scope (`resolvePullScope().excludePrefixes`). Enumerating it here would
293
+ // report every teammate's un-pulled transcripts as missing locally.
294
+ if (manifestPath.startsWith(SESSIONS_SCOPE_PREFIX)) {
295
+ outcome.ignoredFiles += 1;
296
+ return;
297
+ }
298
+ if (outcome.records.has(manifestPath)) {
299
+ // Distinct on-disk names, one wire key. Neither can be reported
300
+ // truthfully, so both are dropped and counted as unrepresentable —
301
+ // the same treatment as any other path the wire cannot express.
302
+ //
303
+ // Checked BEFORE `lstat`, deliberately. The ambiguity is a property of the
304
+ // directory LISTING, not of whether the second spelling happens to resolve:
305
+ // on a normalization-insensitive filesystem (macOS) both names stat to the
306
+ // same file, while on a byte-exact one (Linux) the NFD spelling of a file
307
+ // stored as NFC does not resolve at all. Statting first would make the
308
+ // collision invisible on exactly the platform where the two names really
309
+ // are two different entries, and the manifest would then publish one
310
+ // file's content under the other's name.
311
+ collidedKeys.set(manifestPath, (collidedKeys.get(manifestPath) ?? 0) + 1);
312
+ return;
313
+ }
314
+ let stats;
315
+ try {
316
+ stats = fs.lstatSync(absPath);
317
+ }
318
+ catch {
319
+ return; // Raced away between readdir and lstat.
320
+ }
321
+ if (!stats.isFile() && !stats.isSymbolicLink())
322
+ return;
323
+ outcome.filesWalked += 1;
324
+ const record = {
325
+ size: stats.size,
326
+ mtimeMs: Math.floor(stats.mtimeMs),
327
+ isSymlink: stats.isSymbolicLink(),
328
+ };
329
+ // Only pay for the second string when normalization actually changed the
330
+ // path; `relative === manifestPath` for essentially every ASCII name.
331
+ if (relative !== manifestPath)
332
+ record.raw = relative;
333
+ outcome.records.set(manifestPath, record);
334
+ }
335
+ /**
336
+ * Stat-only, cooperative walk of one scope's WALK ROOTS.
337
+ *
338
+ * `scopeRoot` anchors the relative paths that go on the wire; `walkRoots` are
339
+ * the subtrees actually enumerated. For a company scope the two coincide (one
340
+ * root, the company directory). For the PERSONAL scope they emphatically do
341
+ * not: the personal push leg walks `computePersonalVaultPaths()`, which
342
+ * excludes `companies/` (bar the local, non-cloud carve-outs) along with
343
+ * `repos/`, `workspace/` and `.git`. Walking the whole HQ root here instead
344
+ * would put every cloud company's file under the PERSONAL manifest — a
345
+ * tenant-boundary report the personal vault does not and must not hold.
346
+ *
347
+ * A walk root that is a FILE rather than a directory is recorded directly:
348
+ * `computePersonalVaultPaths()` returns single-file carve-outs
349
+ * (`companies/manifest.yaml`, the continuity pointer) alongside directories.
350
+ */
351
+ async function walkScope(scopeRoot, walkRoots, shouldSync, options) {
352
+ const records = new Map();
353
+ const outcome = {
354
+ records,
355
+ filesWalked: 0,
356
+ ignoredFiles: 0,
357
+ unrepresentablePaths: 0,
358
+ truncated: false,
359
+ lostVisibilityPrefixes: [],
360
+ };
361
+ // Two raw spellings that collapse onto one NFC key are indistinguishable on
362
+ // the wire, and keeping either would silently publish one file's hash under
363
+ // the other's name (last write wins). Both are dropped after the walk.
364
+ // key → how many EXTRA raw spellings landed on it (the first one is in the
365
+ // map already and is removed too).
366
+ const collidedKeys = new Map();
367
+ // The stack carries each directory's SCOPE-RELATIVE prefix alongside its
368
+ // absolute path. Deriving the relative path this way (prefix + "/" + name)
369
+ // rather than calling `path.relative` per file removes a normalize + split +
370
+ // join + array allocation from the hottest loop in the product — worth
371
+ // roughly a third of the walk's resident growth on a 70k-file root.
372
+ const stack = [];
373
+ // Deduped: `computePersonalVaultPaths()` composes several carve-out lists and
374
+ // is not required to return a disjoint set. Walking one root twice would not
375
+ // corrupt the manifest (the collision map would drop the second spelling),
376
+ // but it would count every file underneath as unrepresentable and delete it
377
+ // from the report — a far worse outcome than a `Set`.
378
+ const seenRoots = new Set();
379
+ for (const walkRoot of walkRoots) {
380
+ const resolved = path.resolve(walkRoot);
381
+ if (seenRoots.has(resolved))
382
+ continue;
383
+ seenRoots.add(resolved);
384
+ const relRoot = resolved === path.resolve(scopeRoot) ? "" : path.relative(scopeRoot, resolved);
385
+ // A root outside the scope anchor cannot be given a scope-relative path, so
386
+ // it cannot be reported truthfully. Dropping it is the honest answer.
387
+ if (relRoot.startsWith("..") || path.isAbsolute(relRoot))
388
+ continue;
389
+ const relDir = relRoot.split(path.sep).join("/");
390
+ // The root itself is subject to the ignore filter, exactly like any dirent.
391
+ let rootStats;
392
+ try {
393
+ rootStats = fs.lstatSync(resolved);
394
+ }
395
+ catch {
396
+ // A named root that is not there is a benign race (or a carve-out for a
397
+ // file this machine does not have). Not a lost-visibility prefix: we know
398
+ // it is absent, we did not fail to look.
399
+ continue;
400
+ }
401
+ const rootIsDir = rootStats.isDirectory();
402
+ if (relDir !== "" && !shouldSync(resolved, rootIsDir)) {
403
+ if (!rootIsDir)
404
+ outcome.ignoredFiles += 1;
405
+ continue;
406
+ }
407
+ if (rootIsDir) {
408
+ stack.push({ absDir: resolved, relDir });
409
+ continue;
410
+ }
411
+ if (relDir === "")
412
+ continue; // The scope anchor itself is not a file entry.
413
+ recordWalkedFile(outcome, collidedKeys, resolved, relDir);
414
+ }
415
+ let sinceYield = 0;
416
+ while (stack.length > 0) {
417
+ if (options.abortSignal?.aborted || Date.now() > options.deadlineMs) {
418
+ outcome.truncated = true;
419
+ break;
420
+ }
421
+ const { absDir, relDir } = stack.pop();
422
+ let dirents;
423
+ try {
424
+ dirents = fs.readdirSync(absDir, { withFileTypes: true });
425
+ }
426
+ catch {
427
+ // An unreadable directory (permissions, raced deletion) is a gap in the
428
+ // picture, not a reason to abandon the rest of the tree — so the walk
429
+ // continues. But the gap MUST be recorded: silently skipping it makes
430
+ // every file underneath look like it vanished locally, which then
431
+ // surfaces as a false `source: "ledger"` entry or, worse, a false
432
+ // removal that the server reads as a deletion.
433
+ //
434
+ // We record the PREFIX rather than marking the whole build `truncated`
435
+ // (the other available answer) because it is nearly free here and far
436
+ // more useful: `truncated` would suppress removals, ledger synthesis and
437
+ // the snapshot write for the ENTIRE scope, so one unreadable directory
438
+ // would permanently disable delta mode for the whole root. Scoping the
439
+ // suppression to the affected subtree keeps the rest of the manifest
440
+ // fully usable.
441
+ outcome.lostVisibilityPrefixes.push(relDir === "" ? "" : `${relDir}/`);
442
+ continue;
443
+ }
444
+ for (const dirent of dirents) {
445
+ const absPath = path.join(absDir, dirent.name);
446
+ const isDir = dirent.isDirectory();
447
+ if (!shouldSync(absPath, isDir)) {
448
+ if (!isDir)
449
+ outcome.ignoredFiles += 1;
450
+ continue;
451
+ }
452
+ const relative = relDir === "" ? dirent.name : `${relDir}/${dirent.name}`;
453
+ if (isDir) {
454
+ stack.push({ absDir: absPath, relDir: relative });
455
+ continue;
456
+ }
457
+ recordWalkedFile(outcome, collidedKeys, absPath, relative);
458
+ if (++sinceYield >= options.yieldEvery) {
459
+ sinceYield = 0;
460
+ await yieldToEventLoop();
461
+ if (options.abortSignal?.aborted || Date.now() > options.deadlineMs) {
462
+ outcome.truncated = true;
463
+ return finishWalk(outcome, collidedKeys);
464
+ }
465
+ }
466
+ }
467
+ }
468
+ return finishWalk(outcome, collidedKeys);
469
+ }
470
+ /**
471
+ * Drop every NFC key that two distinct raw filenames landed on. Done once at
472
+ * the end rather than inline, because the FIRST of a colliding pair is already
473
+ * in the map by the time the second is seen.
474
+ */
475
+ function finishWalk(outcome, collidedKeys) {
476
+ for (const [key, extras] of collidedKeys) {
477
+ if (outcome.records.delete(key))
478
+ outcome.filesWalked -= 1;
479
+ // Every raw spelling that collided is unrepresentable: the one that made it
480
+ // into the map, plus each duplicate that was turned away at the door.
481
+ outcome.unrepresentablePaths += extras + 1;
482
+ }
483
+ return outcome;
484
+ }
485
+ // ─── Entry assembly ──────────────────────────────────────────────────────────
486
+ /**
487
+ * True when the ledger's hash can stand in for reading the file.
488
+ *
489
+ * Both halves of the stat pair must agree. `size` alone misses a same-length
490
+ * rewrite; `mtimeMs` alone misses a `cp -p` / `touch -r` restore. The journal
491
+ * also records `ctimeMs` for its own push-skip gate, but the manifest is a
492
+ * REPORTING surface, not a transfer decision: an over-eager re-hash here costs
493
+ * one file read, while a wrong skip on the push path costs a lost upload — so
494
+ * this guard deliberately mirrors the contract's two reported fields rather
495
+ * than inventing a third.
496
+ */
497
+ function canReuseLedgerHash(entry, record) {
498
+ if (!entry || typeof entry.hash !== "string" || entry.hash.length === 0)
499
+ return false;
500
+ if (entry.size !== record.size)
501
+ return false;
502
+ if (typeof entry.mtimeMs !== "number")
503
+ return false;
504
+ return Math.floor(entry.mtimeMs) === record.mtimeMs;
505
+ }
506
+ function withinContractBounds(size, mtimeMs) {
507
+ return (Number.isSafeInteger(size) &&
508
+ size >= 0 &&
509
+ size <= SYNC_MANIFEST_MAX_FILE_SIZE_BYTES &&
510
+ Number.isSafeInteger(mtimeMs) &&
511
+ mtimeMs >= 0 &&
512
+ mtimeMs <= SYNC_MANIFEST_MAX_MTIME_MS);
513
+ }
514
+ function decorateWithLedger(entry, ledgerEntry) {
515
+ if (typeof ledgerEntry.remoteEtag === "string" && ledgerEntry.remoteEtag.length > 0) {
516
+ entry.ledgerRemoteEtag = ledgerEntry.remoteEtag;
517
+ }
518
+ if (typeof ledgerEntry.syncedAt === "string" && ledgerEntry.syncedAt.length > 0) {
519
+ entry.ledgerSyncedAt = ledgerEntry.syncedAt;
520
+ }
521
+ if (ledgerEntry.direction === "up" || ledgerEntry.direction === "down") {
522
+ entry.ledgerDirection = ledgerEntry.direction;
523
+ }
524
+ return entry;
525
+ }
526
+ /**
527
+ * Resolve where a scope lives on disk and which journal shard describes it.
528
+ *
529
+ * The personal vault's sync root IS the HQ root and its shard is the reserved
530
+ * {@link PERSONAL_VAULT_JOURNAL_SLUG} sentinel — never the literal `"personal"`
531
+ * slug, which belongs to the `companies/personal` COMPANY and whose collision
532
+ * once churned ~190 files per sync.
533
+ *
534
+ * The returned `wireScope` is rebuilt field by field rather than spread from
535
+ * the caller's object: that is what guarantees a personal manifest can never
536
+ * carry a `companyUid` (or a stray local `slug`) onto the wire.
537
+ */
538
+ function resolveScope(scope, hqRoot, teamSyncedSlugs) {
539
+ if (scope.kind === "company") {
540
+ if (!scope.companyUid)
541
+ throw new Error("buildSyncManifest: company scope requires companyUid");
542
+ if (!scope.slug)
543
+ throw new Error("buildSyncManifest: company scope requires slug");
544
+ // The slug names a DIRECTORY, and `path.join` will happily resolve
545
+ // `../personal` out of `companies/` entirely — which would walk one tenant's
546
+ // tree and upload it under another's `companyUid`. That is the exact
547
+ // boundary this whole feature is supposed to protect, so the slug is held
548
+ // to the same direct-child rule the reconciler already enforces (shared,
549
+ // not re-derived, so the two can never drift). Throwing rather than
550
+ // degrading is deliberate: only a caller bug can produce such a slug, and
551
+ // it belongs in the same class as "company scope without a uid".
552
+ if (!isSafeCompanySlug(path.join(hqRoot, "companies"), scope.slug)) {
553
+ throw new Error("buildSyncManifest: company scope slug must name a direct child of companies/");
554
+ }
555
+ const companyRoot = path.join(hqRoot, "companies", scope.slug);
556
+ return {
557
+ root: companyRoot,
558
+ walkRoots: [companyRoot],
559
+ journalSlug: scope.slug,
560
+ wireScope: { kind: "company", companyUid: scope.companyUid },
561
+ };
562
+ }
563
+ if (scope.companyUid) {
564
+ throw new Error("buildSyncManifest: personal scope must not carry a companyUid");
565
+ }
566
+ // NOT `[hqRoot]`. The personal manifest must describe exactly what the
567
+ // personal PUSH leg ships (see sync-runner-company.ts, which starts share()
568
+ // at these same paths), and that carve-out excludes `companies/` for every
569
+ // cloud-backed slug. Reporting the whole HQ root here would tell the server
570
+ // the personal vault holds every tenant's files — findings for objects the
571
+ // personal bucket was never meant to contain, on every single pass.
572
+ return {
573
+ root: hqRoot,
574
+ walkRoots: computePersonalVaultPaths(hqRoot, teamSyncedSlugs ? { teamSyncedSlugs } : {}),
575
+ journalSlug: PERSONAL_VAULT_JOURNAL_SLUG,
576
+ wireScope: { kind: "personal" },
577
+ };
578
+ }
579
+ // ─── Builder ─────────────────────────────────────────────────────────────────
580
+ /**
581
+ * Build the manifest chunks for one scope.
582
+ *
583
+ * Never throws for environmental reasons (unreadable journal, unreadable
584
+ * directory, undecodable filename, snapshot write failure) — those degrade to
585
+ * a narrower but honest manifest. It DOES throw for programming errors:
586
+ * a company scope without a uid or slug, or a personal scope carrying a
587
+ * company uid, are caller bugs that must fail loudly rather than upload a
588
+ * mislabelled tenant's file list.
589
+ */
590
+ export async function buildSyncManifest(options) {
591
+ const nowMs = options.now ?? Date.now();
592
+ const maxWallMs = options.maxWallMs ?? DEFAULT_MANIFEST_MAX_WALL_MS;
593
+ const yieldEvery = Math.max(1, options.yieldEvery ?? DEFAULT_MANIFEST_YIELD_EVERY);
594
+ // `never`, NOT `auto`. The daemon calls this after every sync cycle, and the
595
+ // old default read every file the journal could not cover — which on a vault
596
+ // whose journal predates hashing is every file. Callers that genuinely want
597
+ // content hashes ask for them.
598
+ const hashPolicy = options.hashPolicy ?? "never";
599
+ const maxHashedFiles = Math.max(0, options.maxHashedFiles ?? DEFAULT_MANIFEST_MAX_HASHED_FILES);
600
+ const maxHashMs = Math.max(0, options.maxHashMs ?? DEFAULT_MANIFEST_MAX_HASH_MS);
601
+ const { root, walkRoots, journalSlug, wireScope } = resolveScope(options.scope, options.hqRoot, options.teamSyncedSlugs);
602
+ const ledger = readLedger(journalSlug, options.listJournalsImpl);
603
+ const ignoreRules = compileIgnoreRules(options.hqRoot, wireScope.kind);
604
+ // The ignore filter is always anchored at the HQ ROOT, even for a company
605
+ // scope: its rules are expressed hq-root-relative, and every company root
606
+ // lives underneath. Re-anchoring it at the company dir would silently change
607
+ // which patterns match.
608
+ const baseFilter = createIgnoreFilter(options.hqRoot, {
609
+ memoLimit: options.ignoreMemoLimit ?? MANIFEST_IGNORE_MEMO_LIMIT,
610
+ });
611
+ const shouldSync = wireScope.kind === "personal"
612
+ ? wrapFilterWithPersonalVaultDefaults(baseFilter, options.hqRoot, () => {
613
+ // The out-of-policy telemetry event belongs to the sync runner; the
614
+ // manifest only needs the verdict, and the walker counts it as
615
+ // ignored like any other filtered file.
616
+ })
617
+ : baseFilter;
618
+ const walkStartedAt = Date.now();
619
+ // ONE deadline for the whole build, not one per phase: the caller asked for
620
+ // "spend at most `maxWallMs` on this scope", and the hash phase below is as
621
+ // expensive as the walk (it opens files), so a per-phase cap would let the
622
+ // build run for twice the budget.
623
+ const deadlineMs = walkStartedAt + maxWallMs;
624
+ const walk = await walkScope(root, walkRoots, shouldSync, {
625
+ deadlineMs,
626
+ yieldEvery,
627
+ abortSignal: options.abortSignal,
628
+ });
629
+ const walkMs = Date.now() - walkStartedAt;
630
+ const entries = [];
631
+ // `Object.create(null)`, not `{}`: `__proto__` is a perfectly legal filename
632
+ // (and a legal contract path), and on a plain object literal that key would
633
+ // silently fail to become an own property — the file would vanish from the
634
+ // snapshot and then read as REMOVED on the next delta.
635
+ const snapshotEntries = Object.create(null);
636
+ let hashedFiles = 0;
637
+ let skippedUnhashed = 0;
638
+ let unhashedFiles = 0;
639
+ let droppedStatRaces = 0;
640
+ // Wall clock spent READING files, tracked separately from the walk's own
641
+ // deadline so the `auto` cap bounds the expensive half on its own terms.
642
+ const hashPhaseStartedAt = Date.now();
643
+ // Paths the walk SAW on disk that did not make it into `entries` — an
644
+ // unreadable file, or a post-hash re-stat that caught a writer mid-flight.
645
+ // They are absent from this manifest but emphatically NOT deleted, and the
646
+ // delta below must not report them as removed (see `removed`).
647
+ const seenButDropped = new Set();
648
+ // The hash phase obeys the same cooperative contract as the walk. Without
649
+ // this, a first-ever sync — where the ledger has no hashes and every file
650
+ // must be READ — blocks the event loop for the entire run, ignores
651
+ // `abortSignal`, and overruns `maxWallMs` by an unbounded margin.
652
+ let sinceYield = 0;
653
+ for (const [manifestPath, record] of walk.records) {
654
+ if (++sinceYield >= yieldEvery) {
655
+ sinceYield = 0;
656
+ await yieldToEventLoop();
657
+ }
658
+ if (options.abortSignal?.aborted || Date.now() > deadlineMs) {
659
+ // Same honesty rule as the walk: a partial hash phase is a partial
660
+ // picture, so mark it truncated. That suppresses `removedPaths`, the
661
+ // delta base, and the snapshot write further down.
662
+ walk.truncated = true;
663
+ break;
664
+ }
665
+ const ledgerEntry = ledger.entries.get(manifestPath);
666
+ // Filesystem IO uses the RAW spelling (see DiskRecord.raw): the NFC form is
667
+ // the wire key, and on a normalization-sensitive filesystem it may not name
668
+ // any file at all.
669
+ const absPath = path.join(root, record.raw ?? manifestPath);
670
+ let hash;
671
+ // May this pass afford to READ this file? `full` always; `auto` until it
672
+ // has spent its per-pass budget; `never` / `ledger-only` never.
673
+ const mayHash = hashPolicy === "full" ||
674
+ (hashPolicy === "auto" &&
675
+ hashedFiles < maxHashedFiles &&
676
+ Date.now() - hashPhaseStartedAt < maxHashMs);
677
+ if (canReuseLedgerHash(ledgerEntry, record)) {
678
+ hash = ledgerEntry?.hash;
679
+ }
680
+ else if (mayHash) {
681
+ try {
682
+ hash = record.isSymlink
683
+ ? hashSymlinkTarget(fs.readlinkSync(absPath))
684
+ : hashFile(absPath);
685
+ hashedFiles += 1;
686
+ }
687
+ catch {
688
+ // Unreadable or raced away; better absent than wrong.
689
+ seenButDropped.add(manifestPath);
690
+ continue;
691
+ }
692
+ // Re-stat AFTER hashing. A file written while we read it yields a hash
693
+ // that matches neither the before nor the after content — publishing it
694
+ // would report a phantom divergence against the vault. Drop the entry
695
+ // and let the next cycle, when the writer has finished, report it.
696
+ try {
697
+ const after = fs.lstatSync(absPath);
698
+ if (after.size !== record.size || Math.floor(after.mtimeMs) !== record.mtimeMs) {
699
+ droppedStatRaces += 1;
700
+ seenButDropped.add(manifestPath);
701
+ continue;
702
+ }
703
+ }
704
+ catch {
705
+ droppedStatRaces += 1;
706
+ seenButDropped.add(manifestPath);
707
+ continue;
708
+ }
709
+ }
710
+ else if (hashPolicy === "ledger-only") {
711
+ // The benchmark seam, and the ONLY policy that still drops. It is not
712
+ // reachable from any production caller.
713
+ skippedUnhashed += 1;
714
+ seenButDropped.add(manifestPath);
715
+ continue;
716
+ }
717
+ else {
718
+ // Stat-only: the entry ships with no hash at all. This is the important
719
+ // difference from dropping it — the server learns the path is PRESENT
720
+ // (with its size and mtime), it just cannot verify the content this
721
+ // pass. A dropped entry would read as a file the client does not hold,
722
+ // which is precisely the false finding this audit exists to avoid.
723
+ unhashedFiles += 1;
724
+ }
725
+ if (!withinContractBounds(record.size, record.mtimeMs)) {
726
+ skippedUnhashed += 1;
727
+ seenButDropped.add(manifestPath);
728
+ continue;
729
+ }
730
+ const entry = {
731
+ path: manifestPath,
732
+ ...(hash ? { hash } : {}),
733
+ size: record.size,
734
+ mtimeMs: record.mtimeMs,
735
+ // `ledgerUnavailable` means we cannot claim the ledger agrees, so every
736
+ // entry reports as disk-only regardless of what a stale map might say.
737
+ source: ledgerEntry && !ledger.unavailableErrorClass ? "both" : "disk",
738
+ };
739
+ if (ledgerEntry && !ledger.unavailableErrorClass) {
740
+ decorateWithLedger(entry, ledgerEntry);
741
+ // Only claimable when we HAVE a hash. With none, "the ledger's hash
742
+ // differs from ours" is not a statement we can make — and asserting
743
+ // `pendingPush` off an absent hash would flag every file on a stat-only
744
+ // pass as owing an upload.
745
+ if (hash !== undefined && ledgerEntry.hash !== hash)
746
+ entry.pendingPush = true;
747
+ }
748
+ entries.push(entry);
749
+ // The delta snapshot needs a hash to compare against. An unhashed entry has
750
+ // nothing to store, and inventing a placeholder would make the next delta
751
+ // report a change that did not happen — so it is simply left out, and the
752
+ // next pass treats it as new (an ADD, never a removal).
753
+ if (hash !== undefined) {
754
+ snapshotEntries[manifestPath] = { hash, size: record.size, mtimeMs: record.mtimeMs };
755
+ }
756
+ }
757
+ /**
758
+ * True when a path is one the walk could not have observed even if it exists
759
+ * — so its absence from `walk.records` says nothing about the disk.
760
+ *
761
+ * Two causes, both of which previously read as "the local copy vanished":
762
+ *
763
+ * 1. It sits under a directory the walk could not `readdir`. Invisible, not
764
+ * gone.
765
+ * 2. It now matches an ignore rule. A path that became ignored since it was
766
+ * last synced is simply not our business any more; reporting it as
767
+ * missing turns every new `.hqignore` line into a page of false drift.
768
+ *
769
+ * The ignore filter is consulted DIRECTLY rather than inferred from the
770
+ * walk's observations, because the walk never visits an ignored path and so
771
+ * has nothing to report about it. This is off the hot path: it runs once per
772
+ * ledger row that is already absent from disk, never per walked file.
773
+ */
774
+ // Scope-relative, forward-slash prefixes of the subtrees this walk actually
775
+ // enumerated. `[""]` for a company scope (one root, the scope anchor).
776
+ const walkRootPrefixes = walkRoots.map((walkRoot) => {
777
+ const rel = path.relative(root, path.resolve(walkRoot));
778
+ if (rel === "")
779
+ return "";
780
+ return rel.split(path.sep).join("/");
781
+ });
782
+ const isOutsideOurView = (candidatePath) => {
783
+ for (const prefix of walk.lostVisibilityPrefixes) {
784
+ if (prefix === "" || candidatePath.startsWith(prefix))
785
+ return true;
786
+ }
787
+ // THIRD cause, introduced with the personal-vault carve-out: the path is
788
+ // not under any walk root at all. A ledger row under a cloud-backed
789
+ // company directory is a row the PERSONAL leg no longer ships (that
790
+ // company was designated and moved to its own bucket) — reporting it as a
791
+ // vanished local file would manufacture one finding per file of every
792
+ // designated company, forever.
793
+ const insideAWalkRoot = walkRootPrefixes.some((prefix) => prefix === "" ||
794
+ candidatePath === prefix ||
795
+ candidatePath.startsWith(`${prefix}/`));
796
+ if (!insideAWalkRoot)
797
+ return true;
798
+ return !shouldSync(path.join(root, candidatePath), false);
799
+ };
800
+ // Ledger rows with no file on disk. These are the point of the exercise: a
801
+ // vanished local file is undetectable from the server alone. Skipped when
802
+ // the walk truncated — "not reached" is not "not there".
803
+ if (!ledger.unavailableErrorClass && !walk.truncated) {
804
+ for (const [ledgerPath, ledgerEntry] of ledger.entries) {
805
+ if (walk.records.has(ledgerPath))
806
+ continue;
807
+ if (ledgerPath.startsWith(SESSIONS_SCOPE_PREFIX))
808
+ continue;
809
+ if (isOutsideOurView(ledgerPath))
810
+ continue;
811
+ if (toManifestPath(ledgerPath) === null) {
812
+ walk.unrepresentablePaths += 1;
813
+ continue;
814
+ }
815
+ const mtimeMs = Math.floor(ledgerEntry.mtimeMs ?? 0);
816
+ if (typeof ledgerEntry.hash !== "string" || !withinContractBounds(ledgerEntry.size, mtimeMs)) {
817
+ continue;
818
+ }
819
+ const entry = {
820
+ path: ledgerPath,
821
+ hash: ledgerEntry.hash,
822
+ size: ledgerEntry.size,
823
+ mtimeMs,
824
+ source: "ledger",
825
+ };
826
+ decorateWithLedger(entry, ledgerEntry);
827
+ entries.push(entry);
828
+ // `ledgerEntry.hash`, not `entry.hash`: the entry's is optional now, and
829
+ // the guard above has already proved this one is a string.
830
+ snapshotEntries[ledgerPath] = { hash: ledgerEntry.hash, size: entry.size, mtimeMs };
831
+ }
832
+ }
833
+ const walkStats = {
834
+ filesWalked: walk.filesWalked,
835
+ walkMs,
836
+ hashedFiles,
837
+ ignoredFiles: walk.ignoredFiles,
838
+ unhashedFiles,
839
+ truncated: walk.truncated,
840
+ unreadableDirs: walk.lostVisibilityPrefixes.length,
841
+ };
842
+ const snapshot = {
843
+ snapshotId: newManifestSnapshotId(),
844
+ generatedAt: new Date(nowMs).toISOString(),
845
+ sequence: options.sequence,
846
+ entries: snapshotEntries,
847
+ };
848
+ const scopeKey = manifestSnapshotScopeKey(wireScope);
849
+ const previous = options.forceFull ? null : readManifestSnapshot(options.stateDir, scopeKey);
850
+ const usableBase = previous &&
851
+ isManifestSnapshotBaseUsable(previous) &&
852
+ !isManifestSnapshotStale(previous, nowMs) &&
853
+ !walk.truncated
854
+ ? previous
855
+ : null;
856
+ let mode = "full";
857
+ let wireEntries = entries;
858
+ let removedPaths = [];
859
+ if (usableBase) {
860
+ // Carry a hash FORWARD for any entry this pass could not hash but whose
861
+ // stat pair is unchanged since the base. Without this the base ERODES: a
862
+ // stat-only pass would drop every unhashed path from the snapshot it
863
+ // persists, so the pass after it would find no `before` for them and
864
+ // re-send the lot as changed — a delta that is a full manifest wearing a
865
+ // hat. (`snapshotEntries` is the same object `snapshot.entries` holds, so
866
+ // this lands in the persisted record.)
867
+ for (const entry of entries) {
868
+ if (entry.hash !== undefined)
869
+ continue;
870
+ const before = usableBase.entries[entry.path];
871
+ if (!before)
872
+ continue;
873
+ if (before.size !== entry.size || before.mtimeMs !== entry.mtimeMs)
874
+ continue;
875
+ snapshotEntries[entry.path] = before;
876
+ }
877
+ const changed = entries.filter((entry) => {
878
+ const before = usableBase.entries[entry.path];
879
+ if (!before)
880
+ return true;
881
+ if (before.size !== entry.size || before.mtimeMs !== entry.mtimeMs)
882
+ return true;
883
+ // No hash this pass (stat-only): the stat pair is all we have, and it
884
+ // agrees. Comparing `undefined !== before.hash` here would mark EVERY
885
+ // file changed on every stat-only pass, which turns a delta back into a
886
+ // full re-send and defeats the point of having a base at all.
887
+ if (entry.hash === undefined)
888
+ return false;
889
+ return before.hash !== entry.hash;
890
+ });
891
+ // A loop, not `new Set(entries.map(...))`: the `map` materialises a second
892
+ // array of every path string purely to be consumed once and discarded, and
893
+ // on a 199k-entry scope that transient array lands at the pass's peak.
894
+ const present = new Set();
895
+ for (const entry of entries)
896
+ present.add(entry.path);
897
+ const removed = Object.keys(usableBase.entries).filter((candidate) => {
898
+ if (present.has(candidate))
899
+ return false;
900
+ // Walked, but dropped before it could be reported. It exists.
901
+ if (seenButDropped.has(candidate))
902
+ return false;
903
+ // With the ledger unreadable the `ledger`-source loop above never ran,
904
+ // so every ledger-only path in the base is missing from `entries` for a
905
+ // reason that has nothing to do with deletion. We cannot tell the two
906
+ // apart this cycle, so we assert nothing rather than assert wrongly.
907
+ if (ledger.unavailableErrorClass)
908
+ return false;
909
+ // Unreadable subtree or newly-ignored path: absent from `entries` for a
910
+ // reason that is not deletion. Same rule as the ledger loop above.
911
+ if (isOutsideOurView(candidate))
912
+ return false;
913
+ return true;
914
+ });
915
+ // An enormous removal set usually means the tree moved out from under us
916
+ // (an unmounted volume, a relocated HQ root), not that the user deleted
917
+ // 50k files. Re-baselining with a FULL manifest states what we can see
918
+ // instead of asserting a deletion we are not confident about.
919
+ if (removed.length <= SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK) {
920
+ mode = "delta";
921
+ wireEntries = changed;
922
+ removedPaths = removed;
923
+ }
924
+ }
925
+ const chunkInput = {
926
+ entries: wireEntries,
927
+ removedPaths,
928
+ walkStats,
929
+ ignoreRules,
930
+ mode,
931
+ baseSnapshotId: mode === "delta" ? usableBase?.snapshotId : undefined,
932
+ generatedAt: snapshot.generatedAt,
933
+ installationId: options.installationId,
934
+ machineId: options.machineId,
935
+ scope: wireScope,
936
+ source: options.source,
937
+ sequence: options.sequence,
938
+ maxEntriesPerChunk: options.maxEntriesPerChunk,
939
+ ledgerUnavailable: ledger.unavailableErrorClass
940
+ ? { errorClass: ledger.unavailableErrorClass }
941
+ : undefined,
942
+ };
943
+ const chunkPlan = planManifestChunks(chunkInput);
944
+ // A truncated walk saw only part of the tree. Persisting it as the delta
945
+ // base would make every unseen file look REMOVED on the next cycle.
946
+ if (!walk.truncated && options.persistSnapshot !== false) {
947
+ writeManifestSnapshot(options.stateDir, scopeKey, snapshot);
948
+ }
949
+ return {
950
+ // Lazy on purpose — see the doc on `BuildSyncManifestResult.chunks`. The
951
+ // upload pass reads `chunkPlan` and never touches this.
952
+ get chunks() {
953
+ return chunkManifest(chunkInput);
954
+ },
955
+ chunkPlan,
956
+ mode,
957
+ walkStats,
958
+ snapshot,
959
+ unrepresentablePaths: walk.unrepresentablePaths,
960
+ skippedUnhashed,
961
+ unhashedFiles,
962
+ droppedStatRaces,
963
+ };
964
+ }
965
+ /**
966
+ * Split entries into contract-sized chunks. ALWAYS emits at least one chunk:
967
+ * an empty scope is a real, meaningful observation ("this machine has nothing
968
+ * here"), and swallowing it would look identical to "the client never ran".
969
+ *
970
+ * `removedPaths` ride the FIRST chunk only, so the server never has to
971
+ * de-duplicate a removal repeated across chunks.
972
+ */
973
+ export function chunkManifest(input) {
974
+ const plan = planManifestChunks(input);
975
+ const uploads = [];
976
+ for (let chunkIndex = 0; chunkIndex < plan.chunkCount; chunkIndex++) {
977
+ uploads.push(plan.chunk(chunkIndex));
978
+ }
979
+ return uploads;
980
+ }
981
+ /**
982
+ * Plan the chunk set without building it.
983
+ *
984
+ * Everything a chunk header carries is decided HERE, once — `chunkCount`
985
+ * included — so every chunk the plan later mints is byte-identical in the
986
+ * fields the server hashes into the snapshotId. Only the `entries` window and
987
+ * `chunkIndex` differ between chunks, which is the whole reason a chunk can be
988
+ * built on demand and thrown away instead of being held for the pass.
989
+ *
990
+ * ALWAYS plans at least one chunk: an empty scope is a real, meaningful
991
+ * observation ("this machine has nothing here"), and swallowing it would look
992
+ * identical to "the client never ran".
993
+ */
994
+ export function planManifestChunks(input) {
995
+ // Clamped, never merely trusted: a caller-supplied size above the contract
996
+ // ceiling would build chunks the server rejects, and one below 1 would loop
997
+ // forever.
998
+ const perChunk = Math.min(SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK, Math.max(1, Math.floor(input.maxEntriesPerChunk ?? SYNC_MANIFEST_MAX_ENTRIES_PER_CHUNK)));
999
+ const chunkCount = Math.max(1, Math.ceil(input.entries.length / perChunk));
1000
+ // Counted in one pass here rather than by the upload guard, which would
1001
+ // otherwise have to re-walk every chunk it is trying to avoid materialising.
1002
+ let diskEntryCount = 0;
1003
+ for (const entry of input.entries)
1004
+ if (entry.source === "disk")
1005
+ diskEntryCount += 1;
1006
+ // `removedPaths` ride the FIRST chunk only, so the server never has to
1007
+ // de-duplicate a removal repeated across chunks. Bounded once, not per chunk.
1008
+ const firstChunkRemovedPaths = input.removedPaths.slice(0, SYNC_MANIFEST_MAX_REMOVED_PATHS_PER_CHUNK);
1009
+ return {
1010
+ chunkCount,
1011
+ entryCount: input.entries.length,
1012
+ diskEntryCount,
1013
+ mode: input.mode,
1014
+ chunk(chunkIndex) {
1015
+ if (!Number.isInteger(chunkIndex) || chunkIndex < 0 || chunkIndex >= chunkCount) {
1016
+ throw new RangeError(`manifest chunk index ${chunkIndex} is outside 0..${chunkCount - 1}`);
1017
+ }
1018
+ const start = chunkIndex * perChunk;
1019
+ const upload = {
1020
+ contractVersion: SYNC_MANIFEST_CONTRACT_VERSION,
1021
+ installationId: input.installationId,
1022
+ scope: input.scope,
1023
+ machineId: input.machineId,
1024
+ source: input.source,
1025
+ mode: input.mode,
1026
+ generatedAt: input.generatedAt,
1027
+ sequence: input.sequence,
1028
+ chunkIndex,
1029
+ chunkCount,
1030
+ entries: input.entries.slice(start, start + perChunk),
1031
+ removedPaths: chunkIndex === 0 ? firstChunkRemovedPaths : [],
1032
+ walkStats: input.walkStats,
1033
+ ignoreRules: input.ignoreRules,
1034
+ };
1035
+ if (input.baseSnapshotId)
1036
+ upload.baseSnapshotId = input.baseSnapshotId;
1037
+ if (input.ledgerUnavailable)
1038
+ upload.ledgerUnavailable = input.ledgerUnavailable;
1039
+ return upload;
1040
+ },
1041
+ };
1042
+ }
1043
+ //# sourceMappingURL=build-manifest.js.map