@vibe-agent-toolkit/utils 0.2.0-rc.2 → 0.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/compare-code-units.d.ts +28 -0
- package/dist/compare-code-units.d.ts.map +1 -0
- package/dist/compare-code-units.js +33 -0
- package/dist/compare-code-units.js.map +1 -0
- package/dist/crawl-timing.d.ts +680 -0
- package/dist/crawl-timing.d.ts.map +1 -0
- package/dist/crawl-timing.js +809 -0
- package/dist/crawl-timing.js.map +1 -0
- package/dist/file-crawler.d.ts +22 -0
- package/dist/file-crawler.d.ts.map +1 -1
- package/dist/file-crawler.js +31 -7
- package/dist/file-crawler.js.map +1 -1
- package/dist/fs-utils.d.ts +17 -0
- package/dist/fs-utils.d.ts.map +1 -1
- package/dist/fs-utils.js +69 -0
- package/dist/fs-utils.js.map +1 -1
- package/dist/fs.d.ts +3 -1
- package/dist/fs.d.ts.map +1 -1
- package/dist/fs.js +5 -1
- package/dist/fs.js.map +1 -1
- package/dist/git-run.d.ts +167 -0
- package/dist/git-run.d.ts.map +1 -0
- package/dist/git-run.js +154 -0
- package/dist/git-run.js.map +1 -0
- package/dist/git-snapshot.d.ts +181 -0
- package/dist/git-snapshot.d.ts.map +1 -0
- package/dist/git-snapshot.js +232 -0
- package/dist/git-snapshot.js.map +1 -0
- package/dist/git-tracker.d.ts +94 -1
- package/dist/git-tracker.d.ts.map +1 -1
- package/dist/git-tracker.js +135 -3
- package/dist/git-tracker.js.map +1 -1
- package/dist/git-utils.d.ts +38 -0
- package/dist/git-utils.d.ts.map +1 -1
- package/dist/git-utils.js +103 -80
- package/dist/git-utils.js.map +1 -1
- package/dist/git.d.ts +2 -1
- package/dist/git.d.ts.map +1 -1
- package/dist/git.js +2 -1
- package/dist/git.js.map +1 -1
- package/dist/gitignore-checker.d.ts.map +1 -1
- package/dist/gitignore-checker.js +6 -3
- package/dist/gitignore-checker.js.map +1 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +42 -1
- package/dist/index.js.map +1 -1
- package/dist/link-auth/expand-macro.d.ts.map +1 -1
- package/dist/link-auth/expand-macro.js +7 -1
- package/dist/link-auth/expand-macro.js.map +1 -1
- package/dist/link-auth/resolve-token.d.ts.map +1 -1
- package/dist/link-auth/resolve-token.js +15 -1
- package/dist/link-auth/resolve-token.js.map +1 -1
- package/dist/path-core.d.ts +1 -1
- package/dist/path-core.js +1 -1
- package/dist/project-utils.d.ts.map +1 -1
- package/dist/project-utils.js +6 -3
- package/dist/project-utils.js.map +1 -1
- package/dist/safe-exec.d.ts +38 -10
- package/dist/safe-exec.d.ts.map +1 -1
- package/dist/safe-exec.js +88 -19
- package/dist/safe-exec.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/spawn-claude.d.ts +28 -0
- package/dist/skill-test/spawn-claude.d.ts.map +1 -1
- package/dist/skill-test/spawn-claude.js +51 -2
- package/dist/skill-test/spawn-claude.js.map +1 -1
- package/dist/skill-test/transcript.d.ts +41 -2
- package/dist/skill-test/transcript.d.ts.map +1 -1
- package/dist/skill-test/transcript.js +47 -3
- package/dist/skill-test/transcript.js.map +1 -1
- package/dist/test-helpers.d.ts +185 -13
- package/dist/test-helpers.d.ts.map +1 -1
- package/dist/test-helpers.js +274 -27
- package/dist/test-helpers.js.map +1 -1
- package/dist/text-content.d.ts +197 -0
- package/dist/text-content.d.ts.map +1 -0
- package/dist/text-content.js +348 -0
- package/dist/text-content.js.map +1 -0
- package/dist/text-file.d.ts +45 -0
- package/dist/text-file.d.ts.map +1 -0
- package/dist/text-file.js +53 -0
- package/dist/text-file.js.map +1 -0
- package/dist/text.d.ts +16 -0
- package/dist/text.d.ts.map +1 -0
- package/dist/text.js +16 -0
- package/dist/text.js.map +1 -0
- package/dist/timing-dump.d.ts +119 -0
- package/dist/timing-dump.d.ts.map +1 -0
- package/dist/timing-dump.js +164 -0
- package/dist/timing-dump.js.map +1 -0
- package/eslint/README.md +105 -3
- package/eslint/index.cjs +41 -0
- package/eslint/index.d.cts +7 -4
- package/eslint/rules/no-bare-symlink-in-tests.cjs +200 -0
- package/eslint/rules/no-raw-text-decode.cjs +296 -0
- package/eslint/rules/no-self-package-import.cjs +157 -0
- package/package.json +9 -2
|
@@ -0,0 +1,809 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timing accumulators for the work that *finds* documents, as opposed to the
|
|
3
|
+
* work that parses them.
|
|
4
|
+
*
|
|
5
|
+
* `parse-timing.ts` attributes time inside a parser. Its instrumentation points
|
|
6
|
+
* are exhaustively three files, and everything above them — the link walk, the
|
|
7
|
+
* gitignore oracle, the exclude cascade, the closure's reference resolution and
|
|
8
|
+
* its fixpoint iteration — is unattributed. That is not a gap in a report; it is
|
|
9
|
+
* the reason VAT cannot presently answer the one question that matters before
|
|
10
|
+
* either crawler is flipped onto a verb: **which of the two costs more to do its
|
|
11
|
+
* own work?**
|
|
12
|
+
*
|
|
13
|
+
* ## Why this is a keyed map and `parse-timing.ts` is a slot array
|
|
14
|
+
*
|
|
15
|
+
* The parse seam's axis is a CLOSED enum — a parser kind has the passes it has,
|
|
16
|
+
* they are declared in one array, and a `Float64Array` indexed by a compile-time
|
|
17
|
+
* constant is exactly right for a path taken 1,364+ times per command.
|
|
18
|
+
*
|
|
19
|
+
* This axis is not closed. `contributorId` is dynamic: a corpus declares its own
|
|
20
|
+
* extents, so on VAT's own tree there are 61 closure contributors whose ids come
|
|
21
|
+
* out of config, and the fixpoint `pass` is discovered at run time. A fixed-width
|
|
22
|
+
* slot array cannot carry either, and the honest answer is the one
|
|
23
|
+
* {@link ContributorTiming} already models — a keyed accumulator over
|
|
24
|
+
* `(contributorId, stratum, pass)`.
|
|
25
|
+
*
|
|
26
|
+
* The cost of the map is affordable *because* this path is cold relative to the
|
|
27
|
+
* parse path: one record per contributor invocation (66 contributors × 2 passes
|
|
28
|
+
* on VAT's own tree) plus one per walk and one per gitignore oracle read, against
|
|
29
|
+
* ~12,000 parser-pass records. The one genuinely hot site — the closure's
|
|
30
|
+
* per-reference resolution — is charged into a single pre-resolved key.
|
|
31
|
+
*
|
|
32
|
+
* ## Why this seam lives in `utils` and not in the package that owns a crawler
|
|
33
|
+
*
|
|
34
|
+
* Every id below names work in some *other* package — `resources` builds the
|
|
35
|
+
* registry, `agent-skills` walks the link graph, `claude-marketplace` enumerates
|
|
36
|
+
* an inventory. The seam has to sit underneath all of them, and `utils` is the
|
|
37
|
+
* only package that is underneath all of them.
|
|
38
|
+
*
|
|
39
|
+
* It shipped in `resources` and was moved here when {@link CRAWL_SHARED_GIT_TRACKER_ID}
|
|
40
|
+
* needed a bracket, because the code that row measures — `GitTracker.initialize`
|
|
41
|
+
* and its `git ls-files` spawn — is in *this* package, and `utils` may not import
|
|
42
|
+
* `resources`. The alternative was a bracket at each of the six call sites that
|
|
43
|
+
* construct a tracker, which is the arrangement the `ResourceRegistry` section
|
|
44
|
+
* below rejects for exactly the reason it gives; and it would have reached only
|
|
45
|
+
* five of them, because `@vibe-agent-toolkit/discovery` depends on `utils` alone
|
|
46
|
+
* and could not have filed a row at all.
|
|
47
|
+
*
|
|
48
|
+
* ⚠️ What that costs, stated rather than glossed: {@link CrawlStratum}'s `base`
|
|
49
|
+
* and `closure` are the merge driver's names, and the driver is two packages up.
|
|
50
|
+
* A module here now carries vocabulary from above it. That is a naming coupling
|
|
51
|
+
* and not a code one — nothing in this file or {@link timing-dump} imports
|
|
52
|
+
* anything but `node:` builtins and this package's own path helpers — but a
|
|
53
|
+
* reader looking for why `utils` knows the word "fixpoint" is owed the answer.
|
|
54
|
+
*
|
|
55
|
+
* ## What a `stratum` is here, and why the walker and the tracker get their own
|
|
56
|
+
*
|
|
57
|
+
* Two of the four come straight from the merge driver: `base` contributors run
|
|
58
|
+
* once, `closure` contributors iterate to a fixed point. `walkLinkGraph` is
|
|
59
|
+
* neither — it is not a projection contributor at all and the driver never sees
|
|
60
|
+
* it — so it records under `crawl` with a **synthetic contributor id**
|
|
61
|
+
* ({@link CRAWL_WALKER_ID}, {@link CRAWL_WALKER_GITIGNORE_ID}). That is stated
|
|
62
|
+
* here, and named in constants, rather than left to whatever string a call site
|
|
63
|
+
* happened to pass: a synthetic id that arrives by accident is indistinguishable
|
|
64
|
+
* in the dump from a real contributor, and the whole point of the dump is that
|
|
65
|
+
* the two crawlers are legible side by side.
|
|
66
|
+
*
|
|
67
|
+
* The fourth, `shared`, is for work **neither arm owns and both consume** — see
|
|
68
|
+
* its own section below. It exists because the only honest place to charge such
|
|
69
|
+
* work is a stratum that belongs to nobody: charging it to `crawl` would put
|
|
70
|
+
* shared preparation on the incumbent's total, which is the same class of defect
|
|
71
|
+
* as the double-count described further down, with the arms swapped.
|
|
72
|
+
*
|
|
73
|
+
* ## The two arms are bracketed at the same DEPTH, and that took a fix
|
|
74
|
+
*
|
|
75
|
+
* "Side by side" is a claim about depth, not just about presence. This seam
|
|
76
|
+
* shipped with the projection arm bracketed at its driver — `merge.ts` charges
|
|
77
|
+
* every `base` contributor, so the `base` stratum carries the projection's whole
|
|
78
|
+
* PREPARATION — while the incumbent arm was bracketed only at
|
|
79
|
+
* {@link CRAWL_WALKER_ID}, one `walkLinkGraph` call. But `walkLinkGraph` walks a
|
|
80
|
+
* `ResourceRegistry` somebody else already built, and building it is the crawl:
|
|
81
|
+
* `crawlDirectory` to enumerate, one read-parse-index per file to admit, then
|
|
82
|
+
* `resolveLinks` to wire the graph the walk then follows. None of that was
|
|
83
|
+
* charged anywhere. Measured on a real subject, the walker's traversal came in at
|
|
84
|
+
* **1.7 ms** against the projection's ~1,016 ms — and nothing in the output looked
|
|
85
|
+
* wrong, because both numbers were real and both arms reported. A ~600× ratio
|
|
86
|
+
* read off that dump would have been a comparison of a walk against a whole
|
|
87
|
+
* crawl.
|
|
88
|
+
*
|
|
89
|
+
* So the registry's own work is charged under `crawl` too
|
|
90
|
+
* ({@link CRAWL_REGISTRY_ENUMERATE_ID}, {@link CRAWL_REGISTRY_ADD_RESOURCE_ID},
|
|
91
|
+
* {@link CRAWL_REGISTRY_RESOLVE_LINKS_ID}), and the brackets live INSIDE
|
|
92
|
+
* `ResourceRegistry` rather than at the six sites that construct one. Six copies
|
|
93
|
+
* of the same bracket is six chances to disagree, and a seventh construction site
|
|
94
|
+
* added later would silently rot the gate — the one place all six converge is the
|
|
95
|
+
* class itself.
|
|
96
|
+
*
|
|
97
|
+
* ### How to total an arm from this dump
|
|
98
|
+
*
|
|
99
|
+
* Not every row is additive with every other, so the two totals a flip decision
|
|
100
|
+
* rests on are stated here rather than left to a reader's arithmetic:
|
|
101
|
+
*
|
|
102
|
+
* - **Incumbent arm** = the three `resource-registry:*` rows (mutually disjoint —
|
|
103
|
+
* enumeration, admission and link resolution do not contain one another) plus
|
|
104
|
+
* {@link CRAWL_WALKER_ID}. **Not** {@link CRAWL_WALKER_GITIGNORE_ID}, which is
|
|
105
|
+
* charged from inside the walk and is therefore already inside the walk's row.
|
|
106
|
+
* - **Projection arm** = the driver-placed rows in `base` and `closure`, i.e.
|
|
107
|
+
* every row at pass ≥ 1. The pass-0 rows in those strata
|
|
108
|
+
* ({@link CRAWL_CLOSURE_CONTRIBUTE_ID}, {@link CRAWL_CLOSURE_RESOLVE_ID}, and a
|
|
109
|
+
* registry build reached from inside a contributor) are breakdowns of that same
|
|
110
|
+
* time, not additions to it.
|
|
111
|
+
* - **Neither arm** = the `shared` stratum. It is part of what the COMMAND cost
|
|
112
|
+
* and no part of what either crawler cost, so it belongs in a command total and
|
|
113
|
+
* in neither side of the side-by-side. A reader who adds it to one arm has
|
|
114
|
+
* answered a different question than the one they asked.
|
|
115
|
+
*
|
|
116
|
+
* ⚠️ A rollup that sums a stratum's rows without regard to pass double-counts
|
|
117
|
+
* every nested bracket. That is a real reading hazard, not a hypothetical: it is
|
|
118
|
+
* what `packages/lab/src/facets/crawl/dump.ts` did until 2026-08-15, and it
|
|
119
|
+
* inflated the two arms by DIFFERENT factors, because they nest to different
|
|
120
|
+
* depths. That reader now implements the rule above — `crawlRowRole` there is
|
|
121
|
+
* the executable copy of it — so anyone adding a bracket to this seam should
|
|
122
|
+
* expect to place it there too, and will see it land in `unclassified` if they
|
|
123
|
+
* do not.
|
|
124
|
+
*
|
|
125
|
+
* ## A registry built from inside a contributor belongs to the PROJECTION arm
|
|
126
|
+
*
|
|
127
|
+
* Putting the bracket inside `ResourceRegistry` puts it under whoever calls it,
|
|
128
|
+
* and a projection contributor could call it. Nothing shipped does — no file
|
|
129
|
+
* under `src/projection/` imports the class; the base contributors reach for
|
|
130
|
+
* `crawlDirectory`, `GitTracker` and `node:fs` directly — but "nothing does yet"
|
|
131
|
+
* is not an accounting rule. If a contributor ever did, charging its registry
|
|
132
|
+
* build to `crawl` would move a whole crawl onto the incumbent's total on a run
|
|
133
|
+
* the incumbent took no part in: the same defect this section describes, with the
|
|
134
|
+
* arms swapped.
|
|
135
|
+
*
|
|
136
|
+
* So a registry bracket does not name its own stratum. It **inherits** the one
|
|
137
|
+
* the merge driver is running under ({@link withContributorStratum}, an
|
|
138
|
+
* `AsyncLocalStorage` so it survives the `await`s a contributor is full of and
|
|
139
|
+
* cannot be corrupted by a second population interleaving with the first), and
|
|
140
|
+
* falls back to `crawl` — the incumbent — when no contributor is on the stack.
|
|
141
|
+
* The row is then a pass-0 breakdown of the driver's own row for that
|
|
142
|
+
* contributor, exactly as {@link CRAWL_CLOSURE_CONTRIBUTE_ID} already is.
|
|
143
|
+
*
|
|
144
|
+
* **Failure mode of that choice, stated plainly:** the inherited row overlaps the
|
|
145
|
+
* driver's row for the same invocation, so an arm total that adds them
|
|
146
|
+
* double-counts. The alternative — dropping the bracket while inside a
|
|
147
|
+
* contributor — would have removed the overlap by making real work invisible,
|
|
148
|
+
* and an absent row is indistinguishable from code that never ran. Overlap that
|
|
149
|
+
* a reader can see and the totalling rule above resolves beats a silent hole.
|
|
150
|
+
*
|
|
151
|
+
* ## `shared` is for preparation that CANCELS, and cancelling is not free
|
|
152
|
+
*
|
|
153
|
+
* A `GitTracker` is built once and handed to whichever crawler runs: both arms
|
|
154
|
+
* take one as a caller option, so the `git ls-files` spawn behind it is charged
|
|
155
|
+
* to the same side whichever way a verb is flipped. It is therefore invisible to
|
|
156
|
+
* the arm COMPARISON by construction — and it was, for four commits, invisible
|
|
157
|
+
* to the dump as well, which is a different and worse thing.
|
|
158
|
+
*
|
|
159
|
+
* The distinction the `shared` stratum draws is between two questions a reader
|
|
160
|
+
* asks with the same words:
|
|
161
|
+
*
|
|
162
|
+
* - *"Which crawler costs more?"* — `shared` is irrelevant, and adding it to
|
|
163
|
+
* either arm makes the answer wrong.
|
|
164
|
+
* - *"What did this command spend finding documents?"* — `shared` is part of the
|
|
165
|
+
* answer, and omitting it makes THAT answer wrong. On an adopter monorepo one
|
|
166
|
+
* `GitTracker.initialize()` measured 147 ms; the whole incumbent crawl on VAT's
|
|
167
|
+
* own tree measures ~75 ms. A term that can be twice the total it is missing
|
|
168
|
+
* from is not a rounding error.
|
|
169
|
+
*
|
|
170
|
+
* Symmetric under-counting is still under-counting. The rule this seam keeps is
|
|
171
|
+
* that no measured work goes unrecorded, and where a row is placed is answered
|
|
172
|
+
* separately from whether it exists.
|
|
173
|
+
*
|
|
174
|
+
* ⚠️ `shared` is a FALLBACK, not a destination. {@link recordSharedPass} inherits
|
|
175
|
+
* {@link withContributorStratum} exactly as {@link recordRegistryPass} does, for
|
|
176
|
+
* the same reason: a projection contributor that built its own tracker would be
|
|
177
|
+
* paying for it out of its own time, and charging that to `shared` would move a
|
|
178
|
+
* cost off the arm that actually incurred it. Nothing shipped does — the base
|
|
179
|
+
* contributors are handed a tracker rather than building one — but "nothing does
|
|
180
|
+
* yet" is not an accounting rule, and it is the rule this whole file is about.
|
|
181
|
+
*
|
|
182
|
+
* ## `pass` 0 means "recorded from inside the work"
|
|
183
|
+
*
|
|
184
|
+
* The merge driver is the ONLY participant that knows which fixpoint pass is
|
|
185
|
+
* running; a contributor's own `contribute` does not, and neither does a link
|
|
186
|
+
* walk. So a bracket placed inside the measured code records
|
|
187
|
+
* {@link CRAWL_PASS_INSIDE} — a reserved 0 — and aggregates across every pass.
|
|
188
|
+
* A driver-placed record always carries a real pass number at or above 1. The
|
|
189
|
+
* two are therefore never silently summed into one row: they key differently,
|
|
190
|
+
* and a reader can tell a per-pass figure from an all-passes one by looking.
|
|
191
|
+
*
|
|
192
|
+
* ## Commensurability is the whole point, so both arms use one clock
|
|
193
|
+
*
|
|
194
|
+
* Every bracket in this seam — driver, closure, walker — is `performance.now()`,
|
|
195
|
+
* for the same reason `parse-timing.ts` uses it: it is a float and allocates
|
|
196
|
+
* nothing, where `process.hrtime.bigint()` allocates a BigInt per call. The merge
|
|
197
|
+
* driver's `ContributorTiming.elapsedMs` moved to the same clock when this seam
|
|
198
|
+
* landed; it was `Date.now()`, whose ~1ms granularity would have made a
|
|
199
|
+
* driver-level figure and a walker-level figure incomparable at exactly the
|
|
200
|
+
* resolution the comparison needs.
|
|
201
|
+
*
|
|
202
|
+
* ## What this dump deliberately does NOT do
|
|
203
|
+
*
|
|
204
|
+
* It carries the process's own wall and CPU lifetime, like the parse dump, and
|
|
205
|
+
* for the same reason: these brackets are wall-timed, so a reader has to be able
|
|
206
|
+
* to see that the process spent its life waiting. It does **not** invite that
|
|
207
|
+
* figure to be summed across processes. `parse-timing.ts`'s review finding of
|
|
208
|
+
* 2026-08-14 records that the lab sums `process.wallMs` across dumps, which
|
|
209
|
+
* double-counts real time under a multi-process verb because the parent
|
|
210
|
+
* orchestrator's lifetime contains every child's. The reader for THIS dump
|
|
211
|
+
* publishes one lifetime per process and never a total — see
|
|
212
|
+
* `packages/lab/src/facets/crawl/dump.ts`.
|
|
213
|
+
*
|
|
214
|
+
* ## Why the gate is read at module load
|
|
215
|
+
*
|
|
216
|
+
* Same reconciliation `parse-timing.ts` states: `process.env` access in Node is a
|
|
217
|
+
* native call, the gate sits on paths taken thousands of times per command, and
|
|
218
|
+
* the testability the per-construction rule protects is preserved by
|
|
219
|
+
* {@link __setCrawlTimingForTest} rather than by re-reading the environment.
|
|
220
|
+
* (`vitest.setup.js` deletes every `VAT_*` variable before any test module loads,
|
|
221
|
+
* so a test could not usefully set it anyway.)
|
|
222
|
+
*
|
|
223
|
+
* The env var's VALUE is the directory the dump is written to; its presence is
|
|
224
|
+
* what enables the seam. An empty-string value counts as absent.
|
|
225
|
+
*/
|
|
226
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
227
|
+
import { ensureTimingDirectory, normalizeTimingDirectory, readTimingProcess, writeTimingDump, } from './timing-dump.js';
|
|
228
|
+
/**
|
|
229
|
+
* Every stratum, in the order the dump and every report list them.
|
|
230
|
+
*
|
|
231
|
+
* `shared` is appended rather than slotted next to `crawl`, so that adding it did
|
|
232
|
+
* not reorder a single existing row. Dump ordering is what makes two captures of
|
|
233
|
+
* one run comparable line by line, and a reordering is indistinguishable from a
|
|
234
|
+
* measurement change to anything diffing the text.
|
|
235
|
+
*/
|
|
236
|
+
export const CRAWL_STRATA = ['base', 'closure', 'crawl', 'shared'];
|
|
237
|
+
/**
|
|
238
|
+
* The `pass` a bracket placed INSIDE the measured code records.
|
|
239
|
+
*
|
|
240
|
+
* Reserved, and never produced by the merge driver, which numbers its passes from
|
|
241
|
+
* 1. See this module's header: a contributor's own body does not know which
|
|
242
|
+
* fixpoint pass is running, so a row keyed here aggregates across all of them and
|
|
243
|
+
* says so by carrying a pass number no driver-placed row can carry.
|
|
244
|
+
*/
|
|
245
|
+
export const CRAWL_PASS_INSIDE = 0;
|
|
246
|
+
/** Synthetic contributor id for one whole `walkLinkGraph` call. */
|
|
247
|
+
export const CRAWL_WALKER_ID = 'walk-link-graph:walk';
|
|
248
|
+
/**
|
|
249
|
+
* Synthetic contributor id for the link walker's gitignore oracle.
|
|
250
|
+
*
|
|
251
|
+
* Charged on the MISS path only — `WalkState.gitignoreFacts` memoizes the answer
|
|
252
|
+
* within one walk, and a memo hit costs nothing worth a bracket. So `calls` here
|
|
253
|
+
* counts oracle READS (a `git check-ignore` spawn, or a `GitTracker` active-set
|
|
254
|
+
* lookup), not the number of times the cascade asked.
|
|
255
|
+
*/
|
|
256
|
+
export const CRAWL_WALKER_GITIGNORE_ID = 'walk-link-graph:gitignore';
|
|
257
|
+
/**
|
|
258
|
+
* Synthetic contributor id for one `ClosureExtentContributor.contribute` call,
|
|
259
|
+
* aggregated across every declared extent.
|
|
260
|
+
*
|
|
261
|
+
* Distinct from the driver's own `closure:<name>` rows, which are per extent and
|
|
262
|
+
* per fixpoint pass: this one brackets the same work from the inside, so the two
|
|
263
|
+
* together say how much of a contributor invocation is the contributor's body and
|
|
264
|
+
* how much is the driver's merge and digest around it.
|
|
265
|
+
*/
|
|
266
|
+
export const CRAWL_CLOSURE_CONTRIBUTE_ID = 'closure-extent:contribute';
|
|
267
|
+
/** Synthetic contributor id for the closure walk's per-reference resolution. */
|
|
268
|
+
export const CRAWL_CLOSURE_RESOLVE_ID = 'closure-extent:resolve-reference';
|
|
269
|
+
/**
|
|
270
|
+
* What every `ResourceRegistry` id starts with.
|
|
271
|
+
*
|
|
272
|
+
* Exported because "is this row registry preparation?" is a question a reader
|
|
273
|
+
* asks — the three phases are one accounting unit — and a caller answering it by
|
|
274
|
+
* restating the prefix would drift the moment a fourth phase is bracketed.
|
|
275
|
+
*/
|
|
276
|
+
export const CRAWL_REGISTRY_ID_PREFIX = 'resource-registry:';
|
|
277
|
+
/**
|
|
278
|
+
* Synthetic contributor id for the enumeration inside `ResourceRegistry.crawl` —
|
|
279
|
+
* the `crawlDirectory` call, and nothing that follows it.
|
|
280
|
+
*
|
|
281
|
+
* Only the enumeration, so that this row and
|
|
282
|
+
* {@link CRAWL_REGISTRY_ADD_RESOURCE_ID} are additive rather than nested:
|
|
283
|
+
* `crawl()` is enumeration THEN admission, and bracketing the whole method would
|
|
284
|
+
* have produced a row that contains the admission row.
|
|
285
|
+
*
|
|
286
|
+
* A caller that enumerates for itself and hands paths to `addResources` files no
|
|
287
|
+
* row from inside the class, because its `crawlDirectory` call is outside the
|
|
288
|
+
* registry and therefore outside this bracket. That is a property of the class,
|
|
289
|
+
* not a claim that such a route enumerated nothing, and it is pinned as such in
|
|
290
|
+
* `crawl-timing.test.ts`.
|
|
291
|
+
*
|
|
292
|
+
* One such route ships: the marketplace inventory's `crawlSkillLinkRegistry`,
|
|
293
|
+
* which is the registry `vat inventory` hands the incumbent walker. It brackets
|
|
294
|
+
* its own enumeration and files this same row — the same accounting unit, and the
|
|
295
|
+
* two can never both run for one registry, so they cannot double-charge. It has
|
|
296
|
+
* to, because that registry is built for the INCUMBENT and never for the
|
|
297
|
+
* projection: unbracketed, it is a one-sided under-count on exactly the arm the
|
|
298
|
+
* flip decision is taken against, which is worse than a symmetric one.
|
|
299
|
+
*/
|
|
300
|
+
export const CRAWL_REGISTRY_ENUMERATE_ID = 'resource-registry:enumerate';
|
|
301
|
+
/**
|
|
302
|
+
* Synthetic contributor id for one `ResourceRegistry.addResource` — the read, the
|
|
303
|
+
* content key, the parse, the stat, the checksum and the four index writes for
|
|
304
|
+
* one file.
|
|
305
|
+
*
|
|
306
|
+
* The per-file grain is deliberate. It is the only grain every construction route
|
|
307
|
+
* shares (`crawl` and a direct `addResources` both funnel through it), and it is
|
|
308
|
+
* the one that makes the row's ms/call comparable to a projection contributor's:
|
|
309
|
+
* this is what admitting a document costs the incumbent.
|
|
310
|
+
*
|
|
311
|
+
* Charged even when the admission FAILS — a duplicate-id drop and an unreadable
|
|
312
|
+
* file both cost the read and the parse before they are refused, and a seam that
|
|
313
|
+
* charged only successes would report a corpus of collisions as nearly free.
|
|
314
|
+
*/
|
|
315
|
+
export const CRAWL_REGISTRY_ADD_RESOURCE_ID = 'resource-registry:add-resource';
|
|
316
|
+
/** Synthetic contributor id for one whole `ResourceRegistry.resolveLinks` call. */
|
|
317
|
+
export const CRAWL_REGISTRY_RESOLVE_LINKS_ID = 'resource-registry:resolve-links';
|
|
318
|
+
/**
|
|
319
|
+
* Synthetic contributor id for one run of the merge driver's blob stage —
|
|
320
|
+
* `populateBlobs`, which reads and parses every path the base contributors keyed
|
|
321
|
+
* and derives the four blob-keyed tables from it.
|
|
322
|
+
*
|
|
323
|
+
* **This is the projection's analogue of {@link CRAWL_REGISTRY_ADD_RESOURCE_ID},
|
|
324
|
+
* and it went uncharged while that one was charged.** The asymmetry is why the
|
|
325
|
+
* bracket exists: the seam's whole purpose is "which of the two crawlers costs
|
|
326
|
+
* more to do its own work", and an omission on ONE arm biases exactly that
|
|
327
|
+
* comparison — unlike {@link CRAWL_SHARED_GIT_TRACKER_ID}, whose omission at
|
|
328
|
+
* least cancelled. It is latent only while `populate()` has no production caller;
|
|
329
|
+
* the increment that gives it one is the increment that would have read a
|
|
330
|
+
* projection total with its own parse stage missing from it.
|
|
331
|
+
*
|
|
332
|
+
* Charged in `base` at the driver's pass, NOT at {@link CRAWL_PASS_INSIDE}: the
|
|
333
|
+
* stage is placed BY the driver, between the strata, and a pass-0 row in a driver
|
|
334
|
+
* stratum means "a bracket inside a contributor invocation", which this is not.
|
|
335
|
+
* Pass >= 1 is what makes it additive, and additive is correct — nothing else
|
|
336
|
+
* brackets this time.
|
|
337
|
+
*
|
|
338
|
+
* Both driver-placed runs file this one row: the stage before the closure
|
|
339
|
+
* iterates, and the post-promotion run after it. Same accounting unit and the
|
|
340
|
+
* same argument {@link CRAWL_REGISTRY_ENUMERATE_ID} makes for its two routes, so
|
|
341
|
+
* `calls` reads as "how many times the stage ran" (1, or 2 when a closure
|
|
342
|
+
* contributor promoted a demand) and stays divisible.
|
|
343
|
+
*/
|
|
344
|
+
export const CRAWL_BLOB_POPULATE_ID = 'blob-population:derive';
|
|
345
|
+
/**
|
|
346
|
+
* Synthetic contributor id for the merge driver's attempt to answer a population
|
|
347
|
+
* from a {@link ProjectionStore} instead of deriving it.
|
|
348
|
+
*
|
|
349
|
+
* **A row here is the only way a dump can tell a cache HIT from a subject that
|
|
350
|
+
* exercised nothing.** Both look identical from the totals — a hit files no
|
|
351
|
+
* contributor rows at all, because no contributor ran — and "nothing ran" is
|
|
352
|
+
* precisely the reading that has already turned one A/B into a measurement of
|
|
353
|
+
* noise. This row says the lane was reached and what it cost to ask.
|
|
354
|
+
*
|
|
355
|
+
* Charged in `base` at the driver's pass for the same reason
|
|
356
|
+
* {@link CRAWL_BLOB_POPULATE_ID} is: the driver places it, and pass >= 1 is what
|
|
357
|
+
* makes it additive rather than a breakdown of a bracket that does not contain it.
|
|
358
|
+
*/
|
|
359
|
+
export const CRAWL_STORE_READ_ID = 'projection-store:read';
|
|
360
|
+
/**
|
|
361
|
+
* Synthetic contributor id for writing a freshly derived population back to a
|
|
362
|
+
* {@link ProjectionStore}.
|
|
363
|
+
*
|
|
364
|
+
* Filed only on a miss, which is what makes the pair readable: a run with a read
|
|
365
|
+
* row and no write row is a hit, a run with both is a miss that paid to populate
|
|
366
|
+
* and then paid to store, and a run with neither had no store at all.
|
|
367
|
+
*
|
|
368
|
+
* ⚠️ **Those three readings are not exhaustive.** A store that THROWS also files
|
|
369
|
+
* a read row and no write row, because the read is bracketed in a `finally` so
|
|
370
|
+
* that a hit — which runs no contributor and would otherwise leave no trace at
|
|
371
|
+
* all — is still visible. The error propagates and the run dies, so only a
|
|
372
|
+
* post-mortem reader of the dump can be misled by it; a live run cannot.
|
|
373
|
+
*/
|
|
374
|
+
export const CRAWL_STORE_WRITE_ID = 'projection-store:write';
|
|
375
|
+
/**
|
|
376
|
+
* Synthetic contributor id for one `GitTracker.initialize()` — the `git ls-files`
|
|
377
|
+
* spawn and the active-set, ancestor and index maps built from its output.
|
|
378
|
+
*
|
|
379
|
+
* The default stratum is `shared` because a tracker is preparation both crawlers
|
|
380
|
+
* consume and neither owns; see this module's `shared` section.
|
|
381
|
+
*
|
|
382
|
+
* **`calls` counts real initializations, not calls to the method.** `initialize`
|
|
383
|
+
* returns immediately once it has run, and a bracket around that early return
|
|
384
|
+
* would report a caller's re-entry as work. Same rule as
|
|
385
|
+
* {@link CRAWL_WALKER_GITIGNORE_ID}, which counts oracle reads rather than
|
|
386
|
+
* questions asked, and for the same reason: a `calls` column nobody can divide by
|
|
387
|
+
* is a column that misleads.
|
|
388
|
+
*
|
|
389
|
+
* ⚠️ **The `new GitTracker(...)` constructor is deliberately NOT charged.** It
|
|
390
|
+
* resolves one path and allocates four empty containers; a bracket around it
|
|
391
|
+
* would measure `performance.now()` twice and file the result as a finding. This
|
|
392
|
+
* row is named "initialize" rather than "build" so it does not imply otherwise.
|
|
393
|
+
* A tracker that is constructed and never initialized therefore files no row,
|
|
394
|
+
* which is correct — it also spawned nothing.
|
|
395
|
+
*/
|
|
396
|
+
export const CRAWL_SHARED_GIT_TRACKER_ID = 'git-tracker:initialize';
|
|
397
|
+
/**
|
|
398
|
+
* The stratum the merge driver is currently running a contributor under, or
|
|
399
|
+
* absent outside a contributor invocation.
|
|
400
|
+
*
|
|
401
|
+
* `AsyncLocalStorage` rather than a module-level variable because a contributor
|
|
402
|
+
* is a chain of `await`s: a plain flag set before the call and cleared after it
|
|
403
|
+
* would be observed by any other crawl that happened to resume on the event loop
|
|
404
|
+
* in between, and two populations in one process would corrupt each other's
|
|
405
|
+
* attribution. See this module's header for why the inheritance exists at all.
|
|
406
|
+
*/
|
|
407
|
+
const contributorStratum = new AsyncLocalStorage();
|
|
408
|
+
/**
|
|
409
|
+
* Every synthetic id this BUILD is able to charge, whether or not it did.
|
|
410
|
+
*
|
|
411
|
+
* ## Why a dump has to say this, and why the version number could not
|
|
412
|
+
*
|
|
413
|
+
* An absent row is ambiguous in the one way that matters to a comparison: a dump
|
|
414
|
+
* with no `git-tracker:initialize` row is either a build that has no such bracket
|
|
415
|
+
* or a build that has one and never initialized a tracker. **The entries cannot
|
|
416
|
+
* distinguish those, and the difference decides whether two dumps are comparable
|
|
417
|
+
* at all** — the first case means one arm's total is missing a term the other
|
|
418
|
+
* arm's total contains, which is a widening read as a movement; the second means
|
|
419
|
+
* the arms agree and the work genuinely did not happen.
|
|
420
|
+
*
|
|
421
|
+
* {@link CRAWL_SEAM_DUMP_VERSION} was the previous answer and it is a poor one.
|
|
422
|
+
* An integer says "different", never "different how", so the remedy for a real
|
|
423
|
+
* widening and for a typo'd field is the same blunt refusal — and, worse, it only
|
|
424
|
+
* fires if a human remembers to bump it. The `shared` stratum shipped without a
|
|
425
|
+
* bump on an argument that was correct about rows and wrong about totals; nothing
|
|
426
|
+
* mechanical caught that, because nothing mechanical could.
|
|
427
|
+
*
|
|
428
|
+
* This list is derived from the module's own constants and travels in the dump,
|
|
429
|
+
* so a reader diffs CAPABILITIES rather than comparing an opaque number, and a
|
|
430
|
+
* bracket added here is announced without anyone remembering anything.
|
|
431
|
+
*
|
|
432
|
+
* ⚠️ **Synthetic ids only.** A contributor's own id comes out of a corpus's
|
|
433
|
+
* config (`closure:<name>`) and is not a property of the build, so it cannot go
|
|
434
|
+
* here — its absence really does mean "that extent was not declared", which is a
|
|
435
|
+
* corpus difference and not a build one. The strata are declared alongside, in
|
|
436
|
+
* {@link CRAWL_STRATA}, for the same reason and with no such caveat.
|
|
437
|
+
*/
|
|
438
|
+
export const CRAWL_CHARGEABLE_IDS = [
|
|
439
|
+
CRAWL_BLOB_POPULATE_ID,
|
|
440
|
+
CRAWL_CLOSURE_CONTRIBUTE_ID,
|
|
441
|
+
CRAWL_CLOSURE_RESOLVE_ID,
|
|
442
|
+
CRAWL_REGISTRY_ADD_RESOURCE_ID,
|
|
443
|
+
CRAWL_REGISTRY_ENUMERATE_ID,
|
|
444
|
+
CRAWL_REGISTRY_RESOLVE_LINKS_ID,
|
|
445
|
+
CRAWL_SHARED_GIT_TRACKER_ID,
|
|
446
|
+
CRAWL_STORE_READ_ID,
|
|
447
|
+
CRAWL_STORE_WRITE_ID,
|
|
448
|
+
CRAWL_WALKER_GITIGNORE_ID,
|
|
449
|
+
CRAWL_WALKER_ID,
|
|
450
|
+
];
|
|
451
|
+
/**
|
|
452
|
+
* Bumped whenever the dump's layout — **or the meaning of a row already in it** —
|
|
453
|
+
* changes in a way a reader must notice.
|
|
454
|
+
*
|
|
455
|
+
* The meaning half is not pedantry. A reader that refuses an unknown layout but
|
|
456
|
+
* accepts a silently redefined row is worse than one that refuses both: it
|
|
457
|
+
* produces numbers, and nobody can state what they are of.
|
|
458
|
+
*
|
|
459
|
+
* 1 — first version.
|
|
460
|
+
* 2 — the `crawl` stratum gained the incumbent's PREPARATION
|
|
461
|
+
* (`resource-registry:*`). No field changed. What changed is what a `crawl`
|
|
462
|
+
* total is a total OF: traversal alone at v1, the registry build plus the
|
|
463
|
+
* traversal at v2. Holding a v1 dump against a v2 one reads that widening as
|
|
464
|
+
* a several-hundred-fold regression in the walker — see this module's header.
|
|
465
|
+
*
|
|
466
|
+
* 3 — the `shared` stratum, and the projection's blob stage
|
|
467
|
+
* ({@link CRAWL_BLOB_POPULATE_ID}). No field changed here either, and that
|
|
468
|
+
* is exactly why the first attempt at this entry argued no bump was needed:
|
|
469
|
+
* `shared` holds work previously charged NOWHERE, so nothing moved out of an
|
|
470
|
+
* existing row, and a reader predating it buckets the rows in
|
|
471
|
+
* `unclassified`. That argument was **right about rows and wrong about the
|
|
472
|
+
* dump**, because the rule above says "the meaning of a row" and the values a
|
|
473
|
+
* reader actually publishes are DERIVED:
|
|
474
|
+
*
|
|
475
|
+
* - a command TOTAL sums every additive row across every stratum, so it grew
|
|
476
|
+
* by the whole `git ls-files` spawn — 27% to 100% of the crawl budget
|
|
477
|
+
* depending on the corpus. An A/B across the boundary sees that as a real,
|
|
478
|
+
* and perfectly STABLE, regression: every pair says `changed` for the same
|
|
479
|
+
* reason, which reads as agreement rather than as the tool refusing.
|
|
480
|
+
* - `attribution` flips from `nothing-crawled` to `measured` for a command
|
|
481
|
+
* that reached no crawler at all, because one shared row is now present.
|
|
482
|
+
*
|
|
483
|
+
* Both are precisely the v1 -> v2 failure — a widening read as a movement —
|
|
484
|
+
* so both get the same remedy. A reader that refuses the dump and says so is
|
|
485
|
+
* the loud failure; a reader that publishes a confident false delta is the
|
|
486
|
+
* quiet one, and the quiet one is what shipped between these two versions.
|
|
487
|
+
* 4 — the dump gained {@link CrawlTimingDump.charges}, and this number stops
|
|
488
|
+
* being the mechanism. A layout change, so it costs one last bump; after it,
|
|
489
|
+
* a reader diffs what two builds can CHARGE instead of comparing an integer,
|
|
490
|
+
* and a widening announces itself without anyone remembering to bump
|
|
491
|
+
* anything. Read {@link CRAWL_CHARGEABLE_IDS} for why the integer could never
|
|
492
|
+
* have done that job — it says "different", never "different how", and the
|
|
493
|
+
* v3 entry above exists precisely because a human did not notice in time.
|
|
494
|
+
*
|
|
495
|
+
* ⚠️ Keep bumping this for LAYOUT changes; it is still the only thing that can
|
|
496
|
+
* refuse a dump whose fields moved. What it is no longer responsible for is
|
|
497
|
+
* meaning, which the dump now states for itself.
|
|
498
|
+
*/
|
|
499
|
+
export const CRAWL_SEAM_DUMP_VERSION = 4;
|
|
500
|
+
/**
|
|
501
|
+
* Alias kept for this module's own readability at the write site.
|
|
502
|
+
*
|
|
503
|
+
* ⚠️ The exported spelling above exists so the READER can pin itself against the
|
|
504
|
+
* writer. `@vibe-agent-toolkit/lab`'s `CRAWL_DUMP_VERSION` refuses any dump whose
|
|
505
|
+
* version it does not recognise, and the two used to be unrelated literals in
|
|
506
|
+
* two packages — drift was silent, and its symptom is not a subtly wrong number
|
|
507
|
+
* but **every dump getting refused**, which a reader would sooner blame on their
|
|
508
|
+
* own invocation than on a constant. Now the lab pins equality against this
|
|
509
|
+
* export, so a bump here that is not mirrored there fails a test instead.
|
|
510
|
+
*/
|
|
511
|
+
const DUMP_VERSION = CRAWL_SEAM_DUMP_VERSION;
|
|
512
|
+
/** Basename stem of a dump file; the pid (and any collision counter) follow. */
|
|
513
|
+
const DUMP_BASENAME = 'crawl-timing';
|
|
514
|
+
/** What this seam is called in a failure line. */
|
|
515
|
+
const DUMP_NOUN = 'crawl-timing';
|
|
516
|
+
/**
|
|
517
|
+
* Every row so far, keyed by `stratum|pass|contributorId`.
|
|
518
|
+
*
|
|
519
|
+
* The id goes LAST so the key needs no escaping: a stratum is one of three
|
|
520
|
+
* literals and a pass is a number, so neither can contain the separator, and a
|
|
521
|
+
* contributor id may then contain anything at all. (A `\0` separator would have
|
|
522
|
+
* worked too and been unreadable — a file holding one is binary to `grep`, which
|
|
523
|
+
* has cost this repo a confident zero more than once.)
|
|
524
|
+
*/
|
|
525
|
+
const entries = new Map();
|
|
526
|
+
/**
|
|
527
|
+
* Where dumps go, or `null` when the seam is off.
|
|
528
|
+
*
|
|
529
|
+
* Read ONCE, here, from `process.env` — see this module's header.
|
|
530
|
+
*/
|
|
531
|
+
let dumpDirectory = normalizeTimingDirectory(process.env['VAT_CRAWL_TIMING']);
|
|
532
|
+
/**
|
|
533
|
+
* The hot path's gate. A plain boolean rather than `dumpDirectory !== null` so
|
|
534
|
+
* every instrumented call site costs one predictable branch on a memory load.
|
|
535
|
+
*/
|
|
536
|
+
let timingEnabled = dumpDirectory !== null;
|
|
537
|
+
/**
|
|
538
|
+
* The accumulator key for one row.
|
|
539
|
+
*
|
|
540
|
+
* @param contributorId - A contributor's id or a synthetic one
|
|
541
|
+
* @param stratum - Which layer
|
|
542
|
+
* @param pass - The fixpoint pass, or {@link CRAWL_PASS_INSIDE}
|
|
543
|
+
* @returns The map key
|
|
544
|
+
*/
|
|
545
|
+
function keyOf(contributorId, stratum, pass) {
|
|
546
|
+
return `${stratum}|${String(pass)}|${contributorId}`;
|
|
547
|
+
}
|
|
548
|
+
/**
|
|
549
|
+
* Fold one measured invocation into its row.
|
|
550
|
+
*
|
|
551
|
+
* @param contributorId - A contributor's id or a synthetic one
|
|
552
|
+
* @param stratum - Which layer
|
|
553
|
+
* @param pass - The fixpoint pass, or {@link CRAWL_PASS_INSIDE}
|
|
554
|
+
* @param elapsedMs - Wall time this invocation took
|
|
555
|
+
*/
|
|
556
|
+
function addEntry(contributorId, stratum, pass, elapsedMs) {
|
|
557
|
+
// Demoted here rather than at each entry point, so both the bracketed
|
|
558
|
+
// (`recordCrawlPass`) and the pre-measured (`recordContributorInvocation`)
|
|
559
|
+
// routes inherit it — a contained row must not depend on which of the two the
|
|
560
|
+
// driver happened to use. See {@link withOuterBracket}.
|
|
561
|
+
const placed = insideOuterBracket.getStore() === true ? CRAWL_PASS_INSIDE : pass;
|
|
562
|
+
const key = keyOf(contributorId, stratum, placed);
|
|
563
|
+
const bucket = entries.get(key);
|
|
564
|
+
if (bucket === undefined) {
|
|
565
|
+
entries.set(key, { contributorId, stratum, pass: placed, calls: 1, elapsedMs });
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
bucket.calls += 1;
|
|
569
|
+
bucket.elapsedMs += elapsedMs;
|
|
570
|
+
}
|
|
571
|
+
/**
|
|
572
|
+
* Order the rows so two dumps of the same run list them identically.
|
|
573
|
+
*
|
|
574
|
+
* Stratum in declared order rather than alphabetically — `base` really does
|
|
575
|
+
* precede `closure`, and sorting by name would put the walker's `crawl` rows
|
|
576
|
+
* between them for no reason a reader could state.
|
|
577
|
+
*
|
|
578
|
+
* @param left - One row
|
|
579
|
+
* @param right - Another
|
|
580
|
+
* @returns Standard comparator ordering
|
|
581
|
+
*/
|
|
582
|
+
function compareEntries(left, right) {
|
|
583
|
+
const byStratum = CRAWL_STRATA.indexOf(left.stratum) - CRAWL_STRATA.indexOf(right.stratum);
|
|
584
|
+
if (byStratum !== 0)
|
|
585
|
+
return byStratum;
|
|
586
|
+
const byId = left.contributorId.localeCompare(right.contributorId);
|
|
587
|
+
if (byId !== 0)
|
|
588
|
+
return byId;
|
|
589
|
+
return left.pass - right.pass;
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Build the dump from the current accumulator state.
|
|
593
|
+
*
|
|
594
|
+
* @returns A snapshot of every row
|
|
595
|
+
*/
|
|
596
|
+
function buildDump() {
|
|
597
|
+
return {
|
|
598
|
+
dumpVersion: DUMP_VERSION,
|
|
599
|
+
pid: process.pid,
|
|
600
|
+
process: readTimingProcess(),
|
|
601
|
+
charges: { strata: [...CRAWL_STRATA], syntheticIds: [...CRAWL_CHARGEABLE_IDS] },
|
|
602
|
+
entries: [...entries.values()].sort(compareEntries).map((entry) => ({ ...entry })),
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Write the dump, if the seam is on.
|
|
607
|
+
*
|
|
608
|
+
* @returns The path written, or `null` when the seam is off or the write failed
|
|
609
|
+
*/
|
|
610
|
+
function writeDump() {
|
|
611
|
+
return writeTimingDump(DUMP_NOUN, dumpDirectory, DUMP_BASENAME, buildDump);
|
|
612
|
+
}
|
|
613
|
+
if (dumpDirectory !== null) {
|
|
614
|
+
ensureTimingDirectory(DUMP_NOUN, dumpDirectory);
|
|
615
|
+
// Registered ONLY when enabled: a disabled seam must not even add a listener.
|
|
616
|
+
process.on('exit', () => {
|
|
617
|
+
writeDump();
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Start a bracket.
|
|
622
|
+
*
|
|
623
|
+
* @returns `performance.now()` when the seam is on, `0` when it is off
|
|
624
|
+
*/
|
|
625
|
+
export function crawlTimingStart() {
|
|
626
|
+
return timingEnabled ? performance.now() : 0;
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Attribute elapsed time to a `(contributorId, stratum, pass)` row.
|
|
630
|
+
*
|
|
631
|
+
* @param contributorId - A contributor's id, or one of this module's synthetic ids
|
|
632
|
+
* @param stratum - Which layer the work belongs to
|
|
633
|
+
* @param pass - The fixpoint pass, or {@link CRAWL_PASS_INSIDE} from inside the work
|
|
634
|
+
* @param startedAt - The value {@link crawlTimingStart} returned
|
|
635
|
+
*/
|
|
636
|
+
export function recordCrawlPass(contributorId, stratum, pass, startedAt) {
|
|
637
|
+
if (!timingEnabled)
|
|
638
|
+
return;
|
|
639
|
+
addEntry(contributorId, stratum, pass, performance.now() - startedAt);
|
|
640
|
+
}
|
|
641
|
+
/**
|
|
642
|
+
* Attribute elapsed time to whichever arm invoked this work, falling back to a
|
|
643
|
+
* stratum the work belongs to when no arm did.
|
|
644
|
+
*
|
|
645
|
+
* **No `stratum` parameter from the CALL SITE, deliberately.** The measured code
|
|
646
|
+
* here — a registry build, a tracker initialization — does not know whether it is
|
|
647
|
+
* running for the incumbent walker or from inside a projection contributor, and a
|
|
648
|
+
* call site that names a stratum it cannot know is how the work of one arm ends up
|
|
649
|
+
* on the other's total. The answer comes from {@link withContributorStratum}
|
|
650
|
+
* instead.
|
|
651
|
+
*
|
|
652
|
+
* The `fallback` is what the work is when nobody claimed it, and it is per site
|
|
653
|
+
* rather than a constant: an unclaimed registry build was the incumbent preparing
|
|
654
|
+
* to walk (`crawl`), while an unclaimed tracker build was preparation for whoever
|
|
655
|
+
* runs (`shared`). One function with a parameter rather than two nearly identical
|
|
656
|
+
* ones — the branch is the only difference between them, and two copies would be
|
|
657
|
+
* two places for the inheritance rule to drift.
|
|
658
|
+
*
|
|
659
|
+
* @param contributorId - One of this module's synthetic ids
|
|
660
|
+
* @param startedAt - The value {@link crawlTimingStart} returned
|
|
661
|
+
* @param fallback - The stratum to charge when no contributor is on the stack
|
|
662
|
+
*/
|
|
663
|
+
function recordInheritedPass(contributorId, startedAt, fallback) {
|
|
664
|
+
const stratum = contributorStratum.getStore() ?? fallback;
|
|
665
|
+
addEntry(contributorId, stratum, CRAWL_PASS_INSIDE, performance.now() - startedAt);
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Attribute elapsed time to one of the `ResourceRegistry` phases, under whichever
|
|
669
|
+
* arm invoked it — the incumbent when none did.
|
|
670
|
+
*
|
|
671
|
+
* @param contributorId - One of this module's `resource-registry:` ids
|
|
672
|
+
* @param startedAt - The value {@link crawlTimingStart} returned
|
|
673
|
+
*/
|
|
674
|
+
export function recordRegistryPass(contributorId, startedAt) {
|
|
675
|
+
if (!timingEnabled)
|
|
676
|
+
return;
|
|
677
|
+
recordInheritedPass(contributorId, startedAt, 'crawl');
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Attribute elapsed time to preparation both arms consume, under whichever arm
|
|
681
|
+
* invoked it — `shared` when none did, which is the shipped case.
|
|
682
|
+
*
|
|
683
|
+
* See this module's `shared` section: the fallback is the point of this entry
|
|
684
|
+
* point, and the inheritance is what keeps it from becoming a place to hide a
|
|
685
|
+
* cost one arm really did incur.
|
|
686
|
+
*
|
|
687
|
+
* @param contributorId - One of this module's shared ids
|
|
688
|
+
* @param startedAt - The value {@link crawlTimingStart} returned
|
|
689
|
+
*/
|
|
690
|
+
export function recordSharedPass(contributorId, startedAt) {
|
|
691
|
+
if (!timingEnabled)
|
|
692
|
+
return;
|
|
693
|
+
recordInheritedPass(contributorId, startedAt, 'shared');
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Run one contributor invocation with its stratum on the async context, so any
|
|
697
|
+
* bracket reached from inside it is attributed to the projection arm.
|
|
698
|
+
*
|
|
699
|
+
* A pass-through when the seam is off: an `AsyncLocalStorage.run` per contributor
|
|
700
|
+
* is cheap, but the shipped default is "no instrumentation ran at all", and this
|
|
701
|
+
* keeps that literally true.
|
|
702
|
+
*
|
|
703
|
+
* @param stratum - The stratum the driver is running this contributor in
|
|
704
|
+
* @param run - The invocation
|
|
705
|
+
* @returns Whatever the invocation returns
|
|
706
|
+
*/
|
|
707
|
+
export function withContributorStratum(stratum, run) {
|
|
708
|
+
if (!timingEnabled)
|
|
709
|
+
return run();
|
|
710
|
+
return contributorStratum.run(stratum, run);
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Whether the work now running is already inside a top-level span this seam
|
|
714
|
+
* timed.
|
|
715
|
+
*
|
|
716
|
+
* `AsyncLocalStorage`, for {@link contributorStratum}'s reason: the work in
|
|
717
|
+
* between is full of `await`s.
|
|
718
|
+
*/
|
|
719
|
+
const insideOuterBracket = new AsyncLocalStorage();
|
|
720
|
+
/**
|
|
721
|
+
* Run work that is CONTAINED by a bracket the caller has already opened, so its
|
|
722
|
+
* own rows are recorded as nested rather than additive.
|
|
723
|
+
*
|
|
724
|
+
* ## The double count this closes
|
|
725
|
+
*
|
|
726
|
+
* {@link crawlRowRole}'s rule reads *"`pass >= 1` is additive, whatever the
|
|
727
|
+
* stratum — only the merge driver numbers passes, and nothing in a dump can
|
|
728
|
+
* contain a driver-placed row."* The second half was false in exactly one
|
|
729
|
+
* place: `ResourceRegistry.crawl` brackets its enumeration as
|
|
730
|
+
* {@link CRAWL_REGISTRY_ENUMERATE_ID}, and on the projection lane that
|
|
731
|
+
* enumeration IS a whole `populate()` — so every `base` row sat inside a
|
|
732
|
+
* top-level `crawl` row and both were added.
|
|
733
|
+
*
|
|
734
|
+
* Measured on this repository before the fix: `enumerate` 7,508.4 ms against
|
|
735
|
+
* `base` rows totalling 7,501.4 ms — the same work, twice, printed as
|
|
736
|
+
* `base 49.6% / crawl 50.0%`, which reads as an even split between two
|
|
737
|
+
* crawlers. `resource-registry.ts` documented the hazard in a comment
|
|
738
|
+
* (*"summing the two per-arm totals inflates the projection arm... corrupts the
|
|
739
|
+
* RATIO and not merely the total"*) and the instrument printed the sum anyway.
|
|
740
|
+
*
|
|
741
|
+
* ## Why demoting the pass is the whole fix
|
|
742
|
+
*
|
|
743
|
+
* A pass-0 row in a driver stratum is **already** classified as nested by the
|
|
744
|
+
* existing rule, and the renderer already prints nested time on its own line
|
|
745
|
+
* and marks the rows `⊂`. So the containment only ever needed to reach the row,
|
|
746
|
+
* and no dump-format change, no new role and no new render path is involved.
|
|
747
|
+
*
|
|
748
|
+
* The cost is that a contained contributor's fixpoint pass is no longer
|
|
749
|
+
* readable — a pass-0 row aggregates across passes. That is the correct trade:
|
|
750
|
+
* a pass number is a detail, and a total that double-counts is a wrong answer.
|
|
751
|
+
*
|
|
752
|
+
* @param run - The contained work
|
|
753
|
+
* @returns Whatever the work returns
|
|
754
|
+
*/
|
|
755
|
+
export function withOuterBracket(run) {
|
|
756
|
+
if (!timingEnabled)
|
|
757
|
+
return run();
|
|
758
|
+
return insideOuterBracket.run(true, run);
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* Attribute an already-measured invocation, as the merge driver reports it.
|
|
762
|
+
*
|
|
763
|
+
* A second entry point rather than a second clock: the driver has to build a
|
|
764
|
+
* `ContributorTiming` for its own `onContributorTiming` observer anyway, so it
|
|
765
|
+
* measures once and hands the same object to both. Bracketing it here as well
|
|
766
|
+
* would time the observer.
|
|
767
|
+
*
|
|
768
|
+
* @param timing - What one contributor invocation cost
|
|
769
|
+
*/
|
|
770
|
+
export function recordContributorInvocation(timing) {
|
|
771
|
+
if (!timingEnabled)
|
|
772
|
+
return;
|
|
773
|
+
addEntry(timing.contributorId, timing.stratum, timing.pass, timing.elapsedMs);
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* TEST ONLY. Turn the seam on (writing to `directory`) or off, and drop every
|
|
777
|
+
* accumulated row.
|
|
778
|
+
*
|
|
779
|
+
* Exists so tests never have to mutate the real `process.env` — the same
|
|
780
|
+
* justification `__setParseTimingForTest` states. It deliberately does NOT
|
|
781
|
+
* register an `exit` listener; a test drives the write itself via
|
|
782
|
+
* {@link __writeCrawlTimingDumpForTest}, so a test run never litters dumps.
|
|
783
|
+
*
|
|
784
|
+
* @param directory - Where {@link __writeCrawlTimingDumpForTest} writes, or `null` to disable
|
|
785
|
+
*/
|
|
786
|
+
export function __setCrawlTimingForTest(directory) {
|
|
787
|
+
dumpDirectory = normalizeTimingDirectory(directory ?? undefined);
|
|
788
|
+
timingEnabled = dumpDirectory !== null;
|
|
789
|
+
entries.clear();
|
|
790
|
+
if (dumpDirectory !== null)
|
|
791
|
+
ensureTimingDirectory(DUMP_NOUN, dumpDirectory);
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* TEST ONLY. Read the accumulators without writing anything.
|
|
795
|
+
*
|
|
796
|
+
* @returns The dump that would be written right now
|
|
797
|
+
*/
|
|
798
|
+
export function __readCrawlTimingSnapshot() {
|
|
799
|
+
return buildDump();
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* TEST ONLY. Write a dump now, exactly as the exit listener would.
|
|
803
|
+
*
|
|
804
|
+
* @returns The path written, or `null` when the seam is off or the write failed
|
|
805
|
+
*/
|
|
806
|
+
export function __writeCrawlTimingDumpForTest() {
|
|
807
|
+
return writeDump();
|
|
808
|
+
}
|
|
809
|
+
//# sourceMappingURL=crawl-timing.js.map
|