@mnemonik/shared 7.97.1 → 7.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ambientPointer.d.ts.map +1 -1
- package/dist/ambientPointer.js +2 -1
- package/dist/ambientPointer.js.map +1 -1
- package/dist/apiOrigin.d.ts +3 -0
- package/dist/apiOrigin.d.ts.map +1 -0
- package/dist/apiOrigin.js +5 -0
- package/dist/apiOrigin.js.map +1 -0
- package/dist/ast/astChunker.d.ts +12 -12
- package/dist/ast/astChunker.js +32 -32
- package/dist/ast/grammars.d.ts +9 -9
- package/dist/ast/grammars.js +15 -15
- package/dist/codeScanner.d.ts +56 -43
- package/dist/codeScanner.d.ts.map +1 -1
- package/dist/codeScanner.js +262 -143
- package/dist/codeScanner.js.map +1 -1
- package/dist/contextBudget.js +2 -2
- package/dist/contextBudget.js.map +1 -1
- package/dist/gitEditDetector.d.ts +6 -6
- package/dist/gitEditDetector.js +7 -7
- package/dist/hookRuntime.d.ts +35 -2
- package/dist/hookRuntime.d.ts.map +1 -1
- package/dist/hookRuntime.js +127 -33
- package/dist/hookRuntime.js.map +1 -1
- package/dist/hookTimeouts.d.ts +6 -6
- package/dist/hookTimeouts.js +6 -6
- package/dist/hostAdapter.d.ts +135 -0
- package/dist/hostAdapter.d.ts.map +1 -0
- package/dist/hostAdapter.js +323 -0
- package/dist/hostAdapter.js.map +1 -0
- package/dist/hostBinary.d.ts +29 -0
- package/dist/hostBinary.d.ts.map +1 -0
- package/dist/hostBinary.js +179 -0
- package/dist/hostBinary.js.map +1 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -1
- package/dist/index.js.map +1 -1
- package/dist/instructions.d.ts +26 -26
- package/dist/instructions.js +26 -26
- package/dist/jitDeliveryAck.d.ts.map +1 -1
- package/dist/jitDeliveryAck.js +2 -0
- package/dist/jitDeliveryAck.js.map +1 -1
- package/dist/projectIdentityFile.d.ts +25 -0
- package/dist/projectIdentityFile.d.ts.map +1 -0
- package/dist/projectIdentityFile.js +87 -0
- package/dist/projectIdentityFile.js.map +1 -0
- package/dist/projectSetupHandoff.d.ts +24 -0
- package/dist/projectSetupHandoff.d.ts.map +1 -0
- package/dist/projectSetupHandoff.js +249 -0
- package/dist/projectSetupHandoff.js.map +1 -0
- package/dist/protectedPaths.d.ts +9 -0
- package/dist/protectedPaths.d.ts.map +1 -0
- package/dist/protectedPaths.js +66 -0
- package/dist/protectedPaths.js.map +1 -0
- package/dist/readiness.d.ts +143 -0
- package/dist/readiness.d.ts.map +1 -0
- package/dist/readiness.js +304 -0
- package/dist/readiness.js.map +1 -0
- package/dist/repositoryFingerprint.d.ts +50 -0
- package/dist/repositoryFingerprint.d.ts.map +1 -0
- package/dist/repositoryFingerprint.js +160 -0
- package/dist/repositoryFingerprint.js.map +1 -0
- package/dist/repositoryRoot.d.ts +56 -0
- package/dist/repositoryRoot.d.ts.map +1 -0
- package/dist/repositoryRoot.js +226 -0
- package/dist/repositoryRoot.js.map +1 -0
- package/dist/runtimeReader.d.ts +102 -0
- package/dist/runtimeReader.d.ts.map +1 -0
- package/dist/runtimeReader.js +408 -0
- package/dist/runtimeReader.js.map +1 -0
- package/dist/runtimeSigners.d.ts +37 -0
- package/dist/runtimeSigners.d.ts.map +1 -0
- package/dist/runtimeSigners.js +507 -0
- package/dist/runtimeSigners.js.map +1 -0
- package/dist/scannerSupervisor.d.ts +35 -0
- package/dist/scannerSupervisor.d.ts.map +1 -0
- package/dist/scannerSupervisor.js +2 -0
- package/dist/scannerSupervisor.js.map +1 -0
- package/dist/secretPatterns.d.ts +9 -9
- package/dist/secretPatterns.d.ts.map +1 -1
- package/dist/secretPatterns.js +27 -25
- package/dist/secretPatterns.js.map +1 -1
- package/dist/testing/hostAdapterConformance.d.ts +17 -0
- package/dist/testing/hostAdapterConformance.d.ts.map +1 -0
- package/dist/testing/hostAdapterConformance.js +477 -0
- package/dist/testing/hostAdapterConformance.js.map +1 -0
- package/dist/usageGuide.d.ts +1 -1
- package/dist/usageGuide.d.ts.map +1 -1
- package/dist/usageGuide.js +4 -4
- package/package.json +7 -2
- package/queries/bash.tags.scm +1 -1
- package/queries/groovy.tags.scm +1 -1
- package/scripts/vendor-grammars.ts +13 -13
- package/src/ambientPointer.ts +2 -1
- package/src/apiOrigin.ts +4 -0
- package/src/ast/astChunker.ts +39 -39
- package/src/ast/grammars.ts +16 -16
- package/src/codeScanner.ts +274 -149
- package/src/contextBudget.ts +2 -2
- package/src/gitEditDetector.ts +7 -7
- package/src/hookRuntime.ts +162 -34
- package/src/hookTimeouts.ts +6 -6
- package/src/hostAdapter.ts +468 -0
- package/src/hostBinary.ts +205 -0
- package/src/index.ts +20 -0
- package/src/instructions.ts +26 -26
- package/src/jitDeliveryAck.ts +2 -0
- package/src/projectIdentityFile.ts +100 -0
- package/src/projectSetupHandoff.ts +275 -0
- package/src/protectedPaths.ts +88 -0
- package/src/readiness.ts +516 -0
- package/src/repositoryFingerprint.ts +199 -0
- package/src/repositoryRoot.ts +261 -0
- package/src/runtimeReader.ts +495 -0
- package/src/runtimeSigners.ts +577 -0
- package/src/scannerSupervisor.ts +26 -0
- package/src/secretPatterns.ts +27 -25
- package/src/testing/hostAdapterConformance.ts +538 -0
- package/src/usageGuide.ts +4 -4
- package/test-fixtures/identity/runner.d.mts +12 -0
- package/test-fixtures/identity/runner.mjs +282 -0
- package/tests/fixtures/repository-fingerprint-v1.golden.json +834 -0
- package/tests/projectIdentity.test.ts +56 -0
- package/tests/readiness.test.ts +158 -0
- package/tests/repositoryFingerprint.test.ts +26 -0
- package/tsconfig.test.json +9 -0
- package/wasm/bash.tags.scm +1 -1
- package/wasm/groovy.tags.scm +1 -1
- package/wasm/javascript.tags.scm +2 -2
- package/wasm/solidity.tags.scm +1 -1
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* packages to this workspace's devDependencies and npm refused the tree:
|
|
6
6
|
* tree-sitter-groovy declares a `peerOptional` on the NATIVE tree-sitter
|
|
7
7
|
* (^0.21.1), which this design deliberately never installs. Rather than force
|
|
8
|
-
* the resolution
|
|
9
|
-
*
|
|
8
|
+
* the resolution. These packages have no business in the dependency graph at
|
|
9
|
+
* all: we want two files out of each of them, at build time, on our
|
|
10
10
|
* machine. `npm pack` fetches a tarball and performs NO dependency resolution,
|
|
11
11
|
* so there is no peer graph to satisfy, `npm ci` stays fast for every developer
|
|
12
12
|
* and every CI run, and nobody downloads ~283 MB of grammars to build the repo.
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* What ships to a user: the extracted `.wasm` and `.tags.scm` files, committed
|
|
15
15
|
* here and published inside @mnemonik/shared. A user's install therefore pulls
|
|
16
16
|
* no grammar package, runs no postinstall, needs no compiler, and fetches
|
|
17
|
-
* nothing at runtime
|
|
17
|
+
* nothing at runtime - which is the whole reason this plan chose WebAssembly
|
|
18
18
|
* over the native bindings.
|
|
19
19
|
*
|
|
20
20
|
* Run: npx tsx packages/shared/scripts/vendor-grammars.ts [--check]
|
|
@@ -51,8 +51,8 @@ interface GrammarSpec {
|
|
|
51
51
|
pkg: string;
|
|
52
52
|
/**
|
|
53
53
|
* `.wasm` basename inside the tarball when it differs from the package's own
|
|
54
|
-
* name
|
|
55
|
-
* file with an
|
|
54
|
+
* name - tree-sitter-typescript ships two, and tree-sitter-c-sharp names its
|
|
55
|
+
* file with an `_` separator.
|
|
56
56
|
*/
|
|
57
57
|
wasmFile?: string;
|
|
58
58
|
/**
|
|
@@ -70,19 +70,19 @@ interface GrammarSpec {
|
|
|
70
70
|
* remove.
|
|
71
71
|
*
|
|
72
72
|
* Absent on purpose:
|
|
73
|
-
* julia, objc, haskell
|
|
73
|
+
* julia, objc, haskell - the three largest grammars that ALSO ship no tag
|
|
74
74
|
* query, so each costs both its size (6.1/5.2/3.7 MB) and a hand-written
|
|
75
75
|
* query. Dropped to keep the payload under 40 MB; still indexed via the
|
|
76
76
|
* allowlist, still chunked heuristically.
|
|
77
|
-
* dart
|
|
77
|
+
* dart - its bundled .wasm (package v1.0.0, 2023) fails Language.load under
|
|
78
78
|
* web-tree-sitter 0.26.11 with a dylink-metadata error: the artifact
|
|
79
79
|
* predates the current wasm ABI. Presence is not loadability, which is why
|
|
80
80
|
* --check verifies both. Flutter matters, so this is worth revisiting by
|
|
81
81
|
* compiling from source with tree-sitter-cli.
|
|
82
|
-
* kotlin, swift, lua, perl, zig
|
|
82
|
+
* kotlin, swift, lua, perl, zig - bundle no .wasm, so they need a build step
|
|
83
83
|
* with tree-sitter-cli. Deferred until the grammars above are proven; they
|
|
84
84
|
* keep the heuristic chunker meanwhile.
|
|
85
|
-
* markdown
|
|
85
|
+
* markdown - keeps chunkMarkdown. Headers already are a document's semantic
|
|
86
86
|
* unit; an AST would not improve on them.
|
|
87
87
|
*/
|
|
88
88
|
const GRAMMARS: GrammarSpec[] = [
|
|
@@ -124,7 +124,7 @@ const GRAMMARS: GrammarSpec[] = [
|
|
|
124
124
|
* Derived, not declared: the chains themselves live in the runtime registry
|
|
125
125
|
* (`src/ast/grammars.ts`), which is the module that has to compose them for real
|
|
126
126
|
* work. A second hand-maintained copy here could drift from the one the scanner
|
|
127
|
-
* uses, and a drifted chain does not fail
|
|
127
|
+
* uses, and a drifted chain does not fail - it silently under-matches, which is
|
|
128
128
|
* the exact failure this vendoring step exists to catch. See that module for why
|
|
129
129
|
* tag queries are deltas and what composing them is worth (948 TypeScript
|
|
130
130
|
* definitions against 6,471).
|
|
@@ -178,7 +178,7 @@ async function checkOnly(): Promise<Problem[]> {
|
|
|
178
178
|
if (!chain) {
|
|
179
179
|
problems.push({
|
|
180
180
|
id: spec.id,
|
|
181
|
-
detail: `no QUERY_CHAIN entry in src/ast/grammars.ts
|
|
181
|
+
detail: `no QUERY_CHAIN entry in src/ast/grammars.ts - the grammar would load and match nothing`,
|
|
182
182
|
});
|
|
183
183
|
continue;
|
|
184
184
|
}
|
|
@@ -201,7 +201,7 @@ async function checkOnly(): Promise<Problem[]> {
|
|
|
201
201
|
if (!captures.some((c) => c.startsWith('definition.'))) {
|
|
202
202
|
problems.push({
|
|
203
203
|
id: spec.id,
|
|
204
|
-
detail: `composed query compiles but captures no definition.*
|
|
204
|
+
detail: `composed query compiles but captures no definition.* - it would name nothing`,
|
|
205
205
|
});
|
|
206
206
|
}
|
|
207
207
|
} catch (err) {
|
|
@@ -271,7 +271,7 @@ function vendor(): Problem[] {
|
|
|
271
271
|
if (!existsSync(tagsSrc)) {
|
|
272
272
|
problems.push({
|
|
273
273
|
id: spec.id,
|
|
274
|
-
detail: `${spec.pkg} ships no queries/tags.scm
|
|
274
|
+
detail: `${spec.pkg} ships no queries/tags.scm - reclassify this entry as 'ours' and author one`,
|
|
275
275
|
});
|
|
276
276
|
} else {
|
|
277
277
|
copyFileSync(tagsSrc, join(WASM_DIR, `${spec.id}.tags.scm`));
|
package/src/ambientPointer.ts
CHANGED
|
@@ -5,7 +5,8 @@ const LEGACY_MEMORY_ID_SOURCE = `[0-9A-Za-z_-]{${AMBIENT_ID_PREFIX_LENGTH}}`;
|
|
|
5
5
|
|
|
6
6
|
/** Current full UUID pointers plus cached eight-character pointers from older hooks. */
|
|
7
7
|
export const AMBIENT_ITEM_POINTER_SOURCE =
|
|
8
|
-
|
|
8
|
+
`\\u2014\\s*memory_get\\s+(${MEMORY_UUID_SOURCE}|${LEGACY_MEMORY_ID_SOURCE})` +
|
|
9
|
+
'(?![0-9A-Za-z_-])';
|
|
9
10
|
|
|
10
11
|
export function ambientItemPointerRegex(flags = 'g'): RegExp {
|
|
11
12
|
return new RegExp(AMBIENT_ITEM_POINTER_SOURCE, flags);
|
package/src/apiOrigin.ts
ADDED
package/src/ast/astChunker.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* logging of file contents, one pure entry point (`chunkWithAst`).
|
|
4
4
|
*
|
|
5
5
|
* WHY THIS EXISTS. The heuristic chunker matches a regex whose name pattern
|
|
6
|
-
* requires a leading `function`/`class`/`const` keyword
|
|
6
|
+
* requires a leading `function`/`class`/`const` keyword - a keyword a method's
|
|
7
7
|
* first line never has. So it produces method-shaped chunks with no
|
|
8
8
|
* `symbolName`: 1,512 methods in this repo, unnameable and therefore unusable
|
|
9
9
|
* as citation anchors. Measured over the same 1,037 TypeScript files, the
|
|
@@ -13,15 +13,15 @@
|
|
|
13
13
|
* THE NESTING RULE. Emitting both a whole class and each of its methods would
|
|
14
14
|
* duplicate every method body and double the embedding bill, so:
|
|
15
15
|
*
|
|
16
|
-
* - a STRUCTURAL definition
|
|
17
|
-
* (class, interface, enum, struct, trait, object, module, namespace)
|
|
16
|
+
* - a STRUCTURAL definition - one whose body is a container of other definitions
|
|
17
|
+
* (class, interface, enum, struct, trait, object, module, namespace) - emits a
|
|
18
18
|
* HEADER chunk from its own first line to the line before its first nested
|
|
19
19
|
* definition: the declaration plus the fields that precede the first method.
|
|
20
20
|
* Real contiguous source, never an elision;
|
|
21
21
|
* - every other definition is a LEAF and becomes one chunk over its FULL line
|
|
22
22
|
* range, whatever it happens to contain. A local arrow function is part of
|
|
23
23
|
* what the enclosing function IS, not a sibling of it, so definitions nested
|
|
24
|
-
* inside a leaf are not emitted separately
|
|
24
|
+
* inside a leaf are not emitted separately - the leaf's chunk already carries
|
|
25
25
|
* their source verbatim. See `STRUCTURAL_KINDS` for why that distinction is
|
|
26
26
|
* load-bearing rather than cosmetic;
|
|
27
27
|
* - every remaining uncovered line span becomes a `raw` chunk.
|
|
@@ -70,11 +70,11 @@ export interface AstChunk {
|
|
|
70
70
|
chunkType: 'function' | 'class' | 'module' | 'raw';
|
|
71
71
|
/**
|
|
72
72
|
* Bare symbol name, never qualified. Present on every chunk that came from a
|
|
73
|
-
* `definition.*` capture
|
|
73
|
+
* `definition.*` capture - INCLUDING the value-shaped kinds that map to
|
|
74
74
|
* `chunkType: 'raw'` (`variable`, `constant`, `field`, `property`), because the
|
|
75
75
|
* wire enum has no value-shaped member but the identity is real and
|
|
76
76
|
* `memory_file_index.symbol_name` indexes it either way. Absent only on the
|
|
77
|
-
* uncovered-span chunks, which have no capture behind them
|
|
77
|
+
* uncovered-span chunks, which have no capture behind them - those are exactly
|
|
78
78
|
* the chunks with no `symbolKind` either.
|
|
79
79
|
*/
|
|
80
80
|
symbolName?: string;
|
|
@@ -83,14 +83,14 @@ export interface AstChunk {
|
|
|
83
83
|
/** The definition's first line, trimmed, capped at 500 chars. */
|
|
84
84
|
signature?: string;
|
|
85
85
|
/**
|
|
86
|
-
* Name of the immediate STRUCTURAL parent
|
|
86
|
+
* Name of the immediate STRUCTURAL parent - whatever `STRUCTURAL_KINDS` admits
|
|
87
87
|
* for the language. Absent at module scope, and absent inside a leaf (nothing
|
|
88
88
|
* inside a leaf is emitted at all).
|
|
89
89
|
*
|
|
90
90
|
* In TS/JS that is `class`, `interface` and a module-scope `const`; NOT `enum`
|
|
91
91
|
* and NOT `namespace`, despite both being structural kinds. Enum members are
|
|
92
92
|
* never captured as definitions, and `namespace Outer {}` parses as
|
|
93
|
-
* `internal_module`, which typescript.tags.scm does not match
|
|
93
|
+
* `internal_module`, which typescript.tags.scm does not match - `Outer`
|
|
94
94
|
* produces no definition, so it can never be anybody's container. Do not widen
|
|
95
95
|
* this list from `STRUCTURAL_KINDS` alone; a kind is only reachable here if
|
|
96
96
|
* some tag query also captures a definition INSIDE it.
|
|
@@ -108,8 +108,8 @@ export type AstChunkResult =
|
|
|
108
108
|
/**
|
|
109
109
|
* Definitions that reached no chunk. Two chunks over one line range would
|
|
110
110
|
* collide on `filePath:startLine-endLine`, the staleness key ProjectManager
|
|
111
|
-
* compares, so a second definition sharing a range
|
|
112
|
-
* function b() {}` on one line
|
|
111
|
+
* compares, so a second definition sharing a range - `function a() {}
|
|
112
|
+
* function b() {}` on one line - cannot be emitted, and neither can a
|
|
113
113
|
* container whose first nested definition opens on the container's own
|
|
114
114
|
* first line (`class A { m() {} }`). Two captures can also COLLAPSE onto one
|
|
115
115
|
* range while widening: `int f(int), g(int);` binds two declarators to one
|
|
@@ -126,7 +126,7 @@ export type AstChunkResult =
|
|
|
126
126
|
* Parse ceiling, derived from Task 4's measurement rather than guessed: the
|
|
127
127
|
* largest source file in this repo is 415 KB (`src/agent/ContextAgent.ts`) and
|
|
128
128
|
* the slowest per-file parse across 1,061 files was 108.6 ms. 2 MiB is ~5x that
|
|
129
|
-
* largest real file, which bounds the worst case near half a second
|
|
129
|
+
* largest real file, which bounds the worst case near half a second - while
|
|
130
130
|
* still refusing the multi-megabyte generated blobs that `MAX_SCANNED_FILE_BYTES`
|
|
131
131
|
* (10 MB) otherwise lets through. Above the ceiling the caller degrades to the
|
|
132
132
|
* heuristic chunker, which is line-bounded and cheap.
|
|
@@ -150,7 +150,7 @@ const MAX_SIGNATURE_CHARS = 500;
|
|
|
150
150
|
* Kinds absent here (`constant`, `field`, `property`, `variable`) fall to
|
|
151
151
|
* `'raw'`: the enum has no value-shaped member, and claiming a constant is a
|
|
152
152
|
* `class` or a `module` would be false where `'raw'` only says "no coarse class
|
|
153
|
-
* for this"
|
|
153
|
+
* for this" - the identity is still carried by `symbolName` + `symbolKind`.
|
|
154
154
|
*/
|
|
155
155
|
const CHUNK_TYPE_BY_KIND: Readonly<Record<string, AstChunk['chunkType']>> = {
|
|
156
156
|
function: 'function',
|
|
@@ -174,7 +174,7 @@ const CHUNK_TYPE_BY_KIND: Readonly<Record<string, AstChunk['chunkType']>> = {
|
|
|
174
174
|
* The distinction is load-bearing, not cosmetic. When every `definition.*`
|
|
175
175
|
* capture counted as a nested child, a function declaring one local arrow
|
|
176
176
|
* function collapsed to its first two lines: `ContextAgent.registerDocsTools`
|
|
177
|
-
* spans 513 lines and was recorded as 2. A wrong extent is worse than no extent
|
|
177
|
+
* spans 513 lines and was recorded as 2. A wrong extent is worse than no extent -
|
|
178
178
|
* it anchors citations confidently at the wrong place instead of failing to
|
|
179
179
|
* anchor them.
|
|
180
180
|
*
|
|
@@ -185,20 +185,20 @@ const CHUNK_TYPE_BY_KIND: Readonly<Record<string, AstChunk['chunkType']>> = {
|
|
|
185
185
|
* double the embedding bill.
|
|
186
186
|
*
|
|
187
187
|
* `constant` IS IN, and only because the tag queries anchor it to module scope.
|
|
188
|
-
* A module `const` is the ordinary home of a handler map or an evaluator table
|
|
189
|
-
* `export const EVALUATORS = { foo() {...}, bar: () => {...} }`
|
|
188
|
+
* A module `const` is the ordinary home of a handler map or an evaluator table -
|
|
189
|
+
* `export const EVALUATORS = { foo() {...}, bar: () => {...} }` - whose members
|
|
190
190
|
* are separately named definitions a reader looks up by name. As a leaf it
|
|
191
191
|
* suppressed all of them: 49 named symbols vanished from `src/` alone, with
|
|
192
192
|
* `droppedDefinitions` at 0, which is the silent-loss class that counter exists
|
|
193
193
|
* to prevent. A function-valued `const` is NOT affected: it binds
|
|
194
194
|
* `definition.function` on the same range and KIND_SPECIFICITY keeps that kind,
|
|
195
195
|
* so a closure and its local helpers stay one chunk. A `const` with no nested
|
|
196
|
-
* definition never splits either
|
|
197
|
-
* exists
|
|
196
|
+
* definition never splits either - `headerEndRow` is only set when a child
|
|
197
|
+
* exists - so the scalar case is byte-identical to the leaf behaviour.
|
|
198
198
|
*
|
|
199
199
|
* `type` stays out deliberately. C's `(type_definition declarator:
|
|
200
200
|
* (type_identifier) @name) @definition.type` matches `typedef struct {...} Foo;`,
|
|
201
|
-
* whose nested `struct_specifier` opens on the typedef's own first line
|
|
201
|
+
* whose nested `struct_specifier` opens on the typedef's own first line - as a
|
|
202
202
|
* container it would split to an empty header and be dropped entirely, so the
|
|
203
203
|
* typedef is the leaf and keeps the whole range.
|
|
204
204
|
*/
|
|
@@ -249,7 +249,7 @@ interface Definition {
|
|
|
249
249
|
*
|
|
250
250
|
* Async only because `loadGrammar` is; the grammar is cached per process, so
|
|
251
251
|
* after the first file of a given language there is no per-file async cost.
|
|
252
|
-
* Never throws
|
|
252
|
+
* Never throws - every failure is a named `unsupported` reason with a detail the
|
|
253
253
|
* caller can log before degrading.
|
|
254
254
|
*/
|
|
255
255
|
export async function chunkWithAst(
|
|
@@ -299,9 +299,9 @@ export async function chunkWithAst(
|
|
|
299
299
|
try {
|
|
300
300
|
const lines = content.split('\n');
|
|
301
301
|
// A trailing newline TERMINATES the last line, it does not open a new one,
|
|
302
|
-
// but `split` yields a phantom '' for it. Dropping it here
|
|
302
|
+
// but `split` yields a phantom '' for it. Dropping it here - before
|
|
303
303
|
// `lines.length` is read by the coverage array, `uncoveredSpans` or
|
|
304
|
-
// `sliceLines`
|
|
304
|
+
// `sliceLines` - is what keeps `endLine` equal to the file's last line
|
|
305
305
|
// number. Left in, the final raw chunk of every newline-terminated file (109
|
|
306
306
|
// of this repo's 268 `src/` TypeScript files) ran one line past the end, and
|
|
307
307
|
// `memory_file_index.end_line` feeds citation anchoring and the golden eval's
|
|
@@ -313,8 +313,8 @@ export async function chunkWithAst(
|
|
|
313
313
|
chunks: built.chunks,
|
|
314
314
|
errorNodes: countErrorNodes(tree),
|
|
315
315
|
parseMs,
|
|
316
|
-
// A definition can be lost at either stage
|
|
317
|
-
// definition's byte range here, or reaching no chunk in `buildChunks`
|
|
316
|
+
// A definition can be lost at either stage - collapsed onto another
|
|
317
|
+
// definition's byte range here, or reaching no chunk in `buildChunks` - and
|
|
318
318
|
// the caller logs one number, so both have to be in it.
|
|
319
319
|
droppedDefinitions: built.droppedDefinitions + collected.collapsedDefinitions,
|
|
320
320
|
};
|
|
@@ -333,7 +333,7 @@ export async function chunkWithAst(
|
|
|
333
333
|
* `STRUCTURAL_KINDS`, and a fourth table keyed on `definition.<kind>` drifts
|
|
334
334
|
* from the other three the moment a grammar is vendored: listing the strong
|
|
335
335
|
* kinds once already omitted `struct`, `object`, `trait` and `macro`, which
|
|
336
|
-
* scored 0 and LOST to `constant`
|
|
336
|
+
* scored 0 and LOST to `constant` - inverting the whole point of the table and
|
|
337
337
|
* able to demote a structural kind to a leaf. Deny-listing the four value-shaped
|
|
338
338
|
* kinds cannot fail that way: a kind nobody has thought of yet outranks a
|
|
339
339
|
* `constant` instead of losing to it, which is the safe direction.
|
|
@@ -353,7 +353,7 @@ interface CollectedDefinitions {
|
|
|
353
353
|
* Definitions lost to the byte-range dedup below, counted only when the two
|
|
354
354
|
* captures carried DIFFERENT names. Widening COLLAPSES: `int f(int), g(int);`
|
|
355
355
|
* binds two `function_declarator` siblings to one `declaration`, so both widen
|
|
356
|
-
* onto the same node and the second is discarded here
|
|
356
|
+
* onto the same node and the second is discarded here - upstream of
|
|
357
357
|
* `buildChunks`, where every other drop is counted. Left uncounted, widening
|
|
358
358
|
* turned a COUNTED drop into a silent one, which is the exact defect class
|
|
359
359
|
* `droppedDefinitions` exists to eliminate.
|
|
@@ -370,7 +370,7 @@ interface CollectedDefinitions {
|
|
|
370
370
|
*
|
|
371
371
|
* `matches()` rather than `captures()`: a match is one pattern's captures, so
|
|
372
372
|
* the `@name` belonging to a definition is unambiguous. `captures()` returns a
|
|
373
|
-
* flat stream in which pairing is guesswork
|
|
373
|
+
* flat stream in which pairing is guesswork - and the composed QUERY_CHAIN
|
|
374
374
|
* concatenates several files' patterns, so guessing would mispair across
|
|
375
375
|
* grammars.
|
|
376
376
|
*/
|
|
@@ -451,7 +451,7 @@ function collectDefinitions(query: Query, root: Node): CollectedDefinitions {
|
|
|
451
451
|
* (function_declarator declarator: (identifier) @name) @definition.function
|
|
452
452
|
*
|
|
453
453
|
* `function_declarator` ends at the closing paren of the parameter list, so
|
|
454
|
-
* naming it alone reproduces
|
|
454
|
+
* naming it alone reproduces - for two whole languages - the exact defect this
|
|
455
455
|
* module exists to fix: a named signature line plus an anonymous raw chunk
|
|
456
456
|
* holding the body.
|
|
457
457
|
*
|
|
@@ -464,10 +464,10 @@ function collectDefinitions(query: Query, root: Node): CollectedDefinitions {
|
|
|
464
464
|
* owns the body (`function_definition`, `declaration`, `field_declaration`,
|
|
465
465
|
* through any `pointer_declarator` or `array_declarator` in between).
|
|
466
466
|
* 2. A DECLARATOR WRAPPER. `int &f()` parses as
|
|
467
|
-
* `(function_definition declarator: (reference_declarator (function_declarator
|
|
468
|
-
* and `int (*f(void))[4]` as
|
|
467
|
+
* `(function_definition declarator: (reference_declarator (function_declarator ...)))`
|
|
468
|
+
* and `int (*f(void))[4]` as `... (parenthesized_declarator (pointer_declarator ...))`.
|
|
469
469
|
* Neither `reference_declarator` nor `parenthesized_declarator` NAMES the
|
|
470
|
-
* declarator it wraps
|
|
470
|
+
* declarator it wraps - the child sits in an unnamed field - so rule 1 alone
|
|
471
471
|
* stopped dead at `function_declarator` and reproduced the very defect above
|
|
472
472
|
* for `T &operator[]`, `std::string &name()`, and every other
|
|
473
473
|
* reference-returning accessor in real C++. So also ascend when the parent is
|
|
@@ -476,7 +476,7 @@ function collectDefinitions(query: Query, root: Node): CollectedDefinitions {
|
|
|
476
476
|
* Both forms keep the property that made rule 1 safe: neither can reach an
|
|
477
477
|
* unrelated ancestor. Rule 1 follows a field the grammar defines only within a
|
|
478
478
|
* declaration; rule 2 requires BOTH ends to be declarator nodes, and a
|
|
479
|
-
* `field_declaration_list` is not one
|
|
479
|
+
* `field_declaration_list` is not one - a method inside a class body therefore
|
|
480
480
|
* still stops at its own `function_definition` and a class is never swallowed.
|
|
481
481
|
* Grammars that already bind the whole definition (every JS/TS, Python, Go, Rust,
|
|
482
482
|
* Ruby pattern) fail both conditions on the first iteration, which makes this a
|
|
@@ -491,7 +491,7 @@ function widenToDefinition(node: Node): Node {
|
|
|
491
491
|
if (!parent) return current;
|
|
492
492
|
// `childrenForFieldName`, not `childForFieldName`: `int a, f(int x);` binds
|
|
493
493
|
// two declarators to one declaration, and the singular accessor answers only
|
|
494
|
-
// the first
|
|
494
|
+
// the first - which would refuse to widen every declarator but one.
|
|
495
495
|
const boundToDeclaratorField = parent
|
|
496
496
|
.childrenForFieldName('declarator')
|
|
497
497
|
.some((child) => child.id === current.id);
|
|
@@ -507,7 +507,7 @@ function isDeclarator(node: Node): boolean {
|
|
|
507
507
|
|
|
508
508
|
/**
|
|
509
509
|
* True when `parent` is a declarator that wraps `child` without naming the field
|
|
510
|
-
*
|
|
510
|
+
* - `reference_declarator` (`&` / `&&`) and `parenthesized_declarator` (`( ... )`),
|
|
511
511
|
* whose grammar rules are a bare token sequence around the inner declarator.
|
|
512
512
|
*
|
|
513
513
|
* Both ends must be declarators, and the field must be UNNAMED: a declarator's
|
|
@@ -539,7 +539,7 @@ interface BuiltChunks {
|
|
|
539
539
|
|
|
540
540
|
function buildChunks(definitions: Definition[], lines: string[]): BuiltChunks {
|
|
541
541
|
// `headerEndRow[i]` is the 0-based row of definition `i`'s first nested
|
|
542
|
-
// definition
|
|
542
|
+
// definition - set only for STRUCTURAL definitions, since only they split.
|
|
543
543
|
// `suppressed[i]` marks a definition that lies inside a leaf and is therefore
|
|
544
544
|
// already carried, verbatim, by that leaf's own chunk.
|
|
545
545
|
const headerEndRow = new Map<number, number>();
|
|
@@ -598,7 +598,7 @@ function buildChunks(definitions: Definition[], lines: string[]): BuiltChunks {
|
|
|
598
598
|
// starts on `headerEnd + 1`, so its header ends on the line before that:
|
|
599
599
|
// `headerEnd`. When the child opens on the container's own first line
|
|
600
600
|
// (`class A { m() {} }`) that range is empty, `emit` refuses it and the
|
|
601
|
-
// refusal is counted
|
|
601
|
+
// refusal is counted - emitting the line anyway would duplicate the child's
|
|
602
602
|
// source. Clamped to the last real line so a definition whose node runs to
|
|
603
603
|
// end-of-file cannot name a line that does not exist.
|
|
604
604
|
const headerEnd = headerEndRow.get(i);
|
|
@@ -620,7 +620,7 @@ function buildChunks(definitions: Definition[], lines: string[]): BuiltChunks {
|
|
|
620
620
|
// A definition that reaches no chunk reaches no index either, and a citation
|
|
621
621
|
// to it resolves as `unresolved_symbol`. It cannot be emitted without
|
|
622
622
|
// colliding on the staleness key, so it is counted and reported instead of
|
|
623
|
-
// vanishing
|
|
623
|
+
// vanishing - a silent no-op is the defect class, not the collision.
|
|
624
624
|
if (!emitted) droppedDefinitions++;
|
|
625
625
|
}
|
|
626
626
|
|
|
@@ -664,13 +664,13 @@ function uncoveredSpans(covered: Uint8Array, lineCount: number): Array<[number,
|
|
|
664
664
|
|
|
665
665
|
/**
|
|
666
666
|
* Break an uncovered span on line boundaries so no chunk exceeds
|
|
667
|
-
* `MAX_RAW_SPAN_CHARS`. No overlap
|
|
667
|
+
* `MAX_RAW_SPAN_CHARS`. No overlap - overlap would re-embed the same lines - and
|
|
668
668
|
* no mid-line splitting: two chunks over one line range would collide on the
|
|
669
669
|
* `filePath:startLine-endLine` staleness key, so an over-long single line is
|
|
670
670
|
* left whole and the embedding layer truncates it with a warning
|
|
671
671
|
* (`EmbeddingService` caps each input at the 8191-token per-input limit).
|
|
672
672
|
*
|
|
673
|
-
* Whitespace-only spans yield nothing
|
|
673
|
+
* Whitespace-only spans yield nothing - `emit` drops them, and the blank lines
|
|
674
674
|
* between definitions are the one thing total coverage does not owe a chunk.
|
|
675
675
|
*/
|
|
676
676
|
function splitSpan(
|
|
@@ -698,7 +698,7 @@ function splitSpan(
|
|
|
698
698
|
* of inferring it from thin chunks.
|
|
699
699
|
*
|
|
700
700
|
* Short-circuits on a clean tree, and descends only into subtrees whose own
|
|
701
|
-
* `hasError` is true
|
|
701
|
+
* `hasError` is true - 97.6% of this repo's TypeScript files have no error at
|
|
702
702
|
* all, and they must not pay for a full tree walk.
|
|
703
703
|
*/
|
|
704
704
|
function countErrorNodes(tree: Tree): number {
|
package/src/ast/grammars.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Every failure mode is a named value (`wasm_missing`, `wasm_load_failed`,
|
|
6
6
|
* `tags_query_missing`, `query_compile_failed`, `not_covered`) rather than a
|
|
7
7
|
* null or a throw, because the consequence of a degraded install is not a crash
|
|
8
|
-
*
|
|
8
|
+
* - it is quietly worse chunks, forever, with nothing in the logs. A caller that
|
|
9
9
|
* gets `unavailable` can fall back to the heuristic chunker AND say why.
|
|
10
10
|
*
|
|
11
11
|
* Artifacts are vendored, not installed: `packages/shared/wasm/<id>.wasm` plus
|
|
@@ -67,7 +67,7 @@ export const AST_LANGUAGE_IDS: readonly AstLanguageId[] = AST_LANGUAGE_ID_LIST;
|
|
|
67
67
|
* Ids whose vendored artifact basename differs from the id.
|
|
68
68
|
*
|
|
69
69
|
* `tree-sitter-c-sharp` names its file with a hyphen while every consumer of
|
|
70
|
-
* this registry
|
|
70
|
+
* this registry - `languageForExtension`, chunk metadata, the wire - spells the
|
|
71
71
|
* language `csharp`. Mapping the two explicitly is cheaper than renaming a
|
|
72
72
|
* vendored artifact: the vendoring script, the committed files and the
|
|
73
73
|
* `--check` verifier all agree on `c-sharp`, and a rename would make this
|
|
@@ -84,7 +84,7 @@ export function grammarArtifactBasename(id: AstLanguageId): string {
|
|
|
84
84
|
|
|
85
85
|
/**
|
|
86
86
|
* Ordered tag-query files composing each grammar's EFFECTIVE query. THE source
|
|
87
|
-
* of truth
|
|
87
|
+
* of truth - `scripts/vendor-grammars.ts` imports this rather than keeping its
|
|
88
88
|
* own copy, because two chains that can drift is precisely the defect this
|
|
89
89
|
* layer exists to remove.
|
|
90
90
|
*
|
|
@@ -93,7 +93,7 @@ export function grammarArtifactBasename(id: AstLanguageId): string {
|
|
|
93
93
|
* `interface_declaration`, `abstract_class_declaration`);
|
|
94
94
|
* `function_declaration`, `class_declaration`, `method_definition` and arrow
|
|
95
95
|
* functions all live in javascript's query and are meant to be inherited.
|
|
96
|
-
* Nothing in the query engine resolves that
|
|
96
|
+
* Nothing in the query engine resolves that - composing the chain is the
|
|
97
97
|
* consumer's job.
|
|
98
98
|
*
|
|
99
99
|
* Measured cost of not composing it: 948 definitions across 1,037 TypeScript
|
|
@@ -129,7 +129,7 @@ export const QUERY_CHAIN: Readonly<Record<AstLanguageId, readonly AstLanguageId[
|
|
|
129
129
|
* Extensions whose grammar the language string alone gets WRONG.
|
|
130
130
|
*
|
|
131
131
|
* `.ts` and `.tsx` both report language `typescript` and need different
|
|
132
|
-
* grammars out of the same npm package
|
|
132
|
+
* grammars out of the same npm package - a `.tsx` file parsed by the typescript
|
|
133
133
|
* grammar errors on the first JSX element. This is the entire reason
|
|
134
134
|
* `resolveAstLanguage` takes an extension at all.
|
|
135
135
|
*/
|
|
@@ -196,7 +196,7 @@ const WASM_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'wasm
|
|
|
196
196
|
* in parallel forks, so renaming `ruby.wasm` aside is visible to every other
|
|
197
197
|
* file for as long as it lasts, and any file that cold-loads Ruby inside that
|
|
198
198
|
* window caches a `wasm_missing` this checkout does not have. Copy from here
|
|
199
|
-
* into a temp directory and break the copy
|
|
199
|
+
* into a temp directory and break the copy - `loadGrammar` takes the directory.
|
|
200
200
|
*/
|
|
201
201
|
export const VENDORED_ARTIFACT_DIR = WASM_DIR;
|
|
202
202
|
|
|
@@ -216,7 +216,7 @@ export function resolveAstLanguage(extension: string, language: string): AstLang
|
|
|
216
216
|
|
|
217
217
|
/**
|
|
218
218
|
* Accepts `.TSX`, `tsx`, or a whole path, and answers the lowercase dotted
|
|
219
|
-
* extension. Mirrors `languageForExtension`'s ordering
|
|
219
|
+
* extension. Mirrors `languageForExtension`'s ordering - `extname` first, so a
|
|
220
220
|
* dotfile carrying a real extension ('.eslintrc.js') is not swallowed whole.
|
|
221
221
|
*/
|
|
222
222
|
function normalizeExtension(raw: string): string {
|
|
@@ -251,7 +251,7 @@ const loads = new Map<string, Promise<GrammarLoad>>();
|
|
|
251
251
|
/**
|
|
252
252
|
* Ids whose `Language` has actually been instantiated, same keying as `loads`.
|
|
253
253
|
*
|
|
254
|
-
*
|
|
254
|
+
* This is a permanent record rather than a cache index because entries cannot be removed:
|
|
255
255
|
* a loaded Language cannot be freed (see `attemptLoad`), so this set IS the
|
|
256
256
|
* permanent wasm memory this process is holding. `loadedGrammarIds` reads it,
|
|
257
257
|
* and the startup path is asserted against it.
|
|
@@ -283,7 +283,7 @@ export function loadGrammar(
|
|
|
283
283
|
}
|
|
284
284
|
|
|
285
285
|
/**
|
|
286
|
-
* Which grammars this process has instantiated
|
|
286
|
+
* Which grammars this process has instantiated - what is resident in wasm
|
|
287
287
|
* memory right now, not what is vendored or what was asked for.
|
|
288
288
|
*
|
|
289
289
|
* Exists because the cost is permanent and therefore worth being able to state:
|
|
@@ -306,7 +306,7 @@ async function loadUncached(id: AstLanguageId, artifactDir: string): Promise<Gra
|
|
|
306
306
|
|
|
307
307
|
/**
|
|
308
308
|
* The on-disk half of a load: artifacts present, query chain composed. No wasm
|
|
309
|
-
* is instantiated, so this is what the cheap report and the real load SHARE
|
|
309
|
+
* is instantiated, so this is what the cheap report and the real load SHARE -
|
|
310
310
|
* two separate presence checks that could disagree is how a report starts
|
|
311
311
|
* claiming a grammar the loader rejects.
|
|
312
312
|
*/
|
|
@@ -373,7 +373,7 @@ async function attemptLoad(id: AstLanguageId, artifactDir: string): Promise<Gram
|
|
|
373
373
|
// A loaded Language cannot be freed: web-tree-sitter 0.26.11 exposes
|
|
374
374
|
// `delete()` on Tree and Query but NOT on Language. The failure results below
|
|
375
375
|
// are cached, so a grammar whose query is broken leaks its Language exactly
|
|
376
|
-
// once per process rather than once per file
|
|
376
|
+
// once per process rather than once per file - which is why the cache matters
|
|
377
377
|
// for more than latency. Trees are a different story and must be deleted per
|
|
378
378
|
// file by the chunker; the JS heap does not GC wasm memory.
|
|
379
379
|
let query: Query;
|
|
@@ -392,7 +392,7 @@ async function attemptLoad(id: AstLanguageId, artifactDir: string): Promise<Gram
|
|
|
392
392
|
query.delete();
|
|
393
393
|
return {
|
|
394
394
|
unavailable: 'query_compile_failed',
|
|
395
|
-
detail: `${id} tag query [${QUERY_CHAIN[id].join(', ')}] compiled but captures no definition.*
|
|
395
|
+
detail: `${id} tag query [${QUERY_CHAIN[id].join(', ')}] compiled but captures no definition.* - it would name nothing`,
|
|
396
396
|
};
|
|
397
397
|
}
|
|
398
398
|
|
|
@@ -409,13 +409,13 @@ const describe = (err: unknown): string =>
|
|
|
409
409
|
*
|
|
410
410
|
* Every check is a `statSync` plus a few hundred bytes of `.scm`: the grammar's
|
|
411
411
|
* `.wasm` is present and non-empty, and so is every tag-query file in its chain.
|
|
412
|
-
* Microseconds, and zero permanent memory
|
|
412
|
+
* Microseconds, and zero permanent memory - which is what makes it the right
|
|
413
413
|
* answer for a startup line (see `logAstCapabilityOnce`).
|
|
414
414
|
*
|
|
415
415
|
* It shares `resolveArtifacts` with the real loader, so it cannot report a
|
|
416
416
|
* grammar as vendored that `loadGrammar` would reject for a missing artifact.
|
|
417
417
|
* What it deliberately cannot see is the two failure modes that require the
|
|
418
|
-
* grammar in memory
|
|
418
|
+
* grammar in memory - `wasm_load_failed` and `query_compile_failed`. Those are
|
|
419
419
|
* reported by `astCapabilityReport` when a human asks, and warned once per
|
|
420
420
|
* language by the scan path on the first file that needs them
|
|
421
421
|
* (`grammar_unavailable` in codeScanner); paying ~75 MB up front to pre-answer
|
|
@@ -462,12 +462,12 @@ export function astArtifactReport(artifactDir: string = WASM_DIR): {
|
|
|
462
462
|
* NOT free, and the cost is memory rather than only latency: measured on this
|
|
463
463
|
* checkout, ~690 ms and ~75 MB of RSS that is never given back. web-tree-sitter
|
|
464
464
|
* 0.26.11 exposes no `Language.delete` (see `attemptLoad`), so all 18 grammars
|
|
465
|
-
* stay resident for the life of the process
|
|
465
|
+
* stay resident for the life of the process - including the fifteen a given repo
|
|
466
466
|
* has no files for. Caching makes the second call free; it does not make the
|
|
467
467
|
* first one cheap.
|
|
468
468
|
*
|
|
469
469
|
* So: fine for a diagnostic a human asked for (`doctor`, a `--check` script,
|
|
470
|
-
* this suite). Wrong for a daemon's startup line
|
|
470
|
+
* this suite). Wrong for a daemon's startup line - that is `astArtifactReport`.
|
|
471
471
|
*/
|
|
472
472
|
export async function astCapabilityReport(artifactDir: string = WASM_DIR): Promise<{
|
|
473
473
|
available: AstLanguageId[];
|