@browserstack/mcp-server 1.5.0-beta.2 → 1.5.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/capability/loadtesting.capability-index.json +135 -29
- package/capability/tm.capability-index.json +42368 -12057
- package/dist/tools/capability-registry/bind.js +201 -6
- package/dist/tools/capability-registry/egress.js +52 -6
- package/dist/tools/capability-registry/index-loader.d.ts +15 -1
- package/dist/tools/capability-registry/index-loader.js +92 -2
- package/dist/tools/capability-registry/register.d.ts +14 -0
- package/dist/tools/capability-registry/register.js +367 -117
- package/dist/tools/capability-registry/search.d.ts +108 -3
- package/dist/tools/capability-registry/search.js +381 -14
- package/dist/tools/capability-registry/types.d.ts +80 -6
- package/package.json +7 -2
|
@@ -18,6 +18,14 @@ import { Capability, Mode, ProductIndex } from "./types.js";
|
|
|
18
18
|
* camelCase is split before lowercasing, so `testRunId`, `test_run_id` and `test run id`
|
|
19
19
|
* all tokenize alike.
|
|
20
20
|
*/
|
|
21
|
+
/**
|
|
22
|
+
* Is this the browse wildcard rather than a search?
|
|
23
|
+
*
|
|
24
|
+
* Accepts `*` and `all` — both are what a caller reaches for when the intent is "show me
|
|
25
|
+
* everything", and treating only one of them as the wildcard means the other silently
|
|
26
|
+
* becomes a bad search. Whitespace is tolerated so a pasted value still works.
|
|
27
|
+
*/
|
|
28
|
+
export declare function isBrowseQuery(query?: string): boolean;
|
|
21
29
|
export declare function terms(text: string | undefined): string[];
|
|
22
30
|
/**
|
|
23
31
|
* A term plus its naive singular variants.
|
|
@@ -29,6 +37,21 @@ export declare function terms(text: string | undefined): string[];
|
|
|
29
37
|
* for no additional match.
|
|
30
38
|
*/
|
|
31
39
|
export declare function termForms(term: string): string[];
|
|
40
|
+
/**
|
|
41
|
+
* ONE canonical spelling of a word, so two spellings of a noun cannot look like two terms.
|
|
42
|
+
*
|
|
43
|
+
* `termForms` is deliberately generous — it offers every candidate and lets substring
|
|
44
|
+
* containment sort them out. Canonicalising needs the opposite: exactly one answer, and
|
|
45
|
+
* the right one. Taking the SHORTEST candidate is what a first cut did, and it folded
|
|
46
|
+
* `cases` to `cas`, so the vocabulary entry `test case` matched no query containing "test
|
|
47
|
+
* cases" — the phrase it exists for. The ambiguity check then fell through to `test`
|
|
48
|
+
* alone, reaching the right verdict by the wrong route.
|
|
49
|
+
*
|
|
50
|
+
* Strip `es` only after a sibilant, where English actually inserts it (`boxes`,
|
|
51
|
+
* `batches`, `statuses`). Otherwise strip the single `s`, which is right for the `-e`
|
|
52
|
+
* plurals this vocabulary is full of: case, suite, phase, template.
|
|
53
|
+
*/
|
|
54
|
+
export declare function singular(word: string): string;
|
|
32
55
|
export declare function modeHint(query: string | undefined): "" | Mode;
|
|
33
56
|
export declare function wantsCollection(query: string | undefined): boolean;
|
|
34
57
|
/**
|
|
@@ -36,8 +59,9 @@ export declare function wantsCollection(query: string | undefined): boolean;
|
|
|
36
59
|
*
|
|
37
60
|
* Pagination is the reliable signal — a paged operation is a listing by construction. The
|
|
38
61
|
* plural terminal path segment is a weaker fallback for unpaged collections. (The Python
|
|
39
|
-
* side used the capability NAME here
|
|
40
|
-
*
|
|
62
|
+
* side used the capability NAME here. tm now publishes one, but its verbs are not
|
|
63
|
+
* consistent — the endpoint that lists folders is `get_root_folders_v1` — so the path's
|
|
64
|
+
* terminal noun remains the better signal.)
|
|
41
65
|
*/
|
|
42
66
|
export declare function isCollection(capability: Capability): boolean;
|
|
43
67
|
/**
|
|
@@ -45,7 +69,7 @@ export declare function isCollection(capability: Capability): boolean;
|
|
|
45
69
|
*
|
|
46
70
|
* Attribution is not decoration: the response tables are per product, so dereferencing a
|
|
47
71
|
* hit's schemas needs to know whose tables to read. It is also what lets a caller pass
|
|
48
|
-
* `product` to
|
|
72
|
+
* `product` to invokeCapability when two products share an endpoint — until now search
|
|
49
73
|
* ranked across products and then threw away the only thing that could disambiguate them.
|
|
50
74
|
*/
|
|
51
75
|
export interface SearchHit {
|
|
@@ -56,10 +80,91 @@ export interface SearchResult {
|
|
|
56
80
|
hits: SearchHit[];
|
|
57
81
|
truncated: boolean;
|
|
58
82
|
total_matched: number;
|
|
83
|
+
/**
|
|
84
|
+
* Where this page started, and where the next one begins.
|
|
85
|
+
*
|
|
86
|
+
* `truncated` has always said more matched than fit; nothing said how to reach it, so
|
|
87
|
+
* the only way past the first page was to raise `limit` and re-read everything already
|
|
88
|
+
* seen. `next_offset` is absent on the last page, which is what makes "walk until it
|
|
89
|
+
* stops" terminate without the caller comparing counts.
|
|
90
|
+
*/
|
|
91
|
+
offset: number;
|
|
92
|
+
next_offset?: number;
|
|
93
|
+
/** True when the query was the browse wildcard rather than a search. */
|
|
94
|
+
browse?: boolean;
|
|
95
|
+
/**
|
|
96
|
+
* The best pre-penalty term score, and whether it is weak enough to doubt.
|
|
97
|
+
*
|
|
98
|
+
* A vocabulary miss does NOT look like a miss from the outside: "make a new bucket for my
|
|
99
|
+
* tests" returns a full page of eight confident hits, exactly like a query that worked,
|
|
100
|
+
* because containment finds *something* for `make`, `new` and `tests`. The only thing that
|
|
101
|
+
* separates them is how little the match is worth.
|
|
102
|
+
*/
|
|
103
|
+
top_matched: number;
|
|
104
|
+
/** `top_matched` as a fraction of a perfect match — the corpus-independent form. */
|
|
105
|
+
coverage: number;
|
|
106
|
+
weak: boolean;
|
|
107
|
+
}
|
|
108
|
+
/** Which products a query could plausibly be about, when more than one could. */
|
|
109
|
+
export interface ProductAmbiguity {
|
|
110
|
+
/** The products that claim the query's words. Empty when the query settles itself. */
|
|
111
|
+
products: string[];
|
|
112
|
+
/** The shared vocabulary that caused it — what to put in front of the user. */
|
|
113
|
+
terms: string[];
|
|
59
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Products whose OWN vocabulary the query hits, when no word in the query settles it.
|
|
117
|
+
*
|
|
118
|
+
* Requiring `product` made every CALL unambiguous and did nothing about the agent making
|
|
119
|
+
* two of them: "list all projects" was answered by searching tm, then Load Testing, then
|
|
120
|
+
* merging — a question about which product the user meant, answered by guessing both.
|
|
121
|
+
* Tool-description prose asking the agent to check with the user is a suggestion it is
|
|
122
|
+
* free to decline, and declining is cheaper than interrupting someone.
|
|
123
|
+
*
|
|
124
|
+
* WHOLE VOCABULARY ENTRIES, never their constituent words. Splitting them makes almost
|
|
125
|
+
* everything look shared: Load Testing's "load test" and tm's "test case" both yield
|
|
126
|
+
* `test`, which flagged 135 of the 198 eval queries. Matching entries whole flags 27.
|
|
127
|
+
*
|
|
128
|
+
* A UNIQUE TERM SETTLES IT. "create a test case in a folder" contains `folder`, which
|
|
129
|
+
* both products claim, and `test case`, which only tm does — so the query has already
|
|
130
|
+
* answered the question and there is nothing to ask. Only queries whose every recognised
|
|
131
|
+
* word is shared are genuinely undecided.
|
|
132
|
+
*/
|
|
133
|
+
export declare function ambiguousProducts(products: Record<string, ProductIndex>, query: string | undefined): ProductAmbiguity;
|
|
134
|
+
/** One entity's caller-facing vocabulary: what it is called, and what else it is called. */
|
|
135
|
+
export interface VocabularyEntry {
|
|
136
|
+
entity: string;
|
|
137
|
+
description?: string;
|
|
138
|
+
aliases?: string[];
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The product's vocabulary, for a caller whose words are not the product's words.
|
|
142
|
+
*
|
|
143
|
+
* THE ONE FAILURE LEXICAL SEARCH CANNOT FIX. "make a new bucket for my tests" wants the
|
|
144
|
+
* folder-create capability, and `bucket` appears nowhere in the index — no scoring change
|
|
145
|
+
* reaches it, because the word is simply absent. What CAN reach it is the caller: it is a
|
|
146
|
+
* language model, and given tm's entity list it maps bucket -> folder without effort. It
|
|
147
|
+
* just cannot guess the list unprompted.
|
|
148
|
+
*
|
|
149
|
+
* Aliases and a one-line `description`; `title` stays dropped as a near-duplicate of
|
|
150
|
+
* `entity` ("Test run" next to `test_run` buys nothing). Aliases alone route but do not
|
|
151
|
+
* DEFINE — `result -> outcome, execution-result, test-result` never says whether that is
|
|
152
|
+
* the per-case verdict inside a run or a run-level rollup — so an agent holding names and
|
|
153
|
+
* aliases has exactly one way to find out, which is describeEntity once per entity. For
|
|
154
|
+
* tm that is 19 calls at ~1.4KB each, ~27KB, to answer what ~1.6KB of description answers
|
|
155
|
+
* here for every entity at once. The `key_facts` remain out: ten times the size, and the
|
|
156
|
+
* caller can ask describeEntity for the one entity it settles on.
|
|
157
|
+
*
|
|
158
|
+
* Its share of the response grew when search became a shortlist: 3.2KB against a 38KB full
|
|
159
|
+
* search was 8%, against a 6.8KB shortlist it is nearly half. The absolute cost did not
|
|
160
|
+
* change and is still under a thousand tokens — far less than one wrong invoke — so this
|
|
161
|
+
* is budgeted in bytes rather than as a fraction of a baseline that now moves.
|
|
162
|
+
*/
|
|
163
|
+
export declare function vocabularyOf(products: Record<string, ProductIndex>, only?: string): Record<string, VocabularyEntry[]>;
|
|
60
164
|
export declare function searchCapabilities(products: Record<string, ProductIndex>, query?: string, options?: {
|
|
61
165
|
entity?: string;
|
|
62
166
|
product?: string;
|
|
63
167
|
mode?: Mode;
|
|
64
168
|
limit?: number;
|
|
169
|
+
offset?: number;
|
|
65
170
|
}): SearchResult;
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
*
|
|
17
17
|
* `_` used to be a word character, which made `test_case` a single token while every
|
|
18
18
|
* haystack rendered it as "test case" — so the two could never match. That is the exact
|
|
19
|
-
* string `
|
|
19
|
+
* string `listProducts` hands back, so a caller following the documented flow searched with
|
|
20
20
|
* a term guaranteed to score zero: "list test_runs" matched 19 capabilities and put an
|
|
21
21
|
* admin settings endpoint first, where "list test runs" matched 103 and put the test-runs
|
|
22
22
|
* listing first.
|
|
@@ -111,6 +111,17 @@ const PLURAL_INTENT = new Set([
|
|
|
111
111
|
* camelCase is split before lowercasing, so `testRunId`, `test_run_id` and `test run id`
|
|
112
112
|
* all tokenize alike.
|
|
113
113
|
*/
|
|
114
|
+
/**
|
|
115
|
+
* Is this the browse wildcard rather than a search?
|
|
116
|
+
*
|
|
117
|
+
* Accepts `*` and `all` — both are what a caller reaches for when the intent is "show me
|
|
118
|
+
* everything", and treating only one of them as the wildcard means the other silently
|
|
119
|
+
* becomes a bad search. Whitespace is tolerated so a pasted value still works.
|
|
120
|
+
*/
|
|
121
|
+
export function isBrowseQuery(query) {
|
|
122
|
+
const q = (query || "").trim().toLowerCase();
|
|
123
|
+
return q === "*" || q === "all";
|
|
124
|
+
}
|
|
114
125
|
export function terms(text) {
|
|
115
126
|
return [
|
|
116
127
|
...(text || "")
|
|
@@ -145,6 +156,36 @@ export function termForms(term) {
|
|
|
145
156
|
add(term.slice(0, -1));
|
|
146
157
|
return forms;
|
|
147
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* ONE canonical spelling of a word, so two spellings of a noun cannot look like two terms.
|
|
161
|
+
*
|
|
162
|
+
* `termForms` is deliberately generous — it offers every candidate and lets substring
|
|
163
|
+
* containment sort them out. Canonicalising needs the opposite: exactly one answer, and
|
|
164
|
+
* the right one. Taking the SHORTEST candidate is what a first cut did, and it folded
|
|
165
|
+
* `cases` to `cas`, so the vocabulary entry `test case` matched no query containing "test
|
|
166
|
+
* cases" — the phrase it exists for. The ambiguity check then fell through to `test`
|
|
167
|
+
* alone, reaching the right verdict by the wrong route.
|
|
168
|
+
*
|
|
169
|
+
* Strip `es` only after a sibilant, where English actually inserts it (`boxes`,
|
|
170
|
+
* `batches`, `statuses`). Otherwise strip the single `s`, which is right for the `-e`
|
|
171
|
+
* plurals this vocabulary is full of: case, suite, phase, template.
|
|
172
|
+
*/
|
|
173
|
+
export function singular(word) {
|
|
174
|
+
if (word.endsWith("ss"))
|
|
175
|
+
return word;
|
|
176
|
+
// histories -> history. Without this the alias `histories` and the entity word
|
|
177
|
+
// `history` are two unrelated terms, which is the same defect in another spelling.
|
|
178
|
+
if (word.endsWith("ies") && word.length > 4)
|
|
179
|
+
return `${word.slice(0, -3)}y`;
|
|
180
|
+
// Only where the `e` is genuinely inserted. `s` is NOT in this set: `cases` ends in
|
|
181
|
+
// `ses` and is `case` + `s`, not `cas` + `es`, and no rule can tell it from `statuses`
|
|
182
|
+
// by suffix alone — so the commoner reading wins and `-s` is stripped below.
|
|
183
|
+
if (/(?:ch|sh|x|z)es$/.test(word))
|
|
184
|
+
return word.slice(0, -2);
|
|
185
|
+
if (word.endsWith("s") && word.length > 3)
|
|
186
|
+
return word.slice(0, -1);
|
|
187
|
+
return word;
|
|
188
|
+
}
|
|
148
189
|
/** A haystack as one lowercased, space-separated string, ready for containment tests. */
|
|
149
190
|
function haystack(text) {
|
|
150
191
|
return terms(text).join(" ");
|
|
@@ -165,8 +206,9 @@ export function wantsCollection(query) {
|
|
|
165
206
|
*
|
|
166
207
|
* Pagination is the reliable signal — a paged operation is a listing by construction. The
|
|
167
208
|
* plural terminal path segment is a weaker fallback for unpaged collections. (The Python
|
|
168
|
-
* side used the capability NAME here
|
|
169
|
-
*
|
|
209
|
+
* side used the capability NAME here. tm now publishes one, but its verbs are not
|
|
210
|
+
* consistent — the endpoint that lists folders is `get_root_folders_v1` — so the path's
|
|
211
|
+
* terminal noun remains the better signal.)
|
|
170
212
|
*/
|
|
171
213
|
export function isCollection(capability) {
|
|
172
214
|
if (capability.paginated)
|
|
@@ -177,6 +219,26 @@ export function isCollection(capability) {
|
|
|
177
219
|
const tail = segments[segments.length - 1] || "";
|
|
178
220
|
return tail.endsWith("s") && !tail.endsWith("ss");
|
|
179
221
|
}
|
|
222
|
+
/**
|
|
223
|
+
* Does this capability's mode answer what the caller asked to DO?
|
|
224
|
+
*
|
|
225
|
+
* `destructive` counts as a write. It has to: `modeHint` reads "delete" and "remove" as
|
|
226
|
+
* WRITE verbs, but a delete endpoint's mode is `destructive`, so a strict equality check
|
|
227
|
+
* penalised every destructive capability by 20 FOR BEING A DELETE — on exactly the queries
|
|
228
|
+
* that wanted one. "bulk delete test cases" scored 15.8 on terms, the highest of any missed
|
|
229
|
+
* query in the eval, and still fell out of the top 8.
|
|
230
|
+
*
|
|
231
|
+
* That accounted for 15 of the eval's 36 misses. The mode hint exists to separate reading
|
|
232
|
+
* from changing; destructive is changing.
|
|
233
|
+
*
|
|
234
|
+
* A read hint still refuses destructive, which is the useful half: "show me the test plans"
|
|
235
|
+
* must not surface a delete.
|
|
236
|
+
*/
|
|
237
|
+
function satisfiesHint(mode, hint) {
|
|
238
|
+
if (hint === "write")
|
|
239
|
+
return mode === "write" || mode === "destructive";
|
|
240
|
+
return mode === hint;
|
|
241
|
+
}
|
|
180
242
|
/** Everything a caller might say that lives on a parameter rather than in the prose. */
|
|
181
243
|
function parameterText(capability) {
|
|
182
244
|
const parts = [];
|
|
@@ -195,7 +257,49 @@ function parameterText(capability) {
|
|
|
195
257
|
}
|
|
196
258
|
return parts.join(" ");
|
|
197
259
|
}
|
|
198
|
-
/**
|
|
260
|
+
/**
|
|
261
|
+
* Path words are the identity haystack. The published `name` is deliberately NOT scored.
|
|
262
|
+
*
|
|
263
|
+
* That is a measured result, not an oversight. tm now names every capability, and adding
|
|
264
|
+
* the name to the ranking was tried three ways against tests/fixtures/search-eval.json,
|
|
265
|
+
* which the pre-names index serves 18/18:
|
|
266
|
+
*
|
|
267
|
+
* folded into this field, weight 6 17/18
|
|
268
|
+
* its own field, weights 1 / 2 / 3 / 4 / 6 17 / 17 / 17 / 17 / 16
|
|
269
|
+
* only the words the path lacks, 1..6 17 / 16 / 15 / 15 / 14
|
|
270
|
+
*
|
|
271
|
+
* Every variant loses, for two reasons. The nouns in a name are the route restated — the
|
|
272
|
+
* name is snake-cased from the operationId, itself derived from the path — so scoring them
|
|
273
|
+
* again rewards verbose names for repeating themselves: `get_test_cases_for_v1_test_run`
|
|
274
|
+
* displaced `create_test_result_for_test_case` on "record a pass or fail for a test case in
|
|
275
|
+
* a run", putting a read above the write that answers it. And what a name adds beyond the
|
|
276
|
+
* route is mostly its verb, which tm applies inconsistently (`get_root_folders_v1` lists,
|
|
277
|
+
* `list_folder_test_cases_v1` also lists), so the verb is noise as often as signal.
|
|
278
|
+
*
|
|
279
|
+
* The query this was meant to fix, "list all projects", only went 7 -> 4 even where it
|
|
280
|
+
* helped: `projects` is in 156 of 173 paths as a scope prefix, so rarity correctly values it
|
|
281
|
+
* near zero and no amount of name weighting recovers it. That one is fixed instead by the
|
|
282
|
+
* terminal-segment bonus in `score`, which tells "is that thing" from "is scoped by it".
|
|
283
|
+
*
|
|
284
|
+
* Revisit when a product ships a consistent verb convention — then the verb becomes signal.
|
|
285
|
+
*/
|
|
286
|
+
/**
|
|
287
|
+
* The terminal path segment — the thing this endpoint is actually ABOUT.
|
|
288
|
+
*
|
|
289
|
+
* A REST path mixes two different things: the resources it is SCOPED BY, and the resource it
|
|
290
|
+
* ADDRESSES. Only the last segment is the latter. See the bonus in `score` for why that
|
|
291
|
+
* distinction matters and why it is applied flat rather than weighted.
|
|
292
|
+
*
|
|
293
|
+
* Trailing placeholders are skipped, so `/test-cases/{id}` is still about test cases.
|
|
294
|
+
* Action tails (`close`, `edit`, `delete`) are kept rather than skipped: for "close a test
|
|
295
|
+
* run" the tail IS the most specific thing the caller said.
|
|
296
|
+
*/
|
|
297
|
+
function resourceText(capability) {
|
|
298
|
+
const segments = capability.path
|
|
299
|
+
.split("/")
|
|
300
|
+
.filter((s) => s && !s.startsWith("{") && s !== "api");
|
|
301
|
+
return (segments[segments.length - 1] || "").replace(/[-_]/g, " ");
|
|
302
|
+
}
|
|
199
303
|
function identityText(capability) {
|
|
200
304
|
return capability.path
|
|
201
305
|
.split("/")
|
|
@@ -225,11 +329,56 @@ function rarity(documents, forms) {
|
|
|
225
329
|
const total = documents.length || 1;
|
|
226
330
|
return Math.log((total + 1) / (df + 1)) / Math.log(total + 1);
|
|
227
331
|
}
|
|
332
|
+
/**
|
|
333
|
+
* Sized to sit alongside the mode (+6) and cardinality (+8) constants, not to dwarf them.
|
|
334
|
+
* The pinned eval is unchanged at every value from 2 to 14 — the bonus only ever fires on
|
|
335
|
+
* queries it does not cover — so this was chosen on the wider sweep: at 10, "close a test
|
|
336
|
+
* run" starts pulling `close_exploratory_session` into second place on the tail match alone.
|
|
337
|
+
*/
|
|
338
|
+
const RESOURCE_BONUS = 6;
|
|
339
|
+
/**
|
|
340
|
+
* How much a cardinality guess is worth. It used to be 8, and it should not have been.
|
|
341
|
+
*
|
|
342
|
+
* `isCollection` is a GUESS read off the URL, and the index carries nothing better: of 88
|
|
343
|
+
* read capabilities, zero declare an array in their response schema, and `paginated` — the
|
|
344
|
+
* one signal that looks trustworthy — is true for `get_report_detail`, a single-record
|
|
345
|
+
* endpoint. There is no derived cardinality in this artifact to appeal to.
|
|
346
|
+
*
|
|
347
|
+
* At ±8 that guess swung 16 points and buried 11 correct answers: `/test-runs/closed` and
|
|
348
|
+
* `/{entity}/search` are listings whose last path segment is not a plural noun, `users-v2`
|
|
349
|
+
* is plural with a version suffix in the way, and the count endpoints answer "how many"
|
|
350
|
+
* with a scalar. All were penalised for what they are called.
|
|
351
|
+
*
|
|
352
|
+
* The fix is not a better classifier — a qualifier word-list would encode tm's route
|
|
353
|
+
* conventions into generic code and rot on the next product. It is to stop betting so much
|
|
354
|
+
* on an unreliable signal. Swept against the eval, holding top1 at 138 with no ceiling
|
|
355
|
+
* violations: ±7 retires 1, ±6 retires 2, ±5 retires 5, ±4 retires 6. Below that it starts
|
|
356
|
+
* costing more than it returns — ±3.5 retires 8 but breaks a case, ±3 breaks two.
|
|
357
|
+
*/
|
|
358
|
+
const CARDINALITY = 4;
|
|
359
|
+
/**
|
|
360
|
+
* The mode penalty stays at 20, unlike the cardinality one — it is earned.
|
|
361
|
+
*
|
|
362
|
+
* Where `isCollection` is a guess off the URL, `modeHint` is measured right: across the
|
|
363
|
+
* eval it agrees with the correct answer's mode 125 times and disagrees 3. A signal that
|
|
364
|
+
* accurate deserves to be decisive.
|
|
365
|
+
*
|
|
366
|
+
* Sweeping it 20 -> 6 retires none of the three misses filed against it and regresses
|
|
367
|
+
* nothing, which says the penalty is not what holds them back. It isn't: they sit at ranks
|
|
368
|
+
* 41, 62 and 78 of the matched set, far below anything a constant could lift. All three
|
|
369
|
+
* are vocabulary gaps wearing a mode-penalty label — "what gets removed" wants a path
|
|
370
|
+
* spelled `rm-summary`, "get rid of" wants `delete`, and "the option set" wants what the
|
|
371
|
+
* product calls a `dataset`. The target barely matches on TERMS; the hint is incidental.
|
|
372
|
+
*
|
|
373
|
+
* Left at 20 deliberately. There was no evidence for moving it, and an unjustified constant
|
|
374
|
+
* is how this scorer got into trouble in the first place.
|
|
375
|
+
*/
|
|
376
|
+
const MODE_PENALTY = 20;
|
|
228
377
|
function score(capability, wanted, weights, aliases, hint, plural) {
|
|
229
378
|
if (wanted.length === 0)
|
|
230
379
|
return { matched: 1, ranked: 1 };
|
|
231
380
|
const haystacks = [
|
|
232
|
-
[identityText(capability),
|
|
381
|
+
[identityText(capability), IDENTITY_WEIGHT],
|
|
233
382
|
[capability.entity, 4],
|
|
234
383
|
[(aliases[capability.entity] || []).join(" "), 4],
|
|
235
384
|
[capability.intent || "", 2],
|
|
@@ -273,16 +422,220 @@ function score(capability, wanted, weights, aliases, hint, plural) {
|
|
|
273
422
|
}
|
|
274
423
|
}
|
|
275
424
|
const matched = ranked;
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
425
|
+
// THE ENDPOINT IS THAT THING, not merely scoped by it.
|
|
426
|
+
//
|
|
427
|
+
// A REST path mixes the resources it is SCOPED BY with the one it ADDRESSES. `projects` is
|
|
428
|
+
// in 156 of tm's 173 paths but is the terminal segment in 3, so whole-corpus rarity —
|
|
429
|
+
// correctly — values it near nothing, and every project-scoped listing scored the same as
|
|
430
|
+
// the projects listing itself. The top 8 for "list all projects" spanned 15.9 to 14.8,
|
|
431
|
+
// where +8 collection and +6 mode already account for 14: the term signal was ~1 point of
|
|
432
|
+
// noise and the right answer sat 7th.
|
|
433
|
+
//
|
|
434
|
+
// Flat, and deliberately NOT rarity-scaled. Rarity would reintroduce the same problem in
|
|
435
|
+
// reverse — a rare scope noun outranking the real target, which is exactly how a weighted
|
|
436
|
+
// version of this put `/projects/{id}/folders` above `/folder/{id}/test-cases` for "tc list
|
|
437
|
+
// for a folder". This asks one yes/no question instead: is the caller's own word the last
|
|
438
|
+
// thing in the path?
|
|
439
|
+
//
|
|
440
|
+
// Equality is against the QUERY's forms, never the haystack's — the same one-directional
|
|
441
|
+
// rule as containment. `projects` is in forms("projects"), so the projects listing hits;
|
|
442
|
+
// `folders` is not in forms("folder"), so a folder-scoped query does not drag in the
|
|
443
|
+
// folders listing.
|
|
444
|
+
const tail = terms(resourceText(capability));
|
|
445
|
+
if (tail.length &&
|
|
446
|
+
wanted.some((forms) => tail.every((word) => forms.includes(word)))) {
|
|
447
|
+
ranked += RESOURCE_BONUS;
|
|
448
|
+
}
|
|
449
|
+
if (hint)
|
|
450
|
+
ranked += satisfiesHint(capability.mode, hint) ? 6 : -MODE_PENALTY;
|
|
280
451
|
if (plural)
|
|
281
|
-
ranked += isCollection(capability) ?
|
|
452
|
+
ranked += isCollection(capability) ? CARDINALITY : -CARDINALITY;
|
|
282
453
|
return { matched, ranked };
|
|
283
454
|
}
|
|
455
|
+
/**
|
|
456
|
+
* How much of a perfect match the best hit actually achieved, below which we doubt it.
|
|
457
|
+
*
|
|
458
|
+
* A RATIO, because the raw score is corpus-relative and an absolute threshold therefore
|
|
459
|
+
* cannot travel. `rarity` is measured over whatever is being searched, so the same query
|
|
460
|
+
* scores differently depending on scope: "list my load tests" tops out at 1.9 against Load
|
|
461
|
+
* Testing alone and 10.1 against both products. The old absolute cut of 3 called the first
|
|
462
|
+
* of those weak and the second strong — the same query, the same right answer, opposite
|
|
463
|
+
* verdicts, with the smaller corpus penalised precisely because it is smaller.
|
|
464
|
+
*
|
|
465
|
+
* Both halves of this ratio move with the corpus, so it does not:
|
|
466
|
+
*
|
|
467
|
+
* coverage = top_matched / (identity weight x Σ rarity of the query's terms)
|
|
468
|
+
*
|
|
469
|
+
* — "what fraction of a perfect identity match did the best hit manage". Above 1 is
|
|
470
|
+
* ordinary, since a good hit matches several fields, not just the route.
|
|
471
|
+
*
|
|
472
|
+
* Measured across tm-only, LT-only and both: genuine misses land at 0.00–0.18, queries
|
|
473
|
+
* that work at 0.44–1.94. The absolute score cannot separate those (misses reach 2.9,
|
|
474
|
+
* good queries drop to 1.9); this does, with the gap in the same place in all three
|
|
475
|
+
* corpora, which is the property that was missing.
|
|
476
|
+
*
|
|
477
|
+
* RECALIBRATED AT v1.14, and the reason matters more than the number. The ratio is
|
|
478
|
+
* corpus-relative and still is — "make a new bucket for my tests" scores 0.259 against tm
|
|
479
|
+
* alone and 0.258 against both products, which is the property this was built for. What
|
|
480
|
+
* moved is neither corpus size nor query: it is how much TEXT each capability carries.
|
|
481
|
+
* v1.13 gave all 242 entries guidance and tripled the length of `intent`, so the numerator
|
|
482
|
+
* — the best hit's term score — roughly doubled, while the denominator depends only on the
|
|
483
|
+
* rarity of the query's own terms and did not. Both bands shifted up together: genuine
|
|
484
|
+
* misses now land at 0.00–0.46 and working queries at 0.24–4.16.
|
|
485
|
+
*
|
|
486
|
+
* At the old 0.25 the hand-off had silently stopped firing — "bucket", which is nobody's
|
|
487
|
+
* word for a folder, came in at 0.259 and read as a confident answer. Re-measured over 173
|
|
488
|
+
* served eval cases against five queries the product has no words for, 0.50 catches 5 of 5
|
|
489
|
+
* while flagging 3 of 173 served queries (1.7%), which is the same trade the original cut
|
|
490
|
+
* made. Expect to move it again the next time the artifact's text volume changes; the
|
|
491
|
+
* threshold tracks prose per capability, not the number of capabilities.
|
|
492
|
+
*/
|
|
493
|
+
const WEAK_COVERAGE = 0.5;
|
|
494
|
+
/** The heaviest field, and so the yardstick a perfect match is measured against. */
|
|
495
|
+
const IDENTITY_WEIGHT = 6;
|
|
496
|
+
/**
|
|
497
|
+
* Products whose OWN vocabulary the query hits, when no word in the query settles it.
|
|
498
|
+
*
|
|
499
|
+
* Requiring `product` made every CALL unambiguous and did nothing about the agent making
|
|
500
|
+
* two of them: "list all projects" was answered by searching tm, then Load Testing, then
|
|
501
|
+
* merging — a question about which product the user meant, answered by guessing both.
|
|
502
|
+
* Tool-description prose asking the agent to check with the user is a suggestion it is
|
|
503
|
+
* free to decline, and declining is cheaper than interrupting someone.
|
|
504
|
+
*
|
|
505
|
+
* WHOLE VOCABULARY ENTRIES, never their constituent words. Splitting them makes almost
|
|
506
|
+
* everything look shared: Load Testing's "load test" and tm's "test case" both yield
|
|
507
|
+
* `test`, which flagged 135 of the 198 eval queries. Matching entries whole flags 27.
|
|
508
|
+
*
|
|
509
|
+
* A UNIQUE TERM SETTLES IT. "create a test case in a folder" contains `folder`, which
|
|
510
|
+
* both products claim, and `test case`, which only tm does — so the query has already
|
|
511
|
+
* answered the question and there is nothing to ask. Only queries whose every recognised
|
|
512
|
+
* word is shared are genuinely undecided.
|
|
513
|
+
*/
|
|
514
|
+
export function ambiguousProducts(products, query) {
|
|
515
|
+
const none = { products: [], terms: [] };
|
|
516
|
+
if (terms(query).length === 0)
|
|
517
|
+
return none;
|
|
518
|
+
/**
|
|
519
|
+
* One spelling per word, so a plural cannot masquerade as a different term.
|
|
520
|
+
*
|
|
521
|
+
* Both halves need it. On the QUERY side, the entry is `project` and the user types
|
|
522
|
+
* "list all projects" — the reported case, which exact containment missed. On the
|
|
523
|
+
* VOCABULARY side the same fold is what keeps the answer honest: tm lists both `report`
|
|
524
|
+
* and `reports` as aliases, and treating them as two entries made `reports` look
|
|
525
|
+
* tm-exclusive, so "show me the report" read as settled when both products claim it.
|
|
526
|
+
*/
|
|
527
|
+
const fold = (text) => terms(text).map(singular).join(" ");
|
|
528
|
+
const asked_ = ` ${fold(query || "")} `;
|
|
529
|
+
const owners = new Map();
|
|
530
|
+
for (const [product, entries] of Object.entries(vocabularyOf(products))) {
|
|
531
|
+
for (const entry of entries) {
|
|
532
|
+
for (const word of [entry.entity, ...(entry.aliases ?? [])]) {
|
|
533
|
+
const key = fold(String(word));
|
|
534
|
+
if (!key)
|
|
535
|
+
continue;
|
|
536
|
+
if (!owners.has(key))
|
|
537
|
+
owners.set(key, new Set());
|
|
538
|
+
owners.get(key).add(product);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
const shared = new Map();
|
|
543
|
+
for (const [key, claimants] of owners) {
|
|
544
|
+
if (!asked_.includes(` ${key} `))
|
|
545
|
+
continue;
|
|
546
|
+
// A term only one product claims decides the query outright.
|
|
547
|
+
if (claimants.size < 2)
|
|
548
|
+
return none;
|
|
549
|
+
shared.set(key, claimants);
|
|
550
|
+
}
|
|
551
|
+
if (shared.size === 0)
|
|
552
|
+
return none;
|
|
553
|
+
const claimed = new Set();
|
|
554
|
+
for (const claimants of shared.values())
|
|
555
|
+
for (const product of claimants)
|
|
556
|
+
claimed.add(product);
|
|
557
|
+
return { products: [...claimed].sort(), terms: [...shared.keys()].sort() };
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* The product's vocabulary, for a caller whose words are not the product's words.
|
|
561
|
+
*
|
|
562
|
+
* THE ONE FAILURE LEXICAL SEARCH CANNOT FIX. "make a new bucket for my tests" wants the
|
|
563
|
+
* folder-create capability, and `bucket` appears nowhere in the index — no scoring change
|
|
564
|
+
* reaches it, because the word is simply absent. What CAN reach it is the caller: it is a
|
|
565
|
+
* language model, and given tm's entity list it maps bucket -> folder without effort. It
|
|
566
|
+
* just cannot guess the list unprompted.
|
|
567
|
+
*
|
|
568
|
+
* Aliases and a one-line `description`; `title` stays dropped as a near-duplicate of
|
|
569
|
+
* `entity` ("Test run" next to `test_run` buys nothing). Aliases alone route but do not
|
|
570
|
+
* DEFINE — `result -> outcome, execution-result, test-result` never says whether that is
|
|
571
|
+
* the per-case verdict inside a run or a run-level rollup — so an agent holding names and
|
|
572
|
+
* aliases has exactly one way to find out, which is describeEntity once per entity. For
|
|
573
|
+
* tm that is 19 calls at ~1.4KB each, ~27KB, to answer what ~1.6KB of description answers
|
|
574
|
+
* here for every entity at once. The `key_facts` remain out: ten times the size, and the
|
|
575
|
+
* caller can ask describeEntity for the one entity it settles on.
|
|
576
|
+
*
|
|
577
|
+
* Its share of the response grew when search became a shortlist: 3.2KB against a 38KB full
|
|
578
|
+
* search was 8%, against a 6.8KB shortlist it is nearly half. The absolute cost did not
|
|
579
|
+
* change and is still under a thousand tokens — far less than one wrong invoke — so this
|
|
580
|
+
* is budgeted in bytes rather than as a fraction of a baseline that now moves.
|
|
581
|
+
*/
|
|
582
|
+
export function vocabularyOf(products, only) {
|
|
583
|
+
const out = {};
|
|
584
|
+
for (const [name, bundle] of Object.entries(products)) {
|
|
585
|
+
if (only && name !== only)
|
|
586
|
+
continue;
|
|
587
|
+
const entries = [];
|
|
588
|
+
for (const [entity, doc] of Object.entries(bundle.entities || {})) {
|
|
589
|
+
const aliases = (doc.aliases || []);
|
|
590
|
+
const description = doc.description;
|
|
591
|
+
entries.push({
|
|
592
|
+
entity,
|
|
593
|
+
// Before the aliases, because it is what the reader needs first: what the thing
|
|
594
|
+
// is, then what else it is called.
|
|
595
|
+
...(description ? { description } : {}),
|
|
596
|
+
...(aliases.length ? { aliases } : {}),
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
if (entries.length)
|
|
600
|
+
out[name] = entries;
|
|
601
|
+
}
|
|
602
|
+
return out;
|
|
603
|
+
}
|
|
284
604
|
export function searchCapabilities(products, query, options = {}) {
|
|
285
605
|
const limit = options.limit && options.limit > 0 ? options.limit : 8;
|
|
606
|
+
const offset = options.offset && options.offset > 0 ? Math.floor(options.offset) : 0;
|
|
607
|
+
// BROWSE MODE: `*` lists the product instead of searching it.
|
|
608
|
+
//
|
|
609
|
+
// Scoring keeps only capabilities with matched > 0, so there was no query that returned
|
|
610
|
+
// everything — an agent that wanted to see what exists had to guess words until the
|
|
611
|
+
// shortlist looked complete, and could never tell when it was. `*` skips scoring
|
|
612
|
+
// entirely and pages the filtered set in a STABLE order (name, then path), because a
|
|
613
|
+
// ranked order is not a safe thing to paginate: ranking is query-dependent and ties
|
|
614
|
+
// break arbitrarily, so page 2 of a ranked list can repeat or skip rows. Browse is
|
|
615
|
+
// ordered by identity, so offset means the same thing on every call.
|
|
616
|
+
if (isBrowseQuery(query)) {
|
|
617
|
+
const all = Object.entries(products)
|
|
618
|
+
.filter(([name]) => !options.product || name === options.product)
|
|
619
|
+
.flatMap(([name, bundle]) => bundle.capabilities
|
|
620
|
+
.filter((capability) => (!options.entity || capability.entity === options.entity) &&
|
|
621
|
+
(!options.mode || capability.mode === options.mode))
|
|
622
|
+
.map((capability) => ({ product: name, capability })))
|
|
623
|
+
.sort((a, b) => (a.capability.name || "").localeCompare(b.capability.name || "") ||
|
|
624
|
+
a.capability.path.localeCompare(b.capability.path));
|
|
625
|
+
const page = all.slice(offset, offset + limit);
|
|
626
|
+
const end = offset + page.length;
|
|
627
|
+
return {
|
|
628
|
+
hits: page,
|
|
629
|
+
truncated: end < all.length,
|
|
630
|
+
total_matched: all.length,
|
|
631
|
+
top_matched: 0,
|
|
632
|
+
coverage: 1,
|
|
633
|
+
weak: false,
|
|
634
|
+
offset,
|
|
635
|
+
...(end < all.length ? { next_offset: end } : {}),
|
|
636
|
+
browse: true,
|
|
637
|
+
};
|
|
638
|
+
}
|
|
286
639
|
// Forms are computed once per query, not per capability: 173 capabilities x 6 haystacks
|
|
287
640
|
// would otherwise rebuild the same handful of strings a thousand times.
|
|
288
641
|
const wanted = terms(query).map(termForms);
|
|
@@ -332,11 +685,25 @@ export function searchCapabilities(products, query, options = {}) {
|
|
|
332
685
|
}
|
|
333
686
|
}
|
|
334
687
|
scored.sort((a, b) => b.ranked - a.ranked || a.capability.path.localeCompare(b.capability.path));
|
|
688
|
+
// Ranked order decides the page; the best TERM score decides confidence. They are
|
|
689
|
+
// different questions: the mode and cardinality constants can lift a weakly-matched
|
|
690
|
+
// capability to the top of a page that is entirely wrong.
|
|
691
|
+
const topMatched = scored.reduce((best, s) => Math.max(best, s.matched), 0);
|
|
692
|
+
// What a perfect identity match would have scored for THIS query in THIS corpus. Both
|
|
693
|
+
// this and `topMatched` scale with the corpus, so their ratio is comparable across
|
|
694
|
+
// products and across `product`/`entity` narrowing — which the raw score is not.
|
|
695
|
+
const perfect = weights.reduce((sum, w) => sum + IDENTITY_WEIGHT * w, 0);
|
|
696
|
+
const coverage = perfect > 0 ? topMatched / perfect : 1;
|
|
697
|
+
const page = scored.slice(offset, offset + limit);
|
|
698
|
+
const end = offset + page.length;
|
|
335
699
|
return {
|
|
336
|
-
hits:
|
|
337
|
-
|
|
338
|
-
.map(({ product, capability }) => ({ product, capability })),
|
|
339
|
-
truncated: scored.length > limit,
|
|
700
|
+
hits: page.map(({ product, capability }) => ({ product, capability })),
|
|
701
|
+
truncated: end < scored.length,
|
|
340
702
|
total_matched: scored.length,
|
|
703
|
+
offset,
|
|
704
|
+
...(end < scored.length ? { next_offset: end } : {}),
|
|
705
|
+
top_matched: topMatched,
|
|
706
|
+
coverage,
|
|
707
|
+
weak: wanted.length > 0 && perfect > 0 && coverage < WEAK_COVERAGE,
|
|
341
708
|
};
|
|
342
709
|
}
|