@browserstack/mcp-server 1.5.0-beta.11 → 1.5.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/capability/loadtesting.capability-index.json +6 -43
- package/capability/tm.capability-index.json +8 -309
- package/dist/tools/capability-registry/index-loader.d.ts +1 -15
- package/dist/tools/capability-registry/index-loader.js +1 -56
- package/dist/tools/capability-registry/register.js +30 -72
- package/dist/tools/capability-registry/search.d.ts +3 -35
- package/dist/tools/capability-registry/search.js +8 -193
- package/dist/tools/capability-registry/types.d.ts +0 -24
- package/package.json +1 -1
|
@@ -46,8 +46,6 @@ export declare class CapabilityRegistry {
|
|
|
46
46
|
readonly provenance: Record<string, Provenance>;
|
|
47
47
|
/** product -> "METHOD /path" -> capability */
|
|
48
48
|
private readonly byEndpoint;
|
|
49
|
-
/** product -> capability name -> capability. Empty for products that publish no names. */
|
|
50
|
-
private readonly byName;
|
|
51
49
|
constructor(index: RegistryIndex, provenance?: Record<string, Provenance>);
|
|
52
50
|
static fromFile(file: string): CapabilityRegistry;
|
|
53
51
|
/**
|
|
@@ -65,19 +63,7 @@ export declare class CapabilityRegistry {
|
|
|
65
63
|
/** Per-product `{build_id, version}`, for logging and cache-busting only. */
|
|
66
64
|
buildInfo(): Record<string, Provenance>;
|
|
67
65
|
/**
|
|
68
|
-
* Find a capability by
|
|
69
|
-
*
|
|
70
|
-
* Names are unique within a product but not across products, so an ambiguous name is
|
|
71
|
-
* reported rather than resolved by load order. A name that exists in no index is
|
|
72
|
-
* `unknown_capability`: a distinct outcome from a name that exists elsewhere, because the
|
|
73
|
-
* caller's next move differs — search again, versus pass `product`.
|
|
74
|
-
*/
|
|
75
|
-
byNameLookup(name: string, product?: string): {
|
|
76
|
-
product: string;
|
|
77
|
-
capability: Capability;
|
|
78
|
-
};
|
|
79
|
-
/**
|
|
80
|
-
* Find a capability by the endpoint it exposes — the handle for unnamed products.
|
|
66
|
+
* Find a capability by the endpoint it exposes — the published handle.
|
|
81
67
|
*
|
|
82
68
|
* The endpoint is what searchCapability returns, so it is the only thing a caller can
|
|
83
69
|
* hold. `unknown_endpoint` is a defined outcome rather than a generic failure: a caller
|
|
@@ -107,8 +107,6 @@ export class CapabilityRegistry {
|
|
|
107
107
|
provenance;
|
|
108
108
|
/** product -> "METHOD /path" -> capability */
|
|
109
109
|
byEndpoint = new Map();
|
|
110
|
-
/** product -> capability name -> capability. Empty for products that publish no names. */
|
|
111
|
-
byName = new Map();
|
|
112
110
|
constructor(index, provenance = {}) {
|
|
113
111
|
if (index?.schema_version !== SUPPORTED_SCHEMA_VERSION) {
|
|
114
112
|
throw new IndexError(`unsupported index schema_version ${index?.schema_version}; this build reads ` +
|
|
@@ -121,25 +119,10 @@ export class CapabilityRegistry {
|
|
|
121
119
|
this.provenance = provenance;
|
|
122
120
|
for (const [product, bundle] of Object.entries(index.products)) {
|
|
123
121
|
const lookup = new Map();
|
|
124
|
-
const names = new Map();
|
|
125
122
|
for (const capability of bundle.capabilities) {
|
|
126
123
|
lookup.set(endpointKey(capability.method, capability.path), capability);
|
|
127
|
-
if (!capability.name)
|
|
128
|
-
continue;
|
|
129
|
-
const clash = names.get(capability.name);
|
|
130
|
-
if (clash) {
|
|
131
|
-
// A duplicate name makes one of the two permanently unreachable, and which one
|
|
132
|
-
// wins would depend on array order. The export gates this, but a hand-edited or
|
|
133
|
-
// stale artifact must not load and then silently drop an endpoint.
|
|
134
|
-
throw new IndexError(`${product}: capability name '${capability.name}' is used by both ` +
|
|
135
|
-
`${endpointKey(clash.method, clash.path)} and ` +
|
|
136
|
-
`${endpointKey(capability.method, capability.path)}; names must be unique ` +
|
|
137
|
-
`within a product`);
|
|
138
|
-
}
|
|
139
|
-
names.set(capability.name, capability);
|
|
140
124
|
}
|
|
141
125
|
this.byEndpoint.set(product, lookup);
|
|
142
|
-
this.byName.set(product, names);
|
|
143
126
|
}
|
|
144
127
|
}
|
|
145
128
|
static fromFile(file) {
|
|
@@ -200,45 +183,7 @@ export class CapabilityRegistry {
|
|
|
200
183
|
return this.provenance;
|
|
201
184
|
}
|
|
202
185
|
/**
|
|
203
|
-
* Find a capability by
|
|
204
|
-
*
|
|
205
|
-
* Names are unique within a product but not across products, so an ambiguous name is
|
|
206
|
-
* reported rather than resolved by load order. A name that exists in no index is
|
|
207
|
-
* `unknown_capability`: a distinct outcome from a name that exists elsewhere, because the
|
|
208
|
-
* caller's next move differs — search again, versus pass `product`.
|
|
209
|
-
*/
|
|
210
|
-
byNameLookup(name, product) {
|
|
211
|
-
const matches = [];
|
|
212
|
-
for (const [owner, lookup] of this.byName) {
|
|
213
|
-
if (product && owner !== product)
|
|
214
|
-
continue;
|
|
215
|
-
const capability = lookup.get(name);
|
|
216
|
-
if (capability)
|
|
217
|
-
matches.push({ product: owner, capability });
|
|
218
|
-
}
|
|
219
|
-
if (matches.length === 0) {
|
|
220
|
-
// Say whether names are published at all for the product asked about: "no such name"
|
|
221
|
-
// and "this product does not name its capabilities" need different fixes.
|
|
222
|
-
const unnamed = [...this.byName]
|
|
223
|
-
.filter(([owner, lookup]) => (!product || owner === product) && lookup.size === 0)
|
|
224
|
-
.map(([owner]) => owner);
|
|
225
|
-
const hint = unnamed.length
|
|
226
|
-
? ` ${unnamed.sort().join(", ")} ${unnamed.length > 1 ? "publish" : "publishes"} ` +
|
|
227
|
-
`no capability names yet — call those by \`method\` and \`path\` instead.`
|
|
228
|
-
: " Search again — names come from searchCapability and describeEntity.";
|
|
229
|
-
throw new InvocationError(`unknown_capability: ${name}.${hint}`);
|
|
230
|
-
}
|
|
231
|
-
if (matches.length > 1 && !product) {
|
|
232
|
-
const owners = matches
|
|
233
|
-
.map((m) => m.product)
|
|
234
|
-
.sort()
|
|
235
|
-
.join(", ");
|
|
236
|
-
throw new InvocationError(`capability name '${name}' exists in several products (${owners}); pass product`);
|
|
237
|
-
}
|
|
238
|
-
return matches[0];
|
|
239
|
-
}
|
|
240
|
-
/**
|
|
241
|
-
* Find a capability by the endpoint it exposes — the handle for unnamed products.
|
|
186
|
+
* Find a capability by the endpoint it exposes — the published handle.
|
|
242
187
|
*
|
|
243
188
|
* The endpoint is what searchCapability returns, so it is the only thing a caller can
|
|
244
189
|
* hold. `unknown_endpoint` is a defined outcome rather than a generic failure: a caller
|
|
@@ -14,7 +14,7 @@ import { indexPaths, isEnabled, resolveBaseUrl } from "./config.js";
|
|
|
14
14
|
import { fetchTransport } from "./egress.js";
|
|
15
15
|
import { CapabilityRegistry, InvocationError, resolveResponses, } from "./index-loader.js";
|
|
16
16
|
import { invoke } from "./resolve.js";
|
|
17
|
-
import { searchCapabilities
|
|
17
|
+
import { searchCapabilities } from "./search.js";
|
|
18
18
|
export const PERMISSION_VALUES = ["not_asked", "granted", "denied"];
|
|
19
19
|
/**
|
|
20
20
|
* The tool-adder the server factory calls.
|
|
@@ -189,17 +189,13 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
189
189
|
tools.searchCapability = server.tool("searchCapability", "Find endpoints this surface can call, by plain language, optionally narrowed to one " +
|
|
190
190
|
"entity, product or mode. " +
|
|
191
191
|
`Currently loaded products: ${productList} — call listProducts for what each does. ` +
|
|
192
|
-
"Search matches the product's OWN words, not synonyms
|
|
193
|
-
"
|
|
194
|
-
"
|
|
195
|
-
"
|
|
196
|
-
"
|
|
197
|
-
"
|
|
198
|
-
"
|
|
199
|
-
"Each result carries the capability's `name` — the handle you pass to " +
|
|
200
|
-
"invokeCapability — plus its `method` and `path` (use those two only when a result " +
|
|
201
|
-
"has no `name`), and its parameters grouped into path_params / query / body under " +
|
|
202
|
-
"the spec's own names. Pass them straight back, no renaming. `intent` says what it does, " +
|
|
192
|
+
"Search matches the product's OWN words, not synonyms, so when a query returns " +
|
|
193
|
+
"nothing that fits, do not just rephrase it: call listEntities for the product and " +
|
|
194
|
+
"describeEntity on the closest entity, then search again using the vocabulary they " +
|
|
195
|
+
"return. Narrowing with `product` or `entity` sharpens results further. " +
|
|
196
|
+
"Each result carries the endpoint's `method` and `path` plus " +
|
|
197
|
+
"its parameters grouped into path_params / query / body under the spec's own names — " +
|
|
198
|
+
"pass them straight back to invokeEndpoint, no renaming. `intent` says what it does, " +
|
|
203
199
|
"`mode` tells you whether it writes, `product` says which product owns it, and " +
|
|
204
200
|
"`responses` describes what a successful call returns, fully expanded. Results are " +
|
|
205
201
|
"ranked and capped, and `truncated` says when more matched. Search before invoking.", {
|
|
@@ -233,7 +229,7 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
233
229
|
}, async ({ query, entity, product, mode, limit, include_responses }) => {
|
|
234
230
|
track("searchCapability");
|
|
235
231
|
const selection = (include_responses || "success");
|
|
236
|
-
const { hits,
|
|
232
|
+
const { hits, ...rest } = searchCapabilities(registry.index.products, query, {
|
|
237
233
|
entity,
|
|
238
234
|
product,
|
|
239
235
|
mode: mode,
|
|
@@ -259,52 +255,24 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
259
255
|
};
|
|
260
256
|
}),
|
|
261
257
|
...rest,
|
|
262
|
-
// WHEN THE MATCH IS WEAK, HAND OVER THE VOCABULARY.
|
|
263
|
-
//
|
|
264
|
-
// A vocabulary miss returns a full page of confident-looking hits — nothing in the
|
|
265
|
-
// shape of the response says the caller's word does not exist in this product. This
|
|
266
|
-
// is that signal, plus the fix in the same round trip: the caller is a model, and it
|
|
267
|
-
// maps "bucket" to "folder" instantly once it can see the entity list.
|
|
268
|
-
//
|
|
269
|
-
// Additive, never a replacement — the results are still there, and the caller is
|
|
270
|
-
// told to re-search rather than to trust this. That is what makes a generous
|
|
271
|
-
// threshold safe.
|
|
272
|
-
...(weak
|
|
273
|
-
? {
|
|
274
|
-
weak_match: true,
|
|
275
|
-
suggested_vocabulary: vocabularyOf(registry.index.products, product),
|
|
276
|
-
hint: "Nothing matched the product's own words strongly (best term score " +
|
|
277
|
-
`${top_matched.toFixed(1)}). The results below may not answer the ` +
|
|
278
|
-
"question. Re-search using a term from suggested_vocabulary, or call " +
|
|
279
|
-
"describeEntity on the closest entity for its full vocabulary.",
|
|
280
|
-
}
|
|
281
|
-
: {}),
|
|
282
258
|
});
|
|
283
259
|
});
|
|
284
|
-
tools.
|
|
285
|
-
"
|
|
286
|
-
"
|
|
287
|
-
"
|
|
288
|
-
"
|
|
289
|
-
"
|
|
290
|
-
"
|
|
291
|
-
"
|
|
292
|
-
"
|
|
293
|
-
"
|
|
294
|
-
"intent before confirming with the user.", {
|
|
295
|
-
name: z
|
|
296
|
-
.string()
|
|
297
|
-
.optional()
|
|
298
|
-
.describe("The capability's published name, exactly as returned (e.g. 'create_test_run_v1'). " +
|
|
299
|
-
"Preferred over method/path."),
|
|
260
|
+
tools.invokeEndpoint = server.tool("invokeEndpoint", "Call an endpoint returned by searchCapability. Pass `method` and `path` exactly as " +
|
|
261
|
+
"given, with arguments grouped into path_params / query / body under the spec's own " +
|
|
262
|
+
"names. One call makes exactly one request and returns the product's own response " +
|
|
263
|
+
"untouched; when `completed` is false there is another page, which you fetch by " +
|
|
264
|
+
"sending the endpoint's own page parameter. If the endpoint's mode is 'write' you " +
|
|
265
|
+
"MUST ask the user first, then " +
|
|
266
|
+
"resend with user_permission='granted' and a change_summary; both are recorded. " +
|
|
267
|
+
"Endpoints whose mode is 'destructive' (deletes) are refused outright — archiving, " +
|
|
268
|
+
"closing and merging are ordinary writes and DO run, so read the mode and intent " +
|
|
269
|
+
"before confirming with the user.", {
|
|
300
270
|
method: z
|
|
301
271
|
.string()
|
|
302
|
-
.
|
|
303
|
-
.describe("HTTP method — only for capabilities returned without a `name`."),
|
|
272
|
+
.describe("HTTP method, exactly as searchCapability returned it."),
|
|
304
273
|
path: z
|
|
305
274
|
.string()
|
|
306
|
-
.
|
|
307
|
-
.describe("Path with {placeholders} intact — only for capabilities returned without a `name`."),
|
|
275
|
+
.describe("Path with {placeholders} intact, exactly as returned."),
|
|
308
276
|
path_params: z
|
|
309
277
|
.record(z.string(), z.any())
|
|
310
278
|
.optional()
|
|
@@ -319,8 +287,8 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
319
287
|
.describe("Body fields, under the spec's names."),
|
|
320
288
|
product: productArg()
|
|
321
289
|
.optional()
|
|
322
|
-
.describe(`Which product owns the
|
|
323
|
-
"on every result; required only when two products share a
|
|
290
|
+
.describe(`Which product owns the endpoint (${productList}). searchCapability returns it ` +
|
|
291
|
+
"on every result; required only when two products share a path."),
|
|
324
292
|
user_permission: z
|
|
325
293
|
.enum(PERMISSION_VALUES)
|
|
326
294
|
.optional()
|
|
@@ -330,7 +298,7 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
330
298
|
.optional()
|
|
331
299
|
.describe("What will change. Required for writes."),
|
|
332
300
|
}, {
|
|
333
|
-
title: "Invoke
|
|
301
|
+
title: "Invoke Endpoint",
|
|
334
302
|
// Not read-only: this is the one tool that writes. Never destructive, because
|
|
335
303
|
// destructive endpoints are refused before binding — the refusal is enforced here,
|
|
336
304
|
// not merely hinted at. Not idempotent: it creates, clones and starts runs. Closed
|
|
@@ -340,20 +308,9 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
340
308
|
idempotentHint: false,
|
|
341
309
|
openWorldHint: false,
|
|
342
310
|
}, async (input) => {
|
|
343
|
-
track("
|
|
311
|
+
track("invokeEndpoint");
|
|
344
312
|
try {
|
|
345
|
-
|
|
346
|
-
// rather than cross-checking them: a caller pasting a stale path alongside a good
|
|
347
|
-
// name should still reach the right operation, which is the point of naming.
|
|
348
|
-
if (!input.name && !(input.method && input.path)) {
|
|
349
|
-
return failed("pass `name` — or, for a capability returned without one, both `method` and " +
|
|
350
|
-
"`path`, exactly as searchCapability returned them");
|
|
351
|
-
}
|
|
352
|
-
const { product, capability } = input.name
|
|
353
|
-
? registry.byNameLookup(input.name, input.product)
|
|
354
|
-
: registry.byEndpointLookup(input.method, input.path, input.product);
|
|
355
|
-
/** What to call it in errors — the handle the caller actually used. */
|
|
356
|
-
const handle = capability.name || `${capability.method} ${capability.path}`;
|
|
313
|
+
const { product, capability } = registry.byEndpointLookup(input.method, input.path, input.product);
|
|
357
314
|
const args = {
|
|
358
315
|
path_params: input.path_params,
|
|
359
316
|
query: input.query,
|
|
@@ -361,7 +318,8 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
361
318
|
};
|
|
362
319
|
if (capability.mode === "destructive") {
|
|
363
320
|
// Refused before binding, so consent is never sought for something that cannot run.
|
|
364
|
-
return failed(`${
|
|
321
|
+
return failed(`${input.method} ${input.path} is a destructive operation and is not available ` +
|
|
322
|
+
`through this surface`);
|
|
365
323
|
}
|
|
366
324
|
if (capability.mode === "write") {
|
|
367
325
|
const permission = input.user_permission || "not_asked";
|
|
@@ -387,8 +345,8 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
387
345
|
catch (error) {
|
|
388
346
|
if (error instanceof InvocationError)
|
|
389
347
|
return failed(error.message);
|
|
390
|
-
logger.error("
|
|
391
|
-
return failed("that
|
|
348
|
+
logger.error("invokeEndpoint failed: %s", error instanceof Error ? error.message : String(error));
|
|
349
|
+
return failed("that endpoint could not be invoked");
|
|
392
350
|
}
|
|
393
351
|
});
|
|
394
352
|
return tools;
|
|
@@ -36,9 +36,8 @@ export declare function wantsCollection(query: string | undefined): boolean;
|
|
|
36
36
|
*
|
|
37
37
|
* Pagination is the reliable signal — a paged operation is a listing by construction. The
|
|
38
38
|
* plural terminal path segment is a weaker fallback for unpaged collections. (The Python
|
|
39
|
-
* side used the capability NAME here
|
|
40
|
-
*
|
|
41
|
-
* terminal noun remains the better signal.)
|
|
39
|
+
* side used the capability NAME here; the artifact publishes no name, and the path's own
|
|
40
|
+
* terminal noun carries the same signal because operationIds were derived from it.)
|
|
42
41
|
*/
|
|
43
42
|
export declare function isCollection(capability: Capability): boolean;
|
|
44
43
|
/**
|
|
@@ -46,7 +45,7 @@ export declare function isCollection(capability: Capability): boolean;
|
|
|
46
45
|
*
|
|
47
46
|
* Attribution is not decoration: the response tables are per product, so dereferencing a
|
|
48
47
|
* hit's schemas needs to know whose tables to read. It is also what lets a caller pass
|
|
49
|
-
* `product` to
|
|
48
|
+
* `product` to invokeEndpoint when two products share an endpoint — until now search
|
|
50
49
|
* ranked across products and then threw away the only thing that could disambiguate them.
|
|
51
50
|
*/
|
|
52
51
|
export interface SearchHit {
|
|
@@ -57,38 +56,7 @@ export interface SearchResult {
|
|
|
57
56
|
hits: SearchHit[];
|
|
58
57
|
truncated: boolean;
|
|
59
58
|
total_matched: number;
|
|
60
|
-
/**
|
|
61
|
-
* The best pre-penalty term score, and whether it is weak enough to doubt.
|
|
62
|
-
*
|
|
63
|
-
* A vocabulary miss does NOT look like a miss from the outside: "make a new bucket for my
|
|
64
|
-
* tests" returns a full page of eight confident hits, exactly like a query that worked,
|
|
65
|
-
* because containment finds *something* for `make`, `new` and `tests`. The only thing that
|
|
66
|
-
* separates them is how little the match is worth.
|
|
67
|
-
*/
|
|
68
|
-
top_matched: number;
|
|
69
|
-
weak: boolean;
|
|
70
59
|
}
|
|
71
|
-
/** One entity's caller-facing vocabulary: what it is called, and what else it is called. */
|
|
72
|
-
export interface VocabularyEntry {
|
|
73
|
-
entity: string;
|
|
74
|
-
title?: string;
|
|
75
|
-
aliases?: string[];
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* The product's vocabulary, for a caller whose words are not the product's words.
|
|
79
|
-
*
|
|
80
|
-
* THE ONE FAILURE LEXICAL SEARCH CANNOT FIX. "make a new bucket for my tests" wants the
|
|
81
|
-
* folder-create capability, and `bucket` appears nowhere in the index — no scoring change
|
|
82
|
-
* reaches it, because the word is simply absent. What CAN reach it is the caller: it is a
|
|
83
|
-
* language model, and given tm's entity list it maps bucket -> folder without effort. It
|
|
84
|
-
* just cannot guess the list unprompted.
|
|
85
|
-
*
|
|
86
|
-
* Aliases only, deliberately. They are the vocabulary map — 19 entities in ~1.5KB, against a
|
|
87
|
-
* response that is routinely 38KB. The entity `key_facts` are richer prose but ten times the
|
|
88
|
-
* size, and a caller who needs them can ask describeEntity once it knows which entity to ask
|
|
89
|
-
* about — which is exactly what this hands over.
|
|
90
|
-
*/
|
|
91
|
-
export declare function vocabularyOf(products: Record<string, ProductIndex>, only?: string): Record<string, VocabularyEntry[]>;
|
|
92
60
|
export declare function searchCapabilities(products: Record<string, ProductIndex>, query?: string, options?: {
|
|
93
61
|
entity?: string;
|
|
94
62
|
product?: string;
|
|
@@ -165,9 +165,8 @@ export function wantsCollection(query) {
|
|
|
165
165
|
*
|
|
166
166
|
* Pagination is the reliable signal — a paged operation is a listing by construction. The
|
|
167
167
|
* plural terminal path segment is a weaker fallback for unpaged collections. (The Python
|
|
168
|
-
* side used the capability NAME here
|
|
169
|
-
*
|
|
170
|
-
* terminal noun remains the better signal.)
|
|
168
|
+
* side used the capability NAME here; the artifact publishes no name, and the path's own
|
|
169
|
+
* terminal noun carries the same signal because operationIds were derived from it.)
|
|
171
170
|
*/
|
|
172
171
|
export function isCollection(capability) {
|
|
173
172
|
if (capability.paginated)
|
|
@@ -178,26 +177,6 @@ export function isCollection(capability) {
|
|
|
178
177
|
const tail = segments[segments.length - 1] || "";
|
|
179
178
|
return tail.endsWith("s") && !tail.endsWith("ss");
|
|
180
179
|
}
|
|
181
|
-
/**
|
|
182
|
-
* Does this capability's mode answer what the caller asked to DO?
|
|
183
|
-
*
|
|
184
|
-
* `destructive` counts as a write. It has to: `modeHint` reads "delete" and "remove" as
|
|
185
|
-
* WRITE verbs, but a delete endpoint's mode is `destructive`, so a strict equality check
|
|
186
|
-
* penalised every destructive capability by 20 FOR BEING A DELETE — on exactly the queries
|
|
187
|
-
* that wanted one. "bulk delete test cases" scored 15.8 on terms, the highest of any missed
|
|
188
|
-
* query in the eval, and still fell out of the top 8.
|
|
189
|
-
*
|
|
190
|
-
* That accounted for 15 of the eval's 36 misses. The mode hint exists to separate reading
|
|
191
|
-
* from changing; destructive is changing.
|
|
192
|
-
*
|
|
193
|
-
* A read hint still refuses destructive, which is the useful half: "show me the test plans"
|
|
194
|
-
* must not surface a delete.
|
|
195
|
-
*/
|
|
196
|
-
function satisfiesHint(mode, hint) {
|
|
197
|
-
if (hint === "write")
|
|
198
|
-
return mode === "write" || mode === "destructive";
|
|
199
|
-
return mode === hint;
|
|
200
|
-
}
|
|
201
180
|
/** Everything a caller might say that lives on a parameter rather than in the prose. */
|
|
202
181
|
function parameterText(capability) {
|
|
203
182
|
const parts = [];
|
|
@@ -216,49 +195,7 @@ function parameterText(capability) {
|
|
|
216
195
|
}
|
|
217
196
|
return parts.join(" ");
|
|
218
197
|
}
|
|
219
|
-
/**
|
|
220
|
-
* Path words are the identity haystack. The published `name` is deliberately NOT scored.
|
|
221
|
-
*
|
|
222
|
-
* That is a measured result, not an oversight. tm now names every capability, and adding
|
|
223
|
-
* the name to the ranking was tried three ways against tests/fixtures/search-eval.json,
|
|
224
|
-
* which the pre-names index serves 18/18:
|
|
225
|
-
*
|
|
226
|
-
* folded into this field, weight 6 17/18
|
|
227
|
-
* its own field, weights 1 / 2 / 3 / 4 / 6 17 / 17 / 17 / 17 / 16
|
|
228
|
-
* only the words the path lacks, 1..6 17 / 16 / 15 / 15 / 14
|
|
229
|
-
*
|
|
230
|
-
* Every variant loses, for two reasons. The nouns in a name are the route restated — the
|
|
231
|
-
* name is snake-cased from the operationId, itself derived from the path — so scoring them
|
|
232
|
-
* again rewards verbose names for repeating themselves: `get_test_cases_for_v1_test_run`
|
|
233
|
-
* displaced `create_test_result_for_test_case` on "record a pass or fail for a test case in
|
|
234
|
-
* a run", putting a read above the write that answers it. And what a name adds beyond the
|
|
235
|
-
* route is mostly its verb, which tm applies inconsistently (`get_root_folders_v1` lists,
|
|
236
|
-
* `list_folder_test_cases_v1` also lists), so the verb is noise as often as signal.
|
|
237
|
-
*
|
|
238
|
-
* The query this was meant to fix, "list all projects", only went 7 -> 4 even where it
|
|
239
|
-
* helped: `projects` is in 156 of 173 paths as a scope prefix, so rarity correctly values it
|
|
240
|
-
* near zero and no amount of name weighting recovers it. That one is fixed instead by the
|
|
241
|
-
* terminal-segment bonus in `score`, which tells "is that thing" from "is scoped by it".
|
|
242
|
-
*
|
|
243
|
-
* Revisit when a product ships a consistent verb convention — then the verb becomes signal.
|
|
244
|
-
*/
|
|
245
|
-
/**
|
|
246
|
-
* The terminal path segment — the thing this endpoint is actually ABOUT.
|
|
247
|
-
*
|
|
248
|
-
* A REST path mixes two different things: the resources it is SCOPED BY, and the resource it
|
|
249
|
-
* ADDRESSES. Only the last segment is the latter. See the bonus in `score` for why that
|
|
250
|
-
* distinction matters and why it is applied flat rather than weighted.
|
|
251
|
-
*
|
|
252
|
-
* Trailing placeholders are skipped, so `/test-cases/{id}` is still about test cases.
|
|
253
|
-
* Action tails (`close`, `edit`, `delete`) are kept rather than skipped: for "close a test
|
|
254
|
-
* run" the tail IS the most specific thing the caller said.
|
|
255
|
-
*/
|
|
256
|
-
function resourceText(capability) {
|
|
257
|
-
const segments = capability.path
|
|
258
|
-
.split("/")
|
|
259
|
-
.filter((s) => s && !s.startsWith("{") && s !== "api");
|
|
260
|
-
return (segments[segments.length - 1] || "").replace(/[-_]/g, " ");
|
|
261
|
-
}
|
|
198
|
+
/** Path words stand in for the capability name as the identity haystack. */
|
|
262
199
|
function identityText(capability) {
|
|
263
200
|
return capability.path
|
|
264
201
|
.split("/")
|
|
@@ -288,51 +225,6 @@ function rarity(documents, forms) {
|
|
|
288
225
|
const total = documents.length || 1;
|
|
289
226
|
return Math.log((total + 1) / (df + 1)) / Math.log(total + 1);
|
|
290
227
|
}
|
|
291
|
-
/**
|
|
292
|
-
* Sized to sit alongside the mode (+6) and cardinality (+8) constants, not to dwarf them.
|
|
293
|
-
* The pinned eval is unchanged at every value from 2 to 14 — the bonus only ever fires on
|
|
294
|
-
* queries it does not cover — so this was chosen on the wider sweep: at 10, "close a test
|
|
295
|
-
* run" starts pulling `close_exploratory_session` into second place on the tail match alone.
|
|
296
|
-
*/
|
|
297
|
-
const RESOURCE_BONUS = 6;
|
|
298
|
-
/**
|
|
299
|
-
* How much a cardinality guess is worth. It used to be 8, and it should not have been.
|
|
300
|
-
*
|
|
301
|
-
* `isCollection` is a GUESS read off the URL, and the index carries nothing better: of 88
|
|
302
|
-
* read capabilities, zero declare an array in their response schema, and `paginated` — the
|
|
303
|
-
* one signal that looks trustworthy — is true for `get_report_detail`, a single-record
|
|
304
|
-
* endpoint. There is no derived cardinality in this artifact to appeal to.
|
|
305
|
-
*
|
|
306
|
-
* At ±8 that guess swung 16 points and buried 11 correct answers: `/test-runs/closed` and
|
|
307
|
-
* `/{entity}/search` are listings whose last path segment is not a plural noun, `users-v2`
|
|
308
|
-
* is plural with a version suffix in the way, and the count endpoints answer "how many"
|
|
309
|
-
* with a scalar. All were penalised for what they are called.
|
|
310
|
-
*
|
|
311
|
-
* The fix is not a better classifier — a qualifier word-list would encode tm's route
|
|
312
|
-
* conventions into generic code and rot on the next product. It is to stop betting so much
|
|
313
|
-
* on an unreliable signal. Swept against the eval, holding top1 at 138 with no ceiling
|
|
314
|
-
* violations: ±7 retires 1, ±6 retires 2, ±5 retires 5, ±4 retires 6. Below that it starts
|
|
315
|
-
* costing more than it returns — ±3.5 retires 8 but breaks a case, ±3 breaks two.
|
|
316
|
-
*/
|
|
317
|
-
const CARDINALITY = 4;
|
|
318
|
-
/**
|
|
319
|
-
* The mode penalty stays at 20, unlike the cardinality one — it is earned.
|
|
320
|
-
*
|
|
321
|
-
* Where `isCollection` is a guess off the URL, `modeHint` is measured right: across the
|
|
322
|
-
* eval it agrees with the correct answer's mode 125 times and disagrees 3. A signal that
|
|
323
|
-
* accurate deserves to be decisive.
|
|
324
|
-
*
|
|
325
|
-
* Sweeping it 20 -> 6 retires none of the three misses filed against it and regresses
|
|
326
|
-
* nothing, which says the penalty is not what holds them back. It isn't: they sit at ranks
|
|
327
|
-
* 41, 62 and 78 of the matched set, far below anything a constant could lift. All three
|
|
328
|
-
* are vocabulary gaps wearing a mode-penalty label — "what gets removed" wants a path
|
|
329
|
-
* spelled `rm-summary`, "get rid of" wants `delete`, and "the option set" wants what the
|
|
330
|
-
* product calls a `dataset`. The target barely matches on TERMS; the hint is incidental.
|
|
331
|
-
*
|
|
332
|
-
* Left at 20 deliberately. There was no evidence for moving it, and an unjustified constant
|
|
333
|
-
* is how this scorer got into trouble in the first place.
|
|
334
|
-
*/
|
|
335
|
-
const MODE_PENALTY = 20;
|
|
336
228
|
function score(capability, wanted, weights, aliases, hint, plural) {
|
|
337
229
|
if (wanted.length === 0)
|
|
338
230
|
return { matched: 1, ranked: 1 };
|
|
@@ -381,85 +273,14 @@ function score(capability, wanted, weights, aliases, hint, plural) {
|
|
|
381
273
|
}
|
|
382
274
|
}
|
|
383
275
|
const matched = ranked;
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
// correctly — values it near nothing, and every project-scoped listing scored the same as
|
|
389
|
-
// the projects listing itself. The top 8 for "list all projects" spanned 15.9 to 14.8,
|
|
390
|
-
// where +8 collection and +6 mode already account for 14: the term signal was ~1 point of
|
|
391
|
-
// noise and the right answer sat 7th.
|
|
392
|
-
//
|
|
393
|
-
// Flat, and deliberately NOT rarity-scaled. Rarity would reintroduce the same problem in
|
|
394
|
-
// reverse — a rare scope noun outranking the real target, which is exactly how a weighted
|
|
395
|
-
// version of this put `/projects/{id}/folders` above `/folder/{id}/test-cases` for "tc list
|
|
396
|
-
// for a folder". This asks one yes/no question instead: is the caller's own word the last
|
|
397
|
-
// thing in the path?
|
|
398
|
-
//
|
|
399
|
-
// Equality is against the QUERY's forms, never the haystack's — the same one-directional
|
|
400
|
-
// rule as containment. `projects` is in forms("projects"), so the projects listing hits;
|
|
401
|
-
// `folders` is not in forms("folder"), so a folder-scoped query does not drag in the
|
|
402
|
-
// folders listing.
|
|
403
|
-
const tail = terms(resourceText(capability));
|
|
404
|
-
if (tail.length &&
|
|
405
|
-
wanted.some((forms) => tail.every((word) => forms.includes(word)))) {
|
|
406
|
-
ranked += RESOURCE_BONUS;
|
|
407
|
-
}
|
|
408
|
-
if (hint)
|
|
409
|
-
ranked += satisfiesHint(capability.mode, hint) ? 6 : -MODE_PENALTY;
|
|
276
|
+
if (hint && capability.mode !== hint)
|
|
277
|
+
ranked -= 20;
|
|
278
|
+
else if (hint && capability.mode === hint)
|
|
279
|
+
ranked += 6;
|
|
410
280
|
if (plural)
|
|
411
|
-
ranked += isCollection(capability) ?
|
|
281
|
+
ranked += isCollection(capability) ? 8 : -8;
|
|
412
282
|
return { matched, ranked };
|
|
413
283
|
}
|
|
414
|
-
/**
|
|
415
|
-
* Below this, the caller is probably not speaking the product's language.
|
|
416
|
-
*
|
|
417
|
-
* Measured on tm: vocabulary misses top out at 1.98–2.15 ("where do my things live", "make a
|
|
418
|
-
* new bucket for my tests") while queries that work reach 6.5–12. The two ranges do NOT
|
|
419
|
-
* separate cleanly — "list all projects" scores 1.58 and is nonetheless answered correctly
|
|
420
|
-
* at rank 1, because `projects` is in 156 of 173 paths and worth almost nothing.
|
|
421
|
-
*
|
|
422
|
-
* So this is set generously and false positives are accepted, which is sound only because
|
|
423
|
-
* the vocabulary block is ADDITIVE: it arrives next to the results, never instead of them.
|
|
424
|
-
* Over-triggering costs ~1.5KB against a response that is routinely 38KB; under-triggering
|
|
425
|
-
* costs the caller a wrong answer with no hint that it is wrong. Those are not symmetric.
|
|
426
|
-
*/
|
|
427
|
-
const WEAK_MATCH = 3;
|
|
428
|
-
/**
|
|
429
|
-
* The product's vocabulary, for a caller whose words are not the product's words.
|
|
430
|
-
*
|
|
431
|
-
* THE ONE FAILURE LEXICAL SEARCH CANNOT FIX. "make a new bucket for my tests" wants the
|
|
432
|
-
* folder-create capability, and `bucket` appears nowhere in the index — no scoring change
|
|
433
|
-
* reaches it, because the word is simply absent. What CAN reach it is the caller: it is a
|
|
434
|
-
* language model, and given tm's entity list it maps bucket -> folder without effort. It
|
|
435
|
-
* just cannot guess the list unprompted.
|
|
436
|
-
*
|
|
437
|
-
* Aliases only, deliberately. They are the vocabulary map — 19 entities in ~1.5KB, against a
|
|
438
|
-
* response that is routinely 38KB. The entity `key_facts` are richer prose but ten times the
|
|
439
|
-
* size, and a caller who needs them can ask describeEntity once it knows which entity to ask
|
|
440
|
-
* about — which is exactly what this hands over.
|
|
441
|
-
*/
|
|
442
|
-
export function vocabularyOf(products, only) {
|
|
443
|
-
const out = {};
|
|
444
|
-
for (const [name, bundle] of Object.entries(products)) {
|
|
445
|
-
if (only && name !== only)
|
|
446
|
-
continue;
|
|
447
|
-
const entries = [];
|
|
448
|
-
for (const [entity, doc] of Object.entries(bundle.entities || {})) {
|
|
449
|
-
const aliases = (doc.aliases || []);
|
|
450
|
-
entries.push({
|
|
451
|
-
entity,
|
|
452
|
-
...(doc.title
|
|
453
|
-
? { title: doc.title }
|
|
454
|
-
: {}),
|
|
455
|
-
...(aliases.length ? { aliases } : {}),
|
|
456
|
-
});
|
|
457
|
-
}
|
|
458
|
-
if (entries.length)
|
|
459
|
-
out[name] = entries;
|
|
460
|
-
}
|
|
461
|
-
return out;
|
|
462
|
-
}
|
|
463
284
|
export function searchCapabilities(products, query, options = {}) {
|
|
464
285
|
const limit = options.limit && options.limit > 0 ? options.limit : 8;
|
|
465
286
|
// Forms are computed once per query, not per capability: 173 capabilities x 6 haystacks
|
|
@@ -511,17 +332,11 @@ export function searchCapabilities(products, query, options = {}) {
|
|
|
511
332
|
}
|
|
512
333
|
}
|
|
513
334
|
scored.sort((a, b) => b.ranked - a.ranked || a.capability.path.localeCompare(b.capability.path));
|
|
514
|
-
// Ranked order decides the page; the best TERM score decides confidence. They are
|
|
515
|
-
// different questions: the mode and cardinality constants can lift a weakly-matched
|
|
516
|
-
// capability to the top of a page that is entirely wrong.
|
|
517
|
-
const topMatched = scored.reduce((best, s) => Math.max(best, s.matched), 0);
|
|
518
335
|
return {
|
|
519
336
|
hits: scored
|
|
520
337
|
.slice(0, limit)
|
|
521
338
|
.map(({ product, capability }) => ({ product, capability })),
|
|
522
339
|
truncated: scored.length > limit,
|
|
523
340
|
total_matched: scored.length,
|
|
524
|
-
top_matched: topMatched,
|
|
525
|
-
weak: wanted.length > 0 && topMatched < WEAK_MATCH,
|
|
526
341
|
};
|
|
527
342
|
}
|
|
@@ -67,17 +67,6 @@ export interface ResponseDoc extends ComponentRef {
|
|
|
67
67
|
}
|
|
68
68
|
/** A capability, keyed by the endpoint it exposes. There is deliberately no name. */
|
|
69
69
|
export interface Capability {
|
|
70
|
-
/**
|
|
71
|
-
* The caller-facing handle — the snake-cased `operationId`, unique within its product.
|
|
72
|
-
*
|
|
73
|
-
* This is what `invokeCapability` takes, and what `entities[*].capabilities[]` lists. It
|
|
74
|
-
* is stable where a route is not: `/edit` becoming `/edit-v2` must not break a caller
|
|
75
|
-
* holding a handle.
|
|
76
|
-
*
|
|
77
|
-
* OPTIONAL because not every product publishes it yet — loadtesting's index carries none.
|
|
78
|
-
* For those, the endpoint remains the handle, so nothing may assume this is present.
|
|
79
|
-
*/
|
|
80
|
-
name?: string;
|
|
81
70
|
method: string;
|
|
82
71
|
path: string;
|
|
83
72
|
mode: Mode;
|
|
@@ -119,19 +108,6 @@ export interface EntityDoc {
|
|
|
119
108
|
entity?: string;
|
|
120
109
|
via?: string;
|
|
121
110
|
}[];
|
|
122
|
-
/**
|
|
123
|
-
* What a caller gets wrong about this ENTITY, as opposed to one operation.
|
|
124
|
-
*
|
|
125
|
-
* The entity-wide half of guidance: "a run's `id` is the display string, take `uuid`"
|
|
126
|
-
* holds for every capability returning a run, so it is stated once here rather than
|
|
127
|
-
* copied onto each capability's `guidance` — which would repeat across ~20 records and,
|
|
128
|
-
* since guidance is a search haystack, dilute ranking with boilerplate.
|
|
129
|
-
*
|
|
130
|
-
* Filtered at build time and often absent: of tm's 155 authored facts, 73 are publishable
|
|
131
|
-
* as written and the rest name routes, HTTP verbs or wire-shaped parameters. Absent means
|
|
132
|
-
* "none passed", never an error.
|
|
133
|
-
*/
|
|
134
|
-
key_facts?: string[];
|
|
135
111
|
[key: string]: unknown;
|
|
136
112
|
}
|
|
137
113
|
/**
|