@browserstack/mcp-server 1.5.0-beta.15 → 1.5.0-beta.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/capability/loadtesting.capability-index.json +48 -0
- package/capability/tm.capability-index.json +33738 -12144
- package/dist/tools/capability-registry/bind.js +42 -1
- package/dist/tools/capability-registry/egress.js +52 -6
- package/dist/tools/capability-registry/index-loader.js +36 -1
- package/dist/tools/capability-registry/register.d.ts +8 -0
- package/dist/tools/capability-registry/register.js +167 -46
- package/dist/tools/capability-registry/search.d.ts +50 -4
- package/dist/tools/capability-registry/search.js +119 -41
- package/dist/tools/capability-registry/types.d.ts +14 -0
- package/package.json +6 -2
|
@@ -200,6 +200,22 @@ function coerceType(value, param, label = param.name) {
|
|
|
200
200
|
return false;
|
|
201
201
|
throw new InvocationError(`'${label}' must be true or false`);
|
|
202
202
|
}
|
|
203
|
+
// NO DECLARED TYPE: pass the value through as given.
|
|
204
|
+
//
|
|
205
|
+
// The fallthrough below is String(value), which is right for a declared string and
|
|
206
|
+
// silently destructive for anything the spec left untyped. create_report_v2 declares
|
|
207
|
+
// `mail_to` and `report_filters` with no type and the API wants objects for both, so a
|
|
208
|
+
// correct payload was serialised to the literal "[object Object]" and refused:
|
|
209
|
+
//
|
|
210
|
+
// The property '#/mail_to' of type string did not match the following type: object
|
|
211
|
+
//
|
|
212
|
+
// The caller sent the right thing, this layer broke it, and the server's message blames
|
|
213
|
+
// the caller for a string it never wrote. An untyped parameter means the spec declines to
|
|
214
|
+
// say what shape it is — which is a reason to leave it alone, not to guess "string".
|
|
215
|
+
const untyped = !expected;
|
|
216
|
+
if (untyped && (typeof value === "object" || typeof value === "boolean")) {
|
|
217
|
+
return value;
|
|
218
|
+
}
|
|
203
219
|
const text = String(value);
|
|
204
220
|
if (param.values && param.values.length > 0) {
|
|
205
221
|
const allowed = param.values.map((v) => String(v));
|
|
@@ -244,8 +260,33 @@ export function bind(capability, args) {
|
|
|
244
260
|
// misspelled filter would return a larger result set that looks like a correct answer.
|
|
245
261
|
const unknown = Object.keys(supplied).filter((name) => !byName.has(name));
|
|
246
262
|
if (unknown.length > 0) {
|
|
263
|
+
// A CALLER WHO SENT THE WRAPPER WE BUILD DESERVES TO BE TOLD SO.
|
|
264
|
+
//
|
|
265
|
+
// Most of these endpoints want a nested body — Rails `params.require(:test_case)` —
|
|
266
|
+
// and the fields declare that as `json_path: /test_case/name`, so this layer
|
|
267
|
+
// assembles the wrapper itself and the caller sends the fields flat. A caller who
|
|
268
|
+
// wraps it by hand therefore sends a key that is never a declared param, and the
|
|
269
|
+
// bare message reads as "that field does not exist" when the truth is the opposite:
|
|
270
|
+
// it exists, and building it is our job.
|
|
271
|
+
//
|
|
272
|
+
// Worth special-casing because the guidance itself has been telling callers to wrap:
|
|
273
|
+
// 25 tm capabilities carried a line describing the WIRE format to a caller who never
|
|
274
|
+
// writes the wire. Those lines are being corrected, but an agent working from an
|
|
275
|
+
// older artifact, or reasoning from the Rails convention, lands here either way.
|
|
276
|
+
const wrappers = new Set(params
|
|
277
|
+
.map((param) => param.json_path)
|
|
278
|
+
.filter((path) => Boolean(path))
|
|
279
|
+
.map((path) => path.replace(/^\//, "").split("/")[0])
|
|
280
|
+
.filter((segment) => segment && !byName.has(segment)));
|
|
281
|
+
const sentAWrapper = unknown.filter((name) => wrappers.has(name));
|
|
247
282
|
throw new InvocationError(`unknown ${group}: ${unknown.sort().join(", ")}. accepted: ` +
|
|
248
|
-
`${[...byName.keys()].sort().join(", ") || "none"}`
|
|
283
|
+
`${[...byName.keys()].sort().join(", ") || "none"}` +
|
|
284
|
+
(sentAWrapper.length > 0
|
|
285
|
+
? `. ${sentAWrapper.sort().join(" and ")} ` +
|
|
286
|
+
`${sentAWrapper.length > 1 ? "are wrappers" : "is a wrapper"} this ` +
|
|
287
|
+
`surface builds for you from each field's json_path — send the fields ` +
|
|
288
|
+
`directly instead of nesting them`
|
|
289
|
+
: ""));
|
|
249
290
|
}
|
|
250
291
|
for (const [name, raw] of Object.entries(supplied)) {
|
|
251
292
|
const param = byName.get(name);
|
|
@@ -83,6 +83,54 @@ export function authHeaders(credentials, auth = DEFAULT_AUTH) {
|
|
|
83
83
|
throw new InvocationError(`unsupported auth scheme for this product: ` +
|
|
84
84
|
`${JSON.stringify({ type: auth.type, scheme: auth.scheme })}`);
|
|
85
85
|
}
|
|
86
|
+
/** Longest response text kept when it is not JSON. Enough for an error, not a whole page. */
|
|
87
|
+
const TEXT_BODY_LIMIT = 2000;
|
|
88
|
+
/**
|
|
89
|
+
* Read the response, and NEVER silently discard it.
|
|
90
|
+
*
|
|
91
|
+
* This used to parse the body only when the content-type said JSON and return `null`
|
|
92
|
+
* otherwise, which meant an error could arrive as `{status: 500, body: null}` — a status
|
|
93
|
+
* code and nothing else. That is worse than it sounds: an unhandled exception in a Rails
|
|
94
|
+
* app renders `text/html`, so the one case where you most need the message is exactly the
|
|
95
|
+
* case where the content-type is not JSON. It also made two very different situations
|
|
96
|
+
* indistinguishable — "the product sent no message" and "we threw the message away" — and
|
|
97
|
+
* a live probe of `test_case_results_v1` had to leave that ambiguity open in its findings
|
|
98
|
+
* because nothing downstream could tell which had happened.
|
|
99
|
+
*
|
|
100
|
+
* So: JSON is parsed as before. Anything textual is kept as a truncated string. Malformed
|
|
101
|
+
* JSON keeps its raw text rather than becoming `null`, because a body that fails to parse
|
|
102
|
+
* is itself the diagnosis. Binary is described rather than decoded — dumping PDF bytes
|
|
103
|
+
* into an agent's context helps nobody, but knowing a PDF arrived does.
|
|
104
|
+
*/
|
|
105
|
+
async function readBody(response) {
|
|
106
|
+
const contentType = response.headers.get("content-type") || "";
|
|
107
|
+
// Anything that is not plausibly text: report what came back without decoding it.
|
|
108
|
+
const textual = !contentType ||
|
|
109
|
+
/^text\//i.test(contentType) ||
|
|
110
|
+
/\b(json|xml|yaml|csv|javascript|x-www-form-urlencoded)\b/i.test(contentType);
|
|
111
|
+
if (!textual) {
|
|
112
|
+
const size = response.headers.get("content-length");
|
|
113
|
+
return `<non-text response: ${contentType}${size ? `, ${size} bytes` : ""}>`;
|
|
114
|
+
}
|
|
115
|
+
const raw = await response.text().catch(() => "");
|
|
116
|
+
if (!raw.trim())
|
|
117
|
+
return null;
|
|
118
|
+
if (contentType.includes("json")) {
|
|
119
|
+
try {
|
|
120
|
+
return JSON.parse(raw);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Declared JSON that is not JSON. The text is the evidence; keep it.
|
|
124
|
+
return truncate(raw);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return truncate(raw);
|
|
128
|
+
}
|
|
129
|
+
function truncate(text) {
|
|
130
|
+
return text.length <= TEXT_BODY_LIMIT
|
|
131
|
+
? text
|
|
132
|
+
: `${text.slice(0, TEXT_BODY_LIMIT)}… [truncated, ${text.length} chars total]`;
|
|
133
|
+
}
|
|
86
134
|
/** A fetch-based transport. Redirects are NOT followed. */
|
|
87
135
|
export function fetchTransport(timeoutMs = 45_000) {
|
|
88
136
|
return async (method, url, headers, query, body) => {
|
|
@@ -106,12 +154,10 @@ export function fetchTransport(timeoutMs = 45_000) {
|
|
|
106
154
|
redirect: "manual",
|
|
107
155
|
signal: controller.signal,
|
|
108
156
|
});
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
return { status: response.status, body: parsed };
|
|
157
|
+
return {
|
|
158
|
+
status: response.status,
|
|
159
|
+
body: await readBody(response),
|
|
160
|
+
};
|
|
115
161
|
}
|
|
116
162
|
catch {
|
|
117
163
|
// Upstream detail stays out of the reply; the resolver treats status 0 as a failed call.
|
|
@@ -337,7 +337,42 @@ function resolveNode(product, node, seen) {
|
|
|
337
337
|
// revisit a name already on this path.
|
|
338
338
|
if (!target || seen.has(key))
|
|
339
339
|
return node;
|
|
340
|
-
|
|
340
|
+
const resolved = resolveNode(product, target, new Set([...seen, key]));
|
|
341
|
+
// KEYS WRITTEN BESIDE THE REF SURVIVE IT, AND WIN.
|
|
342
|
+
//
|
|
343
|
+
// This used to return the target and drop every sibling, which silently ate 16 nodes
|
|
344
|
+
// in the shipped tm index: 14 authored descriptions and two `nullable: true` flags.
|
|
345
|
+
// The losses are exactly the caveats hardest to rediscover — "only when no step
|
|
346
|
+
// results were submitted", "Present only on root-folder creation", and a `nullable`
|
|
347
|
+
// marking the one field that comes back NULL when a search fails, which is the
|
|
348
|
+
// absent-versus-explicitly-null distinction a caller cannot otherwise make.
|
|
349
|
+
//
|
|
350
|
+
// It also cost real work downstream. A live probe reported one of those caveats as a
|
|
351
|
+
// missing conditional and it had been written all along — this resolver removed it
|
|
352
|
+
// between the author and the caller. Worse, the product team began splitting shared
|
|
353
|
+
// schemas so a caveat would have somewhere to live that survived, trading a lost
|
|
354
|
+
// sentence for two descriptions of one serializer that then drift apart.
|
|
355
|
+
//
|
|
356
|
+
// SIBLINGS WIN over the resolved target, not the reverse. A description written at the
|
|
357
|
+
// reference site is about THAT usage — "only when no step results were submitted" is
|
|
358
|
+
// true of one consumer of TestResult, not of TestResult everywhere — which is the
|
|
359
|
+
// whole reason it was written beside the ref instead of on the schema.
|
|
360
|
+
const siblings = {};
|
|
361
|
+
for (const [field, value] of Object.entries(node)) {
|
|
362
|
+
if (field !== "$schema" && field !== "$response")
|
|
363
|
+
siblings[field] = value;
|
|
364
|
+
}
|
|
365
|
+
if (Object.keys(siblings).length === 0)
|
|
366
|
+
return resolved;
|
|
367
|
+
if (typeof resolved !== "object" ||
|
|
368
|
+
resolved === null ||
|
|
369
|
+
Array.isArray(resolved)) {
|
|
370
|
+
return resolved;
|
|
371
|
+
}
|
|
372
|
+
return {
|
|
373
|
+
...resolved,
|
|
374
|
+
...resolveNode(product, siblings, seen),
|
|
375
|
+
};
|
|
341
376
|
}
|
|
342
377
|
const out = {};
|
|
343
378
|
for (const [field, value] of Object.entries(node)) {
|
|
@@ -18,6 +18,14 @@ import { BrowserStackConfig } from "../../lib/types.js";
|
|
|
18
18
|
import { Credentials, Transport } from "./egress.js";
|
|
19
19
|
import { CapabilityRegistry } from "./index-loader.js";
|
|
20
20
|
export declare const PERMISSION_VALUES: readonly ["not_asked", "granted", "denied"];
|
|
21
|
+
/**
|
|
22
|
+
* The search-side twin of `user_permission`: did a human choose the product, or did you?
|
|
23
|
+
*
|
|
24
|
+
* No `denied`. A refused write is a thing the caller must not do; a refused product choice
|
|
25
|
+
* is not a state — the user either named one, in which case you search it, or has not been
|
|
26
|
+
* asked, in which case there is nothing to search yet.
|
|
27
|
+
*/
|
|
28
|
+
export declare const PRODUCT_CHOICE_VALUES: readonly ["not_asked", "user_confirmed"];
|
|
21
29
|
export interface RegistryDeps {
|
|
22
30
|
registry: CapabilityRegistry;
|
|
23
31
|
/**
|
|
@@ -20,8 +20,16 @@ import { indexPaths, isEnabled, resolveBaseUrl } from "./config.js";
|
|
|
20
20
|
import { fetchTransport } from "./egress.js";
|
|
21
21
|
import { CapabilityRegistry, InvocationError, resolveResponses, } from "./index-loader.js";
|
|
22
22
|
import { invoke } from "./resolve.js";
|
|
23
|
-
import { searchCapabilities, vocabularyOf } from "./search.js";
|
|
23
|
+
import { ambiguousProducts, searchCapabilities, singular, terms, vocabularyOf, } from "./search.js";
|
|
24
24
|
export const PERMISSION_VALUES = ["not_asked", "granted", "denied"];
|
|
25
|
+
/**
|
|
26
|
+
* The search-side twin of `user_permission`: did a human choose the product, or did you?
|
|
27
|
+
*
|
|
28
|
+
* No `denied`. A refused write is a thing the caller must not do; a refused product choice
|
|
29
|
+
* is not a state — the user either named one, in which case you search it, or has not been
|
|
30
|
+
* asked, in which case there is nothing to search yet.
|
|
31
|
+
*/
|
|
32
|
+
export const PRODUCT_CHOICE_VALUES = ["not_asked", "user_confirmed"];
|
|
25
33
|
/**
|
|
26
34
|
* The tool-adder the server factory calls.
|
|
27
35
|
*
|
|
@@ -73,10 +81,13 @@ export function addCapabilityRegistryToolsFromConfig(server, config) {
|
|
|
73
81
|
function ok(payload) {
|
|
74
82
|
return { content: [{ type: "text", text: JSON.stringify(payload) }] };
|
|
75
83
|
}
|
|
76
|
-
function failed(message) {
|
|
84
|
+
function failed(message, extra) {
|
|
77
85
|
return {
|
|
78
86
|
content: [
|
|
79
|
-
{
|
|
87
|
+
{
|
|
88
|
+
type: "text",
|
|
89
|
+
text: JSON.stringify({ ok: false, error: message, ...extra }),
|
|
90
|
+
},
|
|
80
91
|
],
|
|
81
92
|
isError: true,
|
|
82
93
|
};
|
|
@@ -114,9 +125,10 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
114
125
|
// Telemetry must not decide whether a tool call succeeds.
|
|
115
126
|
}
|
|
116
127
|
};
|
|
117
|
-
tools.listProducts = server.tool("listProducts", "List the BrowserStack products this surface can reach
|
|
118
|
-
"
|
|
119
|
-
"
|
|
128
|
+
tools.listProducts = server.tool("listProducts", "List the BrowserStack products this surface can reach: what each one does, the " +
|
|
129
|
+
"entities it models, and a line saying what each entity is. START HERE — " +
|
|
130
|
+
"searchCapability needs a product, and this is what tells you which one. If two " +
|
|
131
|
+
"products could both fit the task, ask the user rather than choosing for them.", {}, {
|
|
120
132
|
title: "List Capability Products",
|
|
121
133
|
readOnlyHint: true,
|
|
122
134
|
destructiveHint: false,
|
|
@@ -124,28 +136,34 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
124
136
|
openWorldHint: false,
|
|
125
137
|
}, async () => {
|
|
126
138
|
track("listProducts");
|
|
127
|
-
const info = registry.buildInfo();
|
|
128
139
|
return ok({
|
|
129
|
-
build_id: registry.buildId,
|
|
130
140
|
products: registry.productNames().map((name) => ({
|
|
131
141
|
name,
|
|
132
142
|
summary: registry.index.products[name].summary,
|
|
133
|
-
// THE
|
|
134
|
-
// alone does not do it: "add a tag to xyz test" reads as
|
|
135
|
-
// you can see that `tag` exists in one and not the other.
|
|
136
|
-
//
|
|
137
|
-
//
|
|
138
|
-
//
|
|
139
|
-
//
|
|
143
|
+
// THE ENTITIES AND WHAT EACH ONE IS. Routing is this tool's whole job, and a
|
|
144
|
+
// product summary alone does not do it: "add a tag to xyz test" reads as
|
|
145
|
+
// either product until you can see that `tag` exists in one and not the other.
|
|
146
|
+
// The one-line description is what makes each name mean something — `version`
|
|
147
|
+
// alone does not say whether it versions a test case or a project — so an
|
|
148
|
+
// agent can choose here instead of calling describeEntity once per entity to
|
|
149
|
+
// find out, which for tm is 19 calls at ~1.4KB apiece.
|
|
140
150
|
//
|
|
141
|
-
//
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
|
|
148
|
-
|
|
151
|
+
// NAME AND DESCRIPTION ONLY. The aliases used to travel here too, and they are
|
|
152
|
+
// the wrong half for this job: they answer "what else is this called", which
|
|
153
|
+
// matters when a search has already failed on vocabulary, not when choosing a
|
|
154
|
+
// product. They still reach the caller at exactly that moment, in
|
|
155
|
+
// searchCapability's weak-match block, where the full vocabulary is the answer
|
|
156
|
+
// rather than 2.5KB of speculative context on every routing call.
|
|
157
|
+
entities: (vocabularyOf(registry.index.products, name)[name] ?? []).map(({ entity, description }) => ({
|
|
158
|
+
entity,
|
|
159
|
+
...(description ? { description } : {}),
|
|
160
|
+
})),
|
|
161
|
+
// NO build_id OR version. They are provenance — for our logs and for cache
|
|
162
|
+
// busting — and capability resolution must never depend on them, which means
|
|
163
|
+
// no caller has anything to do with them. They rode on the one call an agent
|
|
164
|
+
// makes before it knows anything, costing context to say nothing actionable.
|
|
165
|
+
// Still on searchCapability's response, where a support question about which
|
|
166
|
+
// index answered can actually be traced to a result.
|
|
149
167
|
})),
|
|
150
168
|
});
|
|
151
169
|
});
|
|
@@ -182,20 +200,22 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
182
200
|
}
|
|
183
201
|
return ok({ product, entity, ...doc });
|
|
184
202
|
});
|
|
185
|
-
tools.searchCapability = server.tool("searchCapability", "Find endpoints this surface can call, by plain language,
|
|
186
|
-
|
|
187
|
-
|
|
203
|
+
tools.searchCapability = server.tool("searchCapability", "Find endpoints this surface can call, by plain language, within ONE product. " +
|
|
204
|
+
`You must say which: ${productList}. If the task does not name it unambiguously, ` +
|
|
205
|
+
"call listProducts first — it returns what each product does, the entities each " +
|
|
206
|
+
"models, and what every entity means, which is what settles the choice. Where two " +
|
|
207
|
+
"products use the same word for different things, ask the user rather than picking. " +
|
|
188
208
|
"Search matches the product's OWN words, not synonyms. When your words are not the " +
|
|
189
209
|
"product's, the response says so: `weak_match: true` with a `suggested_vocabulary` " +
|
|
190
210
|
"map of the product's entities and their aliases. Results are still returned, but " +
|
|
191
211
|
"treat them as unconfirmed — pick the closest entity from that map and search again " +
|
|
192
212
|
"using its vocabulary, or call describeEntity on it for the fuller picture. That one " +
|
|
193
213
|
"extra round trip is far cheaper than invoking the wrong capability. " +
|
|
194
|
-
"Narrowing with `
|
|
214
|
+
"Narrowing further with `entity` sharpens results. " +
|
|
195
215
|
"THIS IS A SHORTLIST, NOT A CONTRACT. Each result carries only what you need to " +
|
|
196
|
-
"CHOOSE: `name` (the handle), `product` (
|
|
197
|
-
"
|
|
198
|
-
"
|
|
216
|
+
"CHOOSE: `name` (the handle), `product`, `mode` (whether it writes), `intent` and " +
|
|
217
|
+
"`guidance` (what it does and what goes wrong), and `method`/`path` for products " +
|
|
218
|
+
"that publish no name yet. " +
|
|
199
219
|
"It does NOT carry parameters or response shapes. Once you have picked one, call " +
|
|
200
220
|
"describeCapability for its full contract, then invokeCapability. Fetching the " +
|
|
201
221
|
"contract only for the one you chose is the difference between ~1k and ~8.6k tokens " +
|
|
@@ -207,9 +227,31 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
207
227
|
.string()
|
|
208
228
|
.optional()
|
|
209
229
|
.describe("Restrict to one entity (listProducts names them)."),
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
230
|
+
// REQUIRED, and the reason is the tool that is NOT being called. listProducts
|
|
231
|
+
// carries the routing data — each product's purpose, its entities, and what every
|
|
232
|
+
// entity means — but nothing obliged an agent to read it: a search that worked
|
|
233
|
+
// without naming a product meant the routing step could always be skipped, and an
|
|
234
|
+
// optional step in front of a working one is a step that does not happen. Requiring
|
|
235
|
+
// the argument makes the ordering structural instead of advisory.
|
|
236
|
+
//
|
|
237
|
+
// It costs one listProducts call on queries that were already unambiguous — 139 of
|
|
238
|
+
// 147 vocabulary terms resolve to a single product — and buys the eight that are
|
|
239
|
+
// not (run, project, report, result, folder, workspace, execution, history), where
|
|
240
|
+
// the old behaviour was to silently pick whichever product ranked higher. A wasted
|
|
241
|
+
// round trip against a silent wrong-product answer is not a close trade.
|
|
242
|
+
product: productArg().describe(`Which product to search: ${productList}. Call listProducts if the task does ` +
|
|
243
|
+
`not make it obvious, and ask the user when two products could both fit.`),
|
|
244
|
+
// REQUIRED, and self-reported, exactly like `user_permission` on a write. The
|
|
245
|
+
// server cannot see whether you asked anyone; what it can do is refuse to answer a
|
|
246
|
+
// question only the user can settle, and make claiming otherwise an explicit act
|
|
247
|
+
// rather than an omission.
|
|
248
|
+
product_choice: z
|
|
249
|
+
.enum(PRODUCT_CHOICE_VALUES)
|
|
250
|
+
.describe("'user_confirmed' only when the user named the product, or the task names it " +
|
|
251
|
+
"unmistakably. 'not_asked' otherwise — then a query that could mean either " +
|
|
252
|
+
"product is refused and told what to ask, instead of being answered for the " +
|
|
253
|
+
"wrong one. Never search each product in turn and merge the results: that is " +
|
|
254
|
+
"the guess this argument exists to prevent."),
|
|
213
255
|
mode: z
|
|
214
256
|
.enum(["read", "write", "destructive"])
|
|
215
257
|
.optional()
|
|
@@ -221,8 +263,61 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
221
263
|
destructiveHint: false,
|
|
222
264
|
idempotentHint: true,
|
|
223
265
|
openWorldHint: false,
|
|
224
|
-
}, async ({ query, entity, product, mode, limit }) => {
|
|
266
|
+
}, async ({ query, entity, product, product_choice, mode, limit }) => {
|
|
225
267
|
track("searchCapability");
|
|
268
|
+
// THE GATE. Nothing here can tell whether a human was asked — same as the write
|
|
269
|
+
// gate, which also takes the caller's word. What it can do is refuse to answer a
|
|
270
|
+
// question that is genuinely the user's, so that answering it anyway takes a
|
|
271
|
+
// deliberate claim instead of silence.
|
|
272
|
+
//
|
|
273
|
+
// Only when the query itself cannot settle the choice: every recognised word is one
|
|
274
|
+
// both products claim. 27 of the 198 eval queries, against 135 if constituent words
|
|
275
|
+
// counted. `entity` is an explicit narrowing, so a caller that named one has already
|
|
276
|
+
// been specific enough and is not asked again.
|
|
277
|
+
if (product_choice !== "user_confirmed" && !entity) {
|
|
278
|
+
const ambiguity = ambiguousProducts(registry.index.products, query);
|
|
279
|
+
if (ambiguity.products.length > 1) {
|
|
280
|
+
// EVERYTHING NEEDED TO ASK, IN THE REFUSAL. Telling the agent to go and call
|
|
281
|
+
// listProducts costs a round trip and still leaves it composing a question out
|
|
282
|
+
// of nothing — so it tends to guess instead, which is the behaviour being
|
|
283
|
+
// stopped. What makes the choice answerable is what each product calls the
|
|
284
|
+
// shared word and what it means THERE: tm's project owns folders and test
|
|
285
|
+
// cases, Load Testing's does not.
|
|
286
|
+
const options = ambiguity.products.map((name) => {
|
|
287
|
+
const entities = registry.index.products[name]?.entities ?? {};
|
|
288
|
+
const senses = ambiguity.terms
|
|
289
|
+
.map((term) => {
|
|
290
|
+
const key = Object.keys(entities).find((candidate) => [
|
|
291
|
+
candidate,
|
|
292
|
+
...(entities[candidate].aliases ?? []),
|
|
293
|
+
].some((word) => terms(word).map(singular).join(" ") === term));
|
|
294
|
+
const doc = key ? entities[key] : undefined;
|
|
295
|
+
return doc
|
|
296
|
+
? { term, entity: key, means: doc.description }
|
|
297
|
+
: undefined;
|
|
298
|
+
})
|
|
299
|
+
.filter(Boolean);
|
|
300
|
+
return {
|
|
301
|
+
product: name,
|
|
302
|
+
summary: registry.index.products[name]?.summary,
|
|
303
|
+
...(senses.length ? { shared_terms: senses } : {}),
|
|
304
|
+
};
|
|
305
|
+
});
|
|
306
|
+
return failed(`'${query}' could mean ${ambiguity.products.join(" or ")} — ` +
|
|
307
|
+
`${ambiguity.terms.map((t) => `'${t}'`).join(", ")} ` +
|
|
308
|
+
`${ambiguity.terms.length === 1 ? "belongs" : "belong"} to both. ` +
|
|
309
|
+
"Put the choice in `clarify` to the USER in their own terms, wait for an " +
|
|
310
|
+
"answer, then resend with product_choice='user_confirmed'. Do NOT search " +
|
|
311
|
+
"each product in turn and merge the results — that answers the question " +
|
|
312
|
+
"instead of asking it.", {
|
|
313
|
+
clarify: {
|
|
314
|
+
question: `Which product do you mean — ${ambiguity.products.join(" or ")}?`,
|
|
315
|
+
shared: ambiguity.terms,
|
|
316
|
+
options,
|
|
317
|
+
},
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
226
321
|
const { hits, weak, top_matched, ...rest } = searchCapabilities(registry.index.products, query, {
|
|
227
322
|
entity,
|
|
228
323
|
product,
|
|
@@ -230,22 +325,35 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
230
325
|
limit,
|
|
231
326
|
});
|
|
232
327
|
return ok({
|
|
233
|
-
build_id
|
|
328
|
+
// NO build_id. It is provenance — for our logs and for cache busting — and
|
|
329
|
+
// resolution must never depend on it, which is exactly why no caller has anything
|
|
330
|
+
// to do with it. It was also the WHOLE registry's id, so a search scoped to one
|
|
331
|
+
// product still announced every other product's build: metadata about builds the
|
|
332
|
+
// caller did not ask about and cannot act on. The startup log already records
|
|
333
|
+
// what loaded, which is where a question about a stale index gets answered.
|
|
334
|
+
//
|
|
234
335
|
// A SHORTLIST: only what choosing requires. Parameters and response shapes are 86%
|
|
235
336
|
// of a full record and are needed for exactly ONE of the eight — the one the caller
|
|
236
337
|
// picks — so they move to describeCapability. Measured over eight queries: 8.6k
|
|
237
338
|
// tokens a search becomes ~1k, and even describing all eight results still costs
|
|
238
339
|
// slightly less than today.
|
|
239
340
|
//
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
// because Load Testing
|
|
243
|
-
// unaddressable
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
341
|
+
// NO ROUTE, AND NO PRODUCT. Both were justified and both justifications expired.
|
|
342
|
+
//
|
|
343
|
+
// `method`/`path` were unconditional because Load Testing published no names, so
|
|
344
|
+
// its rows would otherwise have been unaddressable. It now names all 20, as tm
|
|
345
|
+
// names all 173 — every row on this surface is reachable by name. Publishing the
|
|
346
|
+
// route anyway contradicts the premise the whole registry rests on: an agent
|
|
347
|
+
// addresses a capability by a handle that outlives the route. They remain as a
|
|
348
|
+
// fallback for a product that ships unnamed capabilities, emitted only for the
|
|
349
|
+
// rows that actually need them, which today is none.
|
|
350
|
+
//
|
|
351
|
+
// `product` was here because results could span products. They cannot: `product`
|
|
352
|
+
// is a required argument, so every row is the product the caller named.
|
|
353
|
+
capabilities: hits.map(({ capability }) => ({
|
|
354
|
+
...(capability.name
|
|
355
|
+
? { name: capability.name }
|
|
356
|
+
: { method: capability.method, path: capability.path }),
|
|
249
357
|
mode: capability.mode,
|
|
250
358
|
entity: capability.entity,
|
|
251
359
|
...(capability.intent ? { intent: capability.intent } : {}),
|
|
@@ -332,11 +440,24 @@ export function addCapabilityRegistryTools(server, deps, config) {
|
|
|
332
440
|
// Dropped rather than overwritten: the raw field holds `{"$response": …}` pointers,
|
|
333
441
|
// and spreading the capability would leak them through whenever the resolved value
|
|
334
442
|
// is absent.
|
|
335
|
-
const { responses: unresolved, ...contract } = capability;
|
|
443
|
+
const { responses: unresolved, method, path, ...contract } = capability;
|
|
336
444
|
void unresolved;
|
|
337
445
|
return ok({
|
|
338
|
-
build_id:
|
|
446
|
+
// No build_id here either, same reason. `product` stays: describeCapability
|
|
447
|
+
// resolves by NAME and its product argument is optional, so the answer has to
|
|
448
|
+
// say whose contract came back.
|
|
339
449
|
product: owner,
|
|
450
|
+
// NO ROUTE, for the same reason the shortlist has none. A named capability is
|
|
451
|
+
// invoked by its name; the route is how WE reach the product, not something
|
|
452
|
+
// the caller acts on, and a contract that shows both invites the caller to
|
|
453
|
+
// hold the half that breaks when `/edit` becomes `/edit-v2`. The parameters
|
|
454
|
+
// below still carry `path_params`, so the caller knows what to supply — it
|
|
455
|
+
// just never sees the template they are substituted into.
|
|
456
|
+
//
|
|
457
|
+
// Emitted only when there is no name to use instead, which is what
|
|
458
|
+
// invokeCapability falls back to for a product that publishes none. No shipped
|
|
459
|
+
// product is in that state today.
|
|
460
|
+
...(capability.name ? {} : { method, path }),
|
|
340
461
|
...contract,
|
|
341
462
|
...(responses ? { responses } : {}),
|
|
342
463
|
});
|
|
@@ -29,6 +29,21 @@ export declare function terms(text: string | undefined): string[];
|
|
|
29
29
|
* for no additional match.
|
|
30
30
|
*/
|
|
31
31
|
export declare function termForms(term: string): string[];
|
|
32
|
+
/**
|
|
33
|
+
* ONE canonical spelling of a word, so two spellings of a noun cannot look like two terms.
|
|
34
|
+
*
|
|
35
|
+
* `termForms` is deliberately generous — it offers every candidate and lets substring
|
|
36
|
+
* containment sort them out. Canonicalising needs the opposite: exactly one answer, and
|
|
37
|
+
* the right one. Taking the SHORTEST candidate is what a first cut did, and it folded
|
|
38
|
+
* `cases` to `cas`, so the vocabulary entry `test case` matched no query containing "test
|
|
39
|
+
* cases" — the phrase it exists for. The ambiguity check then fell through to `test`
|
|
40
|
+
* alone, reaching the right verdict by the wrong route.
|
|
41
|
+
*
|
|
42
|
+
* Strip `es` only after a sibilant, where English actually inserts it (`boxes`,
|
|
43
|
+
* `batches`, `statuses`). Otherwise strip the single `s`, which is right for the `-e`
|
|
44
|
+
* plurals this vocabulary is full of: case, suite, phase, template.
|
|
45
|
+
*/
|
|
46
|
+
export declare function singular(word: string): string;
|
|
32
47
|
export declare function modeHint(query: string | undefined): "" | Mode;
|
|
33
48
|
export declare function wantsCollection(query: string | undefined): boolean;
|
|
34
49
|
/**
|
|
@@ -70,9 +85,36 @@ export interface SearchResult {
|
|
|
70
85
|
coverage: number;
|
|
71
86
|
weak: boolean;
|
|
72
87
|
}
|
|
88
|
+
/** Which products a query could plausibly be about, when more than one could. */
|
|
89
|
+
export interface ProductAmbiguity {
|
|
90
|
+
/** The products that claim the query's words. Empty when the query settles itself. */
|
|
91
|
+
products: string[];
|
|
92
|
+
/** The shared vocabulary that caused it — what to put in front of the user. */
|
|
93
|
+
terms: string[];
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Products whose OWN vocabulary the query hits, when no word in the query settles it.
|
|
97
|
+
*
|
|
98
|
+
* Requiring `product` made every CALL unambiguous and did nothing about the agent making
|
|
99
|
+
* two of them: "list all projects" was answered by searching tm, then Load Testing, then
|
|
100
|
+
* merging — a question about which product the user meant, answered by guessing both.
|
|
101
|
+
* Tool-description prose asking the agent to check with the user is a suggestion it is
|
|
102
|
+
* free to decline, and declining is cheaper than interrupting someone.
|
|
103
|
+
*
|
|
104
|
+
* WHOLE VOCABULARY ENTRIES, never their constituent words. Splitting them makes almost
|
|
105
|
+
* everything look shared: Load Testing's "load test" and tm's "test case" both yield
|
|
106
|
+
* `test`, which flagged 135 of the 198 eval queries. Matching entries whole flags 27.
|
|
107
|
+
*
|
|
108
|
+
* A UNIQUE TERM SETTLES IT. "create a test case in a folder" contains `folder`, which
|
|
109
|
+
* both products claim, and `test case`, which only tm does — so the query has already
|
|
110
|
+
* answered the question and there is nothing to ask. Only queries whose every recognised
|
|
111
|
+
* word is shared are genuinely undecided.
|
|
112
|
+
*/
|
|
113
|
+
export declare function ambiguousProducts(products: Record<string, ProductIndex>, query: string | undefined): ProductAmbiguity;
|
|
73
114
|
/** One entity's caller-facing vocabulary: what it is called, and what else it is called. */
|
|
74
115
|
export interface VocabularyEntry {
|
|
75
116
|
entity: string;
|
|
117
|
+
description?: string;
|
|
76
118
|
aliases?: string[];
|
|
77
119
|
}
|
|
78
120
|
/**
|
|
@@ -84,10 +126,14 @@ export interface VocabularyEntry {
|
|
|
84
126
|
* language model, and given tm's entity list it maps bucket -> folder without effort. It
|
|
85
127
|
* just cannot guess the list unprompted.
|
|
86
128
|
*
|
|
87
|
-
* Aliases
|
|
88
|
-
* `
|
|
89
|
-
*
|
|
90
|
-
*
|
|
129
|
+
* Aliases and a one-line `description`; `title` stays dropped as a near-duplicate of
|
|
130
|
+
* `entity` ("Test run" next to `test_run` buys nothing). Aliases alone route but do not
|
|
131
|
+
* DEFINE — `result -> outcome, execution-result, test-result` never says whether that is
|
|
132
|
+
* the per-case verdict inside a run or a run-level rollup — so an agent holding names and
|
|
133
|
+
* aliases has exactly one way to find out, which is describeEntity once per entity. For
|
|
134
|
+
* tm that is 19 calls at ~1.4KB each, ~27KB, to answer what ~1.6KB of description answers
|
|
135
|
+
* here for every entity at once. The `key_facts` remain out: ten times the size, and the
|
|
136
|
+
* caller can ask describeEntity for the one entity it settles on.
|
|
91
137
|
*
|
|
92
138
|
* Its share of the response grew when search became a shortlist: 3.2KB against a 38KB full
|
|
93
139
|
* search was 8%, against a 6.8KB shortlist it is nearly half. The absolute cost did not
|