@browserstack/mcp-server 1.5.0-beta.15 → 1.5.0-beta.17

package/dist/tools/capability-registry/search.js

@@ -145,6 +145,36 @@ export function termForms(term) {
         add(term.slice(0, -1));
     return forms;
 }
+/**
+ * ONE canonical spelling of a word, so two spellings of a noun cannot look like two terms.
+ *
+ * `termForms` is deliberately generous — it offers every candidate and lets substring
+ * containment sort them out. Canonicalising needs the opposite: exactly one answer, and
+ * the right one. Taking the SHORTEST candidate is what a first cut did, and it folded
+ * `cases` to `cas`, so the vocabulary entry `test case` matched no query containing "test
+ * cases" — the phrase it exists for. The ambiguity check then fell through to `test`
+ * alone, reaching the right verdict by the wrong route.
+ *
+ * Strip `es` only after a sibilant, where English actually inserts it (`boxes`,
+ * `batches`, `statuses`). Otherwise strip the single `s`, which is right for the `-e`
+ * plurals this vocabulary is full of: case, suite, phase, template.
+ */
+export function singular(word) {
+    if (word.endsWith("ss"))
+        return word;
+    // histories -> history. Without this the alias `histories` and the entity word
+    // `history` are two unrelated terms, which is the same defect in another spelling.
+    if (word.endsWith("ies") && word.length > 4)
+        return `${word.slice(0, -3)}y`;
+    // Only where the `e` is genuinely inserted. `s` is NOT in this set: `cases` ends in
+    // `ses` and is `case` + `s`, not `cas` + `es`, and no rule can tell it from `statuses`
+    // by suffix alone — so the commoner reading wins and `-s` is stripped below.
+    if (/(?:ch|sh|x|z)es$/.test(word))
+        return word.slice(0, -2);
+    if (word.endsWith("s") && word.length > 3)
+        return word.slice(0, -1);
+    return word;
+}
 /** A haystack as one lowercased, space-separated string, ready for containment tests. */
 function haystack(text) {
     return terms(text).join(" ");
@@ -432,50 +462,88 @@ function score(capability, wanted, weights, aliases, hint, plural) {
  * that work at 0.44–1.94. The absolute score cannot separate those (misses reach 2.9,
  * good queries drop to 1.9); this does, with the gap in the same place in all three
  * corpora, which is the property that was missing.
+ *
+ * RECALIBRATED AT v1.14, and the reason matters more than the number. The ratio is
+ * corpus-relative and still is — "make a new bucket for my tests" scores 0.259 against tm
+ * alone and 0.258 against both products, which is the property this was built for. What
+ * moved is neither corpus size nor query: it is how much TEXT each capability carries.
+ * v1.13 gave all 242 entries guidance and tripled the length of `intent`, so the numerator
+ * — the best hit's term score — roughly doubled, while the denominator depends only on the
+ * rarity of the query's own terms and did not. Both bands shifted up together: genuine
+ * misses now land at 0.00–0.46 and working queries at 0.24–4.16.
+ *
+ * At the old 0.25 the hand-off had silently stopped firing — "bucket", which is nobody's
+ * word for a folder, came in at 0.259 and read as a confident answer. Re-measured over 173
+ * served eval cases against five queries the product has no words for, 0.50 catches 5 of 5
+ * while flagging 3 of 173 served queries (1.7%), which is the same trade the original cut
+ * made. Expect to move it again the next time the artifact's text volume changes; the
+ * threshold tracks prose per capability, not the number of capabilities.
  */
-const WEAK_COVERAGE = 0.25;
+const WEAK_COVERAGE = 0.5;
 /** The heaviest field, and so the yardstick a perfect match is measured against. */
 const IDENTITY_WEIGHT = 6;
-/** Slots a matching product is guaranteed, before the rest of the page fills by rank. */
-const PRODUCT_FLOOR = 2;
 /**
- * Fill the page by rank, but never let one product's SIZE shut another out entirely.
- *
- * Scores are comparable across products — rarity spans the whole corpus by design — but
- * the number of chances to score is not: tm has 173 capabilities to Load Testing's 20. On
- * a query using words both products share ("test", "config", "tag"), tm simply has more
- * entries near the top and takes the page. Measured before this: "add a tag to xyz test"
- * returned tm tm LT tm tm, and an agent reading the first row went to the wrong product.
- *
- * So each product that matched at all is guaranteed a couple of slots, and everything else
- * is still strict rank order. This does NOT reorder anything or touch scoring — the best
- * hit stays the best hit — it only refuses to let a product be invisible because it is
- * small. The cost when a query really is single-product is a row or two of another
- * product's shortlist, which since the split is a few hundred bytes.
+ * Products whose OWN vocabulary the query hits, when no word in the query settles it.
+ *
+ * Requiring `product` made every CALL unambiguous and did nothing about the agent making
+ * two of them: "list all projects" was answered by searching tm, then Load Testing, then
+ * merging — a question about which product the user meant, answered by guessing both.
+ * Tool-description prose asking the agent to check with the user is a suggestion it is
+ * free to decline, and declining is cheaper than interrupting someone.
+ *
+ * WHOLE VOCABULARY ENTRIES, never their constituent words. Splitting them makes almost
+ * everything look shared: Load Testing's "load test" and tm's "test case" both yield
+ * `test`, which flagged 135 of the 198 eval queries. Matching entries whole flags 27.
+ *
+ * A UNIQUE TERM SETTLES IT. "create a test case in a folder" contains `folder`, which
+ * both products claim, and `test case`, which only tm does — so the query has already
+ * answered the question and there is nothing to ask. Only queries whose every recognised
+ * word is shared are genuinely undecided.
  */
-function withEveryProductRepresented(scored, limit) {
-    if (scored.length <= limit)
-        return scored;
-    const products = new Set(scored.map((s) => s.product));
-    if (products.size < 2)
-        return scored.slice(0, limit);
-    const taken = new Set();
-    // Reserve first, so a product near the bottom of the ranking still gets its footing.
-    for (const product of products) {
-        for (const hit of scored
-            .filter((s) => s.product === product)
-            .slice(0, PRODUCT_FLOOR)) {
-            if (taken.size < limit)
-                taken.add(hit);
+export function ambiguousProducts(products, query) {
+    const none = { products: [], terms: [] };
+    if (terms(query).length === 0)
+        return none;
+    /**
+     * One spelling per word, so a plural cannot masquerade as a different term.
+     *
+     * Both halves need it. On the QUERY side, the entry is `project` and the user types
+     * "list all projects" — the reported case, which exact containment missed. On the
+     * VOCABULARY side the same fold is what keeps the answer honest: tm lists both `report`
+     * and `reports` as aliases, and treating them as two entries made `reports` look
+     * tm-exclusive, so "show me the report" read as settled when both products claim it.
+     */
+    const fold = (text) => terms(text).map(singular).join(" ");
+    const asked_ = ` ${fold(query || "")} `;
+    const owners = new Map();
+    for (const [product, entries] of Object.entries(vocabularyOf(products))) {
+        for (const entry of entries) {
+            for (const word of [entry.entity, ...(entry.aliases ?? [])]) {
+                const key = fold(String(word));
+                if (!key)
+                    continue;
+                if (!owners.has(key))
+                    owners.set(key, new Set());
+                owners.get(key).add(product);
+            }
         }
     }
-    for (const hit of scored) {
-        if (taken.size >= limit)
-            break;
-        taken.add(hit);
+    const shared = new Map();
+    for (const [key, claimants] of owners) {
+        if (!asked_.includes(` ${key} `))
+            continue;
+        // A term only one product claims decides the query outright.
+        if (claimants.size < 2)
+            return none;
+        shared.set(key, claimants);
     }
-    // Emit in the original ranked order: the floor decides WHO appears, never in what order.
-    return scored.filter((hit) => taken.has(hit));
+    if (shared.size === 0)
+        return none;
+    const claimed = new Set();
+    for (const claimants of shared.values())
+        for (const product of claimants)
+            claimed.add(product);
+    return { products: [...claimed].sort(), terms: [...shared.keys()].sort() };
 }
 /**
  * The product's vocabulary, for a caller whose words are not the product's words.
@@ -486,10 +554,14 @@ function withEveryProductRepresented(scored, limit) {
  * language model, and given tm's entity list it maps bucket -> folder without effort. It
  * just cannot guess the list unprompted.
  *
- * Aliases only, and `title` dropped as a near-duplicate of `entity` ("Test run" next to
- * `test_run` buys nothing). The entity `key_facts` are richer prose but ten times the size,
- * and a caller who needs them can ask describeEntity once it knows which entity to ask
- * about — which is exactly what this hands over.
+ * Aliases and a one-line `description`; `title` stays dropped as a near-duplicate of
+ * `entity` ("Test run" next to `test_run` buys nothing). Aliases alone route but do not
+ * DEFINE — `result -> outcome, execution-result, test-result` never says whether that is
+ * the per-case verdict inside a run or a run-level rollup — so an agent holding names and
+ * aliases has exactly one way to find out, which is describeEntity once per entity. For
+ * tm that is 19 calls at ~1.4KB each, ~27KB, to answer what ~1.6KB of description answers
+ * here for every entity at once. The `key_facts` remain out: ten times the size, and the
+ * caller can ask describeEntity for the one entity it settles on.
  *
  * Its share of the response grew when search became a shortlist: 3.2KB against a 38KB full
  * search was 8%, against a 6.8KB shortlist it is nearly half. The absolute cost did not
@@ -504,8 +576,12 @@ export function vocabularyOf(products, only) {
         const entries = [];
         for (const [entity, doc] of Object.entries(bundle.entities || {})) {
             const aliases = (doc.aliases || []);
+            const description = doc.description;
             entries.push({
                 entity,
+                // Before the aliases, because it is what the reader needs first: what the thing
+                // is, then what else it is called.
+                ...(description ? { description } : {}),
                 ...(aliases.length ? { aliases } : {}),
             });
         }
@@ -575,7 +651,9 @@ export function searchCapabilities(products, query, options = {}) {
     const perfect = weights.reduce((sum, w) => sum + IDENTITY_WEIGHT * w, 0);
     const coverage = perfect > 0 ? topMatched / perfect : 1;
     return {
-        hits: withEveryProductRepresented(scored, limit).map(({ product, capability }) => ({ product, capability })),
+        hits: scored
+            .slice(0, limit)
+            .map(({ product, capability }) => ({ product, capability })),
         truncated: scored.length > limit,
         total_matched: scored.length,
         top_matched: topMatched,

package/dist/tools/capability-registry/types.d.ts

@@ -148,6 +148,20 @@ export interface Capability {
 }
 export interface EntityDoc {
     title?: string;
+    /**
+     * One line saying what this entity IS. Rides on `listProducts`, next to the aliases.
+     *
+     * Aliases route; they do not define. `version -> history, revision, version-history`
+     * tells a caller which words land here and nothing about whether that is the versioning
+     * of a test case or of a project. Without this line the only way to find out is
+     * `describeEntity`, once per entity — 19 calls at ~1.4KB for tm, against ~1.6KB to
+     * answer it for all of them at once.
+     *
+     * Capped at 140 characters by the build, and dropped rather than truncated when longer.
+     * Often absent: a product that has not authored these emits none (loadtesting has 0 of
+     * 9 today), and absence means "unwritten", never an error.
+     */
+    description?: string;
     aliases?: string[];
     id_convention?: string;
     parents?: string[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@browserstack/mcp-server",
-  "version": "1.5.0-beta.15",
+  "version": "1.5.0-beta.17",
   "description": "BrowserStack's Official MCP Server",
   "mcpName": "io.github.browserstack/mcp-server",
   "main": "dist/index.js",
@@ -15,7 +15,11 @@
     "dev": "tsx watch --clear-screen=false src/index.ts",
     "test": "vitest run",
     "lint": "eslint . --ext .ts",
-    "format": "prettier --write \"src/**/*.ts\""
+    "format": "prettier --write \"src/**/*.ts\"",
+    "eval:index": "tsx scripts/eval-index.mts",
+    "seed:live": "tsx scripts/seed-live.mts",
+    "plan:tests": "tsx scripts/plan-capability-tests.mts",
+    "replay:live": "tsx scripts/replay-live.mts"
   },
   "bin": {
     "browserstack-mcp-server": "dist/index.js"