@astryxdesign/cli 0.7.2 → 0.7.3-canary.06872b4

package/api/search/_module-doc.mjs

@@ -0,0 +1,78 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Reading a component or hook doc for search: the compiled doc and its
+ * usage guidance. Imported only by the adapter's gatherers.
+ */
+
+import {readDocView} from '../../foundation/doc-compiler/read.mjs';
+
+/** @typedef {import('./_gather-components.mjs').Candidate} Candidate */
+
+/**
+ * A component or hook doc, compiled, or null when it cannot be read.
+ * @param {string} docPath
+ * @param {string} [exportName]
+ * @param {'components' | 'hooks'} [root]
+ * @returns {Promise<any>}
+ */
+export async function loadModuleDoc(
+  docPath,
+  exportName = 'docs',
+  root = 'components',
+) {
+  try {
+    // Support both the stamped default export and the legacy named export.
+    return (
+      (await readDocView(docPath, {
+        root,
+        loader: 'native',
+        exports: ['default', exportName],
+      })) ?? null
+    );
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Build component candidates from core's own tree: name + keywords +
+ * usage/description from the component's .doc.mjs.
+ * @param {string} coreDir
+ * @returns {Promise<Candidate[]>}
+ */
+
+/**
+ * Usage guidance from a component doc: its feature list and its best-practice
+ * advice, flattened to plain strings.
+ *
+ * This is where a reader's vocabulary usually lives. `Banner` calls itself "a
+ * persistent message" and only its guidance names "form errors, system
+ * updates, maintenance notices" — so a search for the words people actually
+ * type finds nothing without it.
+ *
+ * @param {any} doc
+ * @returns {string[]}
+ */
+export function guidanceFrom(doc) {
+  if (!doc) return [];
+  const features = Array.isArray(doc.features) ? doc.features : [];
+  const practices = Array.isArray(doc.usage?.bestPractices)
+    ? doc.usage.bestPractices
+    : [];
+  return [...features, ...practices]
+    .map(entry =>
+      typeof entry === 'string'
+        ? entry
+        : [
+            entry?.title,
+            entry?.text,
+            entry?.description,
+            entry?.do,
+            entry?.dont,
+          ]
+            .filter(Boolean)
+            .join(' '),
+    )
+    .filter(Boolean);
+}

package/api/search/_order.d.mts

@@ -0,0 +1,32 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * The search response: the lexical matches, each band in the learned order.
+ * Nothing is added or dropped, so the count and the scores stay the lexical
+ * ranker's; only the order inside a band, and so which matches fill the
+ * limit, changes.
+ * @param {EntryTable} table
+ * @param {number[]} order entry positions, best first, by the searcher's weights
+ * @param {import('./search.type.mjs').SearchResponse} lexical every lexical match
+ * @param {string} query
+ * @param {number} limit
+ * @returns {import('./search.type.mjs').SearchResponse}
+ */
+export function orderMatches(table: EntryTable, order: number[], lexical: import("./search.type.mjs").SearchResponse, query: string, limit: number): import("./search.type.mjs").SearchResponse;
+/**
+ * An integration's matches that the lexical ranker scored the same keep its
+ * order among themselves: they have no cards, so the learned order would only
+ * guess between them.
+ * @param {Item[]} items
+ * @returns {Item[]}
+ */
+export function integrationTies(items: Item[]): Item[];
+export function bandOf(score: number): number;
+export type Item = {
+    res: any;
+    i: number;
+    band: number;
+    learned: number;
+};
+export type EntryTable = import("./_entries.mjs").EntryTable;

package/api/search/_order.mjs

@@ -0,0 +1,162 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file How a searcher orders the lexical ranker's matches. Search answers
+ * only with what the lexical ranker matched, each with its own score and
+ * reason. A match never moves past one of another score band (BANDS below)
+ * or another kind, doc or not; between runs the lexical order holds. A band
+ * can be wide: 55–99 holds keyword matches beside one-word description hits,
+ * and a synonym name match can score under 55, so inside a band a weaker match
+ * can lead a stronger one when the weights favor it. Inside such a
+ * run the learned order places the matches, with these exceptions: matches
+ * that scored the same keep their lexical order, as one block; an entry the
+ * query names leads; in a query of one or two words, the matches on every
+ * word keep their lexical order; and an integration's matches that scored the
+ * same keep theirs.
+ */
+
+import {entryOf} from './_entries.mjs';
+import {words} from './_query-words.mjs';
+import {searchedComponentsOf} from './_result.mjs';
+
+/** @typedef {import('./_entries.mjs').EntryTable} EntryTable */
+
+/**
+ * Where each lexical band starts, highest first: the whole query as a name,
+ * as a keyword, as a phrase in a title, a title in the query, every word
+ * matched, a strong partial match, a name or keyword match, and below that,
+ * prose alone.
+ */
+const BANDS = [195, 188, 170, 160, 151, 100, 55];
+/** The lowest score of the every-word top tier. */
+const FULL_COVERAGE = 151;
+
+/** @param {number} score */
+export const bandOf = score => {
+  const i = BANDS.findIndex(floor => score >= floor);
+  return i === -1 ? BANDS.length : i;
+};
+
+/**
+ * The search response: the lexical matches, each band in the learned order.
+ * Nothing is added or dropped, so the count and the scores stay the lexical
+ * ranker's; only the order inside a band, and so which matches fill the
+ * limit, changes.
+ * @param {EntryTable} table
+ * @param {number[]} order entry positions, best first, by the searcher's weights
+ * @param {import('./search.type.mjs').SearchResponse} lexical every lexical match
+ * @param {string} query
+ * @param {number} limit
+ * @returns {import('./search.type.mjs').SearchResponse}
+ */
+export function orderMatches(table, order, lexical, query, limit) {
+  const all = lexical.data.results;
+  /** @type {Map<number, number>} */
+  const place = new Map(order.map((k, i) => [k, i]));
+  // Runs of consecutive lexical matches in one band and of one kind (doc or
+  // not) are the only places the learned order acts: a match never crosses
+  // into another band, or past a doc or a non-doc the lexical ranker placed.
+  let run = -1;
+  /** @type {string} */
+  let last = '';
+  const items = all.map((res, i) => {
+    // Each match's own band: the lexical merge can place a weaker match
+    // ahead of a stronger doc, and the two must never share a run.
+    const key = `${bandOf(res.score ?? 0)}:${res.domain === 'doc'}`;
+    if (key !== last) {
+      run++;
+      last = key;
+    }
+    const k = entryOf(table, res);
+    return {
+      res,
+      i,
+      band: run,
+      learned: k === undefined ? Infinity : (place.get(k) ?? Infinity),
+    };
+  });
+  const named = (/** @type {Item} */ it) =>
+    (it.res.score ?? 0) >= 95 &&
+    /^(exact name|plural of the name|name )/.test(it.res.reason ?? '');
+  // Matches the lexical ranker scored the same move as one block, in its
+  // order; the learned position of the block's first match places the block
+  // in its run (not its best member's, which would let one strong member lift
+  // a whole block of weak ones), and an entry the query names leads its run.
+  // Moving each match on its own breaks search's existing rules for tied
+  // matches (`file required` leads with Field, `ci` with the integration
+  // check) and grows answers, so ties move together; where the block lands is
+  // measured as spec:AST-083 FR6 requires.
+  /** @type {Map<string, number>} */
+  const block = new Map();
+  const blockOf = (/** @type {Item} */ it) => `${it.band}:${it.res.score}`;
+  for (const it of items) {
+    const at = named(it) ? -1 : it.learned;
+    const k = blockOf(it);
+    if (!block.has(k) || at === -1) block.set(k, at);
+  }
+  let ranked = [...items].sort(
+    (x, y) =>
+      x.band - y.band ||
+      (block.get(blockOf(x)) ?? 0) - (block.get(blockOf(y)) ?? 0) ||
+      x.i - y.i,
+  );
+  if (words(query).length <= 2) {
+    ranked = keepLexical(ranked, it => (it.res.score ?? 0) >= FULL_COVERAGE);
+  }
+  ranked = integrationTies(ranked);
+  const response = {
+    ...lexical,
+    data: {...lexical.data, results: ranked.slice(0, limit).map(it => it.res)},
+  };
+  const searched = searchedComponentsOf.get(lexical);
+  if (searched) searchedComponentsOf.set(response, searched);
+  return /** @type {import('./search.type.mjs').SearchResponse} */ (response);
+}
+
+/**
+ * @typedef {{res: any, i: number, band: number, learned: number}} Item
+ */
+
+/**
+ * The matches `pick` selects keep the places they hold, filled in lexical
+ * order.
+ * @param {Item[]} items
+ * @param {(item: Item) => boolean} pick
+ * @returns {Item[]}
+ */
+function keepLexical(items, pick) {
+  /** @type {number[]} */
+  const at = [];
+  items.forEach((it, j) => pick(it) && at.push(j));
+  if (at.length < 2) return items;
+  const sorted = at.map(j => items[j]).sort((a, b) => a.i - b.i);
+  const out = [...items];
+  at.forEach((j, n) => (out[j] = sorted[n]));
+  return out;
+}
+
+/**
+ * An integration's matches that the lexical ranker scored the same keep its
+ * order among themselves: they have no cards, so the learned order would only
+ * guess between them.
+ * @param {Item[]} items
+ * @returns {Item[]}
+ */
+export function integrationTies(items) {
+  /** @type {Map<number, number[]>} */
+  const groups = new Map();
+  items.forEach((it, j) => {
+    const pkg = it.res.package ?? '';
+    if (!pkg || pkg.startsWith('@astryxdesign/')) return;
+    const key = it.res.score ?? 0;
+    if (!groups.has(key)) groups.set(key, []);
+    /** @type {number[]} */ (groups.get(key)).push(j);
+  });
+  const out = [...items];
+  for (const at of groups.values()) {
+    if (at.length < 2) continue;
+    const sorted = at.map(j => items[j]).sort((a, b) => a.i - b.i);
+    at.forEach((j, n) => (out[j] = sorted[n]));
+  }
+  return out;
+}

package/api/search/_order.test.mjs

@@ -0,0 +1,72 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Invariants of search's ordering, checked against the lexical ranker on
+ * the real catalog: search answers only with lexical matches, keeps each one's
+ * score and reason, and never moves a match past one of another band or kind.
+ */
+
+import {describe, it, expect} from 'vitest';
+import * as path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {search, searchCore} from './search.mjs';
+import {bandOf} from './_order.mjs';
+
+const cwd = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '../../../..',
+);
+
+/** @param {any} r */
+const keyOf = r => `${r.domain}:${r.name}:${r.command}`;
+/** @param {any} r */
+const runOf = r => `${bandOf(r.score ?? 0)}:${r.domain === 'doc'}`;
+
+const QUERIES = [
+  'account settings',
+  'table of contents',
+  'content',
+  'dark mode',
+  'illustration',
+  'date picker in a card',
+  'how do I add a theme',
+  'button',
+  'side panel that resizes',
+];
+
+describe('search ordering invariants', () => {
+  for (const q of QUERIES) {
+    it(`keeps the lexical matches, bands and kinds for "${q}"`, async () => {
+      const ours = await search(q, {cwd, limit: 50});
+      const lexical = await searchCore(q, {
+        cwd,
+        limit: Number.MAX_SAFE_INTEGER,
+      });
+      expect(ours.data.matchCount).toBe(lexical.data.matchCount);
+      expect(ours.data.results.length).toBeLessThanOrEqual(
+        Math.min(50, lexical.data.matchCount),
+      );
+      /** @type {Map<string, number>} */
+      const at = new Map(lexical.data.results.map((r, i) => [keyOf(r), i]));
+      const byKey = new Map(lexical.data.results.map(r => [keyOf(r), r]));
+      for (const r of ours.data.results) {
+        const own = byKey.get(keyOf(r));
+        expect(own, `${keyOf(r)} is a lexical match`).toBeDefined();
+        expect(r.score).toBe(own?.score);
+        expect(r.reason).toBe(own?.reason);
+      }
+      // Two results of different runs (band or kind) keep the lexical order.
+      const rs = ours.data.results;
+      for (let i = 0; i < rs.length; i++) {
+        for (let j = i + 1; j < rs.length; j++) {
+          if (runOf(rs[i]) === runOf(rs[j])) continue;
+          expect(
+            /** @type {number} */ (at.get(keyOf(rs[i]))) <
+              /** @type {number} */ (at.get(keyOf(rs[j]))),
+            `${keyOf(rs[i])} above ${keyOf(rs[j])}`,
+          ).toBe(true);
+        }
+      }
+    }, 60_000);
+  }
+});

package/api/search/_priority.d.mts

@@ -0,0 +1,50 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
+/**
+ * A match's domain priority: 0 (none), 1, or 2 (a doc the whole query names).
+ * A component, hook, template, or theme with priority ranks ahead of any doc
+ * without it, whatever the two scores are. Docs among themselves rank by
+ * priority, then score; everything else ranks among itself by score.
+ *
+ * A component, hook, template, or theme has priority (1) when one of the
+ * query's words, or the whole query, hits its name or an authored keyword at
+ * {@link STRONG_TOKEN_SCORE} or above, or when it matches every word.
+ *
+ * A doc has priority when the reader asked for that doc:
+ * - 2: the whole query is the topic's name, its last route segment, or its
+ *   own title, as words and whatever the plural (`font setup` is
+ *   typography/font-setup, `side panel` is "Side panels");
+ * - 1: a word of the query is the topic's name (`illustration` in a longer
+ *   question is the illustrations guide);
+ * - 1: a query of two or more words is one of the topic's own declared
+ *   keywords, whatever the plural (`code review` on a guide that declares it);
+ * - 1: the query holds one of the doc's titles whole (`resizable side panels`
+ *   names "Side panels"; `light dark mode button` names "Light/Dark Mode");
+ * - 1: the doc matches every word of the query and one of them is a word of
+ *   its title (`switch to a dark theme` and "Use a theme").
+ * A section answers to its topic's name.
+ *
+ * A doc that matched only by a heading holding the query, by one word, or by
+ * words spread through its text has no priority. Docs are split into many
+ * small topics, each with its own headings, so a common phrase such as
+ * `font size` hits a heading exactly (170) while the component the reader is
+ * after matches one word by keyword (`Text`, 98). Ranked on text alone, every
+ * split adds another doc above the component. Broad reference pages match every word of many queries
+ * in their text the same way. The score stays the text-match strength the
+ * result reports, so callers that gate on it (`build`) see the same numbers.
+ *
+ * @param {string} term - Lowercased search term.
+ * @param {string[]} tokens - Content tokens from tokenizeQuery(term).
+ * @param {Candidate} candidate
+ * @param {{score: number, matched: number, total: number}} hit - The
+ *   candidate's scoreQuery result.
+ * @returns {0 | 1 | 2}
+ */
+export function domainPriority(term: string, tokens: string[], candidate: Candidate, hit: {
+    score: number;
+    matched: number;
+    total: number;
+}, queryWords?: string[]): 0 | 1 | 2;
+export type Candidate = import("./_lexical.mjs").Candidate;

package/api/search/_priority.mjs

@@ -0,0 +1,139 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Domain priority: when a component, hook, template, or theme ranks
+ * ahead of docs the reader did not ask for.
+ */
+
+import {sameWord} from './_words.mjs';
+import {phraseWords, samePhraseWord, samePhrase} from './_query.mjs';
+import {compoundHit, compoundOf} from './_compound.mjs';
+import {
+  bestForToken,
+  FULL_COVERAGE_SCORE,
+  STRONG_TOKEN_SCORE,
+} from './_score.mjs';
+import {TITLE_PHRASE_SCORE, TITLE_IN_QUERY_SCORE} from './_titles.mjs';
+import {scoreCandidate} from './_candidate-score.mjs';
+
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
+
+/**
+ * A match's domain priority: 0 (none), 1, or 2 (a doc the whole query names).
+ * A component, hook, template, or theme with priority ranks ahead of any doc
+ * without it, whatever the two scores are. Docs among themselves rank by
+ * priority, then score; everything else ranks among itself by score.
+ *
+ * A component, hook, template, or theme has priority (1) when one of the
+ * query's words, or the whole query, hits its name or an authored keyword at
+ * {@link STRONG_TOKEN_SCORE} or above, or when it matches every word.
+ *
+ * A doc has priority when the reader asked for that doc:
+ * - 2: the whole query is the topic's name, its last route segment, or its
+ *   own title, as words and whatever the plural (`font setup` is
+ *   typography/font-setup, `side panel` is "Side panels");
+ * - 1: a word of the query is the topic's name (`illustration` in a longer
+ *   question is the illustrations guide);
+ * - 1: a query of two or more words is one of the topic's own declared
+ *   keywords, whatever the plural (`code review` on a guide that declares it);
+ * - 1: the query holds one of the doc's titles whole (`resizable side panels`
+ *   names "Side panels"; `light dark mode button` names "Light/Dark Mode");
+ * - 1: the doc matches every word of the query and one of them is a word of
+ *   its title (`switch to a dark theme` and "Use a theme").
+ * A section answers to its topic's name.
+ *
+ * A doc that matched only by a heading holding the query, by one word, or by
+ * words spread through its text has no priority. Docs are split into many
+ * small topics, each with its own headings, so a common phrase such as
+ * `font size` hits a heading exactly (170) while the component the reader is
+ * after matches one word by keyword (`Text`, 98). Ranked on text alone, every
+ * split adds another doc above the component. Broad reference pages match every word of many queries
+ * in their text the same way. The score stays the text-match strength the
+ * result reports, so callers that gate on it (`build`) see the same numbers.
+ *
+ * @param {string} term - Lowercased search term.
+ * @param {string[]} tokens - Content tokens from tokenizeQuery(term).
+ * @param {Candidate} candidate
+ * @param {{score: number, matched: number, total: number}} hit - The
+ *   candidate's scoreQuery result.
+ * @returns {0 | 1 | 2}
+ */
+export function domainPriority(
+  term,
+  tokens,
+  candidate,
+  hit,
+  queryWords = phraseWords(term),
+) {
+  const words = [term, ...tokens];
+  if (candidate.domain === 'doc') {
+    const topic = String(candidate._topic ?? candidate.name).toLowerCase();
+    // The whole query, read as words, is the topic's route, its last
+    // segment, or its own title, whatever the plural: `side panel` is
+    // layout/side-panels, and `header and footer` is "Headers and footers".
+    // A section's title is a heading inside a topic, not the topic's name.
+    const named = [topic, topic.slice(topic.lastIndexOf('/') + 1)];
+    if (candidate._section == null && candidate.titles?.[0]) {
+      named.push(candidate.titles[0]);
+    }
+    if (named.some(n => samePhrase(term, n))) {
+      return 2;
+    }
+    // A query of two or more words that is one of the doc's own declared
+    // keywords, read as words whatever the plural, is the author naming the
+    // doc for that phrase: `code review` on a guide that declares it. Only
+    // what the author declared counts: the code terms and table cells a
+    // doc's other keywords add (`missing_field`) name what it mentions, not
+    // what it is for. One word is left out too, and a section declares
+    // nothing.
+    if (
+      phraseWords(term).length >= 2 &&
+      candidate._section == null &&
+      (candidate._declared ?? []).some(k => samePhrase(term, String(k)))
+    ) {
+      return 1;
+    }
+    if (
+      words.some(
+        w =>
+          (scoreCandidate(w, {name: topic}, {fuzzy: false})?.score ?? 0) >= 95,
+      )
+    ) {
+      return 1;
+    }
+    // A whole title inside the query is the only path to 160-169.
+    if (hit.score >= TITLE_IN_QUERY_SCORE && hit.score < TITLE_PHRASE_SCORE) {
+      return 1;
+    }
+    // Every word matched, and one of them names the doc in its title.
+    if (
+      hit.score >= FULL_COVERAGE_SCORE &&
+      hit.score < TITLE_IN_QUERY_SCORE &&
+      hit.matched === hit.total
+    ) {
+      const titleWords = phraseWords(candidate.titles?.[0] ?? '');
+      if (
+        (tokens.length ? tokens : [term]).some(t =>
+          titleWords.some(w => samePhraseWord(t, w) || sameWord(t, w)),
+        )
+      ) {
+        return 1;
+      }
+    }
+    return 0;
+  }
+  if (hit.score >= FULL_COVERAGE_SCORE) return 1;
+  const compound = compoundOf.has(candidate)
+    ? compoundOf.get(candidate)
+    : compoundHit(queryWords, candidate);
+  if ((compound?.score ?? 0) >= STRONG_TOKEN_SCORE) {
+    return 1;
+  }
+  const fuzzy = tokens.length <= 1;
+  return words.some(
+    w =>
+      (bestForToken(w, candidate, {fuzzy})?.score ?? 0) >= STRONG_TOKEN_SCORE,
+  )
+    ? 1
+    : 0;
+}

package/api/search/_query-words.d.mts

@@ -0,0 +1,25 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/** Plural folding: one fixed rule per suffix. @param {string} w */
+export function fold(w: string): string;
+/** A name's words: `DateRangeInput` is date, range, input. @param {string} name */
+export function nameWords(name: string): string[];
+/** @param {string} q @returns {string[]} unique words, then adjacent pairs */
+export function queryTokens(q: string): string[];
+/** The router's inputs: the query's tokens plus four shape flags. @param {string} q */
+export function routerTokens(q: string): string[];
+/**
+ * The router's probability for each class: the shared classifier over the
+ * query's tokens.
+ * @param {import('../../foundation/route/route.mjs').ClassifierWeights} weights
+ * @param {string} q
+ * @returns {number[]}
+ */
+export function routeProbs(weights: import("../../foundation/route/route.mjs").ClassifierWeights, q: string): number[];
+/** The domains, in the order the weights files index them. */
+export const DOMAINS: string[];
+/** The router's classes: the domain a query wants, or none. */
+export const ROUTES: string[];
+export function words(text: string): string[];
+export function queryWords(q: string): string[];

package/api/search/_query-words.mjs

@@ -0,0 +1,92 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file How search's searchers and classifier read a query: its words,
+ * folded forms, word pairs, and the classifier's tokens and class
+ * probabilities.
+ */
+
+import {classify} from '../../foundation/route/route.mjs';
+
+/** The domains, in the order the weights files index them. */
+export const DOMAINS = ['component', 'hook', 'doc', 'template', 'theme'];
+
+/** The router's classes: the domain a query wants, or none. */
+export const ROUTES = [...DOMAINS, 'NONE'];
+
+const STOP = new Set(
+  'a an the of for to with and or in on by is it my me i we you this that be as at from use using how what which show make add create get set any can do does not no into via'.split(
+    ' ',
+  ),
+);
+
+const QUESTION = new Set([
+  'how',
+  'what',
+  'why',
+  'when',
+  'where',
+  'can',
+  'does',
+  'is',
+]);
+
+/** @param {string} text @returns {string[]} */
+export const words = text =>
+  (text || '').toLowerCase().match(/[a-z0-9]+/g) ?? [];
+
+/** Plural folding: one fixed rule per suffix. @param {string} w */
+export function fold(w) {
+  if (w.length > 4 && w.endsWith('ies')) return `${w.slice(0, -3)}y`;
+  if (w.length > 3 && w.endsWith('es') && 'sxz'.includes(w[w.length - 3])) {
+    return w.slice(0, -2);
+  }
+  if (w.length > 3 && w.endsWith('s') && !w.endsWith('ss'))
+    return w.slice(0, -1);
+  return w;
+}
+
+/** A name's words: `DateRangeInput` is date, range, input. @param {string} name */
+export function nameWords(name) {
+  const s = name
+    .replace(/[/-]/g, ' ')
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .replace(/([A-Z]+)([A-Z][a-z])/g, '$1 $2');
+  return words(s);
+}
+
+/** The query's content words, folded. @param {string} q */
+export const queryWords = q =>
+  words(q)
+    .filter(w => !STOP.has(w))
+    .map(fold);
+
+/** @param {string} q @returns {string[]} unique words, then adjacent pairs */
+export function queryTokens(q) {
+  const qw = queryWords(q);
+  const pairs = qw.slice(1).map((w, i) => `${qw[i]}_${w}`);
+  return [...new Set([...qw, ...pairs])];
+}
+
+/** The router's inputs: the query's tokens plus four shape flags. @param {string} q */
+export function routerTokens(q) {
+  const ws = words(q);
+  const flags = [`#n${Math.min(ws.length, 6)}`];
+  if (ws.length && QUESTION.has(ws[0])) flags.push('#question');
+  if (q.split(/\s+/).some(w => /^use[a-z]+/.test(w)) || /\buse[A-Z]/.test(q)) {
+    flags.push('#usehook');
+  }
+  if (/[A-Z][a-z]+[A-Z]/.test(q)) flags.push('#camel');
+  return [...queryTokens(q), ...flags, '#bias'];
+}
+
+/**
+ * The router's probability for each class: the shared classifier over the
+ * query's tokens.
+ * @param {import('../../foundation/route/route.mjs').ClassifierWeights} weights
+ * @param {string} q
+ * @returns {number[]}
+ */
+export function routeProbs(weights, q) {
+  return classify(weights, routerTokens(q));
+}