@astryxdesign/cli 0.7.2 → 0.7.3-canary.0bf23bd

package/api/search/ORDERING.md

@@ -0,0 +1,28 @@
+# Search ordering data
+
+`astryx search` decides **what matches**, and each match's score and reason, with the lexical ranker (`_lexical.mjs` and the files it imports). The files in this folder only decide **the order of matches inside one score band**: they never add or drop a match, change a score, or move a match past one of another band or kind (doc or not). The bands are scores 195 and up, 188–194, 170–187, 160–169, 151–159, 100–150, 55–99, and under 55. A band can be wide (55–99 holds keyword matches beside one-word description hits), so inside a band a lower score can lead a higher one.
+
+## Rule
+
+`spec:AST-083` owns search's result order; this file describes the data that implements it. Inside one score band, a searcher's static weights decide the order. Matches with the same score move together as one block, in the lexical ranker's order, placed by the learned position of the block's first match: moving each one on its own breaks search's existing rules for tied matches. Matches never cross bands or kinds. There is no model runtime: every number is a plain static value read at load time.
+
+## Files
+
+| File                                                                                           | What it holds                                                                                                                                                                                                                                 |
+| ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `tree.weights.json`                                                                            | The root classifier: for each query word or word pair, one number per class (component, hook, doc, template, theme, none). The query's tokens vote, and a topic searcher takes the query only when its class wins by the floor in `tree.mjs`. |
+| `_main-searcher.weights.json`, `_doc-searcher.weights.json`, `_template-searcher.weights.json` | One weight, mean and spread per signal for each searcher (see `SIGNALS` and `PER_DOMAIN` in `_features.mjs`). A match's place inside its band is the weighted sum of its standardized signals.                                                |
+| `_cards.json`                                                                                  | One card per Core component and hook: what it is, the words people use for it, and its main props, written from the item's own source. Cards feed one signal, word overlap with the card.                                                     |
+
+The signals are counts over the query and each entry's own text: the lexical score and rank, BM25 over the entry's name, docs text and card, how many query words the entry covers, whether the query holds the entry's name, and the classifier's guess of the domain.
+
+## Status
+
+These files are static and reviewed. They are not computed when the CLI runs, and nothing at run time writes them.
+
+## Refreshing
+
+- A Core component or hook without a card is still matched and scored as usual; inside its band it is ordered by its other signals.
+- When Core adds or renames items, a reviewed change adds or updates their cards and, if needed, refreshed weights.
+- Any refresh must keep every test that pins search's behavior passing, including the ordering invariants in `_order.test.mjs`. It must also keep the `astryx search` output's matches, scores and reasons identical to the lexical ranker's.
+- Any refresh must also be measured as `spec:AST-083` FR6 requires: against the held-out, fresh and synthetic evaluation sets, with no row worse than the previous order (a row is worse when the paired 95% confidence interval of its change lies entirely below zero).

package/api/search/_adapter.d.mts

@@ -1,11 +1,91 @@
 // @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
 // DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
 
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
 /**
  * The Core a typed search reads: the project's, else the release's own,
- * installed into the per-user cache first when needed. Throws as
- * requireCoreDir does.
+ * installed into the per-user cache first when needed (a network install and
+ * an npm subprocess; spec AST-081 FR3). Throws as requireCoreDir does.
  * @param {string} cwd
  * @returns {string}
  */
 export function requireSearchCoreDir(cwd: string): string;
+/**
+ * Gathers every candidate the query's domains hold.
+ *
+ * @param {{cwd?: string, type?: string}} options
+ * @returns {Promise<{all: Candidate[], components: Candidate[], searchedComponents: boolean, outside: boolean, coreSkipped: string | null}>}
+ */
+export function gatherCandidates(options: {
+    cwd?: string;
+    type?: string;
+}): Promise<{
+    all: Candidate[];
+    components: Candidate[];
+    searchedComponents: boolean;
+    outside: boolean;
+    coreSkipped: string | null;
+}>;
+/**
+ * gatherCandidates, reused within one process: a later search in the same
+ * folder, for the same `type`, ranks the candidates an earlier search
+ * gathered when none of the files that gather read, and none of the folders
+ * it listed or probed, changed since (each is checked by its change time and
+ * size, so an edited doc in a linked package is seen on the next call). A
+ * long-running caller (a docs site's search, an agent harness) then pays the
+ * gather once. A one-shot `astryx search` gathers once, as before.
+ * @param {{cwd?: string, type?: string}} options
+ * @returns {Promise<Gathered>}
+ */
+export function gatherCandidatesReused(options: {
+    cwd?: string;
+    type?: string;
+}): Promise<Gathered>;
+/**
+ * The search tree's static data, read once: the entry cards and each node's
+ * weights file.
+ *
+ * These files are static, reviewed data that only order matches inside a score
+ * band; ORDERING.md beside them says what each holds and how
+ * they are refreshed.
+ * @returns {SearchData}
+ */
+export function loadSearchData(): SearchData;
+export { componentKeywords } from "./_gather-components.mjs";
+export type Candidate = import("./_lexical.mjs").Candidate;
+export type Gathered = Awaited<ReturnType<typeof gatherCandidates>>;
+export type SearcherWeights = {
+    w: number[];
+    mu: number[];
+    sd: number[];
+};
+export type SearchData = {
+    /**
+     *  Each Core component's and hook's source-built card: what it is, the words
+     *  people type for it, its main props.
+     */
+    cards: Record<string, {
+        what?: string;
+        synonyms?: string[];
+        props?: string[];
+    }>;
+    /**
+     * The root classifier's weights.
+     */
+    router: import("../../foundation/route/route.mjs").ClassifierWeights;
+    /**
+     * The main searcher's weights.
+     */
+    main: SearcherWeights;
+    /**
+     * The doc searcher's weights.
+     */
+    doc: SearcherWeights;
+    /**
+     * The template searcher's weights.
+     */
+    template: SearcherWeights;
+};
+export type SearchCatalog = Awaited<ReturnType<typeof gatherCandidates>> & {
+    data: SearchData;
+};

package/api/search/_adapter.mjs

@@ -1,25 +1,172 @@
 // Copyright (c) Meta Platforms, Inc. and affiliates.
 
 /**
- * @file Environment adapter for search: locating the Core a search reads,
- * including the network install and npm subprocess that fetch a release's
- * own Core outside a project (spec AST-081 FR3; architecture:cli-surface
- * INV21).
- *
- * @input  the folder a search runs in
- * @output the Core folder a search reads
- * @position packages/cli/api/search — imported by search.mjs only
+ * @file Search's adapter: the only place search reads the environment. It
+ * gathers every candidate from the domains a query asks for (one file per
+ * domain, _gather-*.mjs) and loads the tree's static data before it runs.
  */
 
+import {readFileSync} from 'node:fs';
+import * as path from 'node:path';
 import {requireCoreDir} from '../../foundation/fs/paths.mjs';
+import {recordReads, unchangedSince} from '../../foundation/fs/read-record.mjs';
+import {isOutsideProject} from '../../foundation/env/project-scope.mjs';
+import {coreDirOrThrow} from '../error.mjs';
+import {CORELESS_DOMAINS} from './_query.mjs';
+import {gatherComponents} from './_gather-components.mjs';
+import {gatherHooks} from './_gather-hooks.mjs';
+import {gatherDocs} from './_gather-docs.mjs';
+import {gatherTemplates, gatherThemes} from './_gather-templates.mjs';
+
+export {componentKeywords} from './_gather-components.mjs';
+
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
 
 /**
  * The Core a typed search reads: the project's, else the release's own,
- * installed into the per-user cache first when needed. Throws as
- * requireCoreDir does.
+ * installed into the per-user cache first when needed (a network install and
+ * an npm subprocess; spec AST-081 FR3). Throws as requireCoreDir does.
  * @param {string} cwd
  * @returns {string}
  */
 export function requireSearchCoreDir(cwd) {
   return requireCoreDir(cwd);
 }
+
+/**
+ * Gathers every candidate the query's domains hold.
+ *
+ * @param {{cwd?: string, type?: string}} options
+ * @returns {Promise<{all: Candidate[], components: Candidate[], searchedComponents: boolean, outside: boolean, coreSkipped: string | null}>}
+ */
+export async function gatherCandidates(options) {
+  const {cwd = process.cwd(), type} = options;
+  // `astryx docs` reads docs without @astryxdesign/core, and `astryx theme
+  // list` reads themes without it (bundled themes need no project), so a
+  // search of either must too. Every other domain reads core: asked for by
+  // name, it is an error without core; an open search then covers the docs
+  // and themes alone.
+  const needsCore = !type || !CORELESS_DOMAINS.includes(type);
+  // A domain that reads Core gets it as `component` does: the project's, else
+  // the answering release's own, installed into the per-user cache on first
+  // need (spec AST-081 FR3). Asked for by name, a Core that cannot be had is
+  // the error; an open search answers from the rest and says what it skipped
+  // and why, never silently.
+  /** @type {string | null} */
+  let coreDir = null;
+  /** @type {string | null} */
+  let coreSkipped = null;
+  if (needsCore && type) {
+    coreDir = coreDirOrThrow(requireSearchCoreDir, cwd);
+  } else if (needsCore) {
+    try {
+      coreDir = requireSearchCoreDir(cwd);
+    } catch (error) {
+      coreSkipped = /** @type {Error} */ (error).message;
+    }
+  }
+
+  // Gather candidates from each requested domain in parallel.
+  /** @param {string} d */
+  const wants = d =>
+    (!type && (coreDir != null || CORELESS_DOMAINS.includes(d))) || type === d;
+  const [components, hooks, docTopics, templates, themes] = await Promise.all([
+    wants('component')
+      ? gatherComponents(/** @type {string} */ (coreDir), cwd)
+      : [],
+    wants('hook') ? gatherHooks(/** @type {string} */ (coreDir)) : [],
+    wants('doc') ? gatherDocs(cwd) : [],
+    wants('template') ? gatherTemplates(cwd) : [],
+    wants('theme') ? gatherThemes(cwd) : [],
+  ]);
+
+  const all = [...components, ...hooks, ...docTopics, ...templates, ...themes];
+  // Outside a project a theme cannot be added to an app, so its command forks
+  // the source instead (spec AST-081 FR3).
+  const outside = themes.length > 0 && isOutsideProject(cwd);
+  return {
+    all,
+    components,
+    searchedComponents: wants('component'),
+    outside,
+    coreSkipped,
+  };
+}
+
+/**
+ * @typedef {Awaited<ReturnType<typeof gatherCandidates>>} Gathered
+ * @type {Map<string, {reads: Map<string, string>, value: Promise<Gathered>}>}
+ */
+const gathered = new Map();
+
+/**
+ * gatherCandidates, reused within one process: a later search in the same
+ * folder, for the same `type`, ranks the candidates an earlier search
+ * gathered when none of the files that gather read, and none of the folders
+ * it listed or probed, changed since (each is checked by its change time and
+ * size, so an edited doc in a linked package is seen on the next call). A
+ * long-running caller (a docs site's search, an agent harness) then pays the
+ * gather once. A one-shot `astryx search` gathers once, as before.
+ * @param {{cwd?: string, type?: string}} options
+ * @returns {Promise<Gathered>}
+ */
+export async function gatherCandidatesReused(options) {
+  const cwd = path.resolve(options.cwd ?? process.cwd());
+  const key = `${cwd}\0${options.type ?? ''}`;
+  const hit = gathered.get(key);
+  if (hit && unchangedSince(hit.reads)) return hit.value;
+  /** @type {Map<string, string>} */
+  const seen = new Map();
+  const value = recordReads(seen, () => gatherCandidates({...options, cwd}));
+  try {
+    await value;
+  } catch (error) {
+    if (gathered.get(key)?.value === value) gathered.delete(key);
+    throw error;
+  }
+  // Each path's state was taken as it was first read, so an edit made while
+  // the gather ran shows as a change on the next call.
+  gathered.set(key, {reads: seen, value});
+  return value;
+}
+
+/**
+ * @typedef {{w: number[], mu: number[], sd: number[]}} SearcherWeights
+ * @typedef {object} SearchData
+ * @property {Record<string, {what?: string, synonyms?: string[], props?: string[]}>} cards
+ *   Each Core component's and hook's source-built card: what it is, the words
+ *   people type for it, its main props.
+ * @property {import('../../foundation/route/route.mjs').ClassifierWeights} router The root classifier's weights.
+ * @property {SearcherWeights} main The main searcher's weights.
+ * @property {SearcherWeights} doc The doc searcher's weights.
+ * @property {SearcherWeights} template The template searcher's weights.
+ * @typedef {Awaited<ReturnType<typeof gatherCandidates>> & {data: SearchData}} SearchCatalog
+ */
+
+/** @type {SearchData | null} */
+let searchData = null;
+
+/**
+ * The search tree's static data, read once: the entry cards and each node's
+ * weights file.
+ *
+ * These files are static, reviewed data that only order matches inside a score
+ * band; ORDERING.md beside them says what each holds and how
+ * they are refreshed.
+ * @returns {SearchData}
+ */
+export function loadSearchData() {
+  if (!searchData) {
+    /** @param {string} file */
+    const read = file =>
+      JSON.parse(readFileSync(new URL(file, import.meta.url), 'utf8'));
+    searchData = {
+      cards: read('./_cards.json'),
+      router: read('./tree.weights.json'),
+      main: read('./_main-searcher.weights.json'),
+      doc: read('./_doc-searcher.weights.json'),
+      template: read('./_template-searcher.weights.json'),
+    };
+  }
+  return searchData;
+}

package/api/search/_adapter.test.mjs

@@ -0,0 +1,173 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Tests for search's adapter: a long-running caller reuses the gathered
+ * candidates while their inputs are unchanged (gatherCandidatesReused), and
+ * gathers again when an input changes or for another folder or `type`.
+ *
+ * Whether inputs changed is read-record's answer (tested in
+ * read-record.test.mjs). Other tests running at the same time change shared
+ * folders a gather reads (the temp folder, the repository root), so here the
+ * reuse case holds that answer at "unchanged", and every other case only
+ * checks that a change forces a new gather, which concurrent changes cannot
+ * undo.
+ */
+
+import {describe, it, expect, vi} from 'vitest';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import * as readRecord from '../../foundation/fs/read-record.mjs';
+import {gatherCandidates, gatherCandidatesReused} from './_adapter.mjs';
+
+vi.mock('../../foundation/fs/read-record.mjs', async original => {
+  const real = /** @type {typeof readRecord} */ (await original());
+  return {...real, unchangedSince: vi.fn(real.unchangedSince)};
+});
+const unchangedSince = vi.mocked(readRecord.unchangedSince);
+
+const SLOW = 60_000;
+
+/** A folder with only a package.json: a docs search there reads the CLI's docs. */
+function scratchProject() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-gather-reuse-'));
+  fs.writeFileSync(path.join(dir, 'package.json'), '{"name":"app"}');
+  return dir;
+}
+
+describe('gatherCandidatesReused', () => {
+  it(
+    'reuses the gathered candidates while the inputs are unchanged',
+    async () => {
+      const cwd = scratchProject();
+      const first = await gatherCandidatesReused({cwd, type: 'doc'});
+      unchangedSince.mockReturnValueOnce(true);
+      expect(await gatherCandidatesReused({cwd, type: 'doc'})).toBe(first);
+      expect(first.all.length).toBeGreaterThan(0);
+      // The same candidates a fresh gather finds.
+      expect(JSON.parse(JSON.stringify(first))).toEqual(
+        JSON.parse(JSON.stringify(await gatherCandidates({cwd, type: 'doc'}))),
+      );
+    },
+    SLOW,
+  );
+
+  it(
+    'gathers again when an input file changes',
+    async () => {
+      const cwd = scratchProject();
+      const first = await gatherCandidatesReused({cwd, type: 'doc'});
+      const later = new Date(Date.now() + 5_000);
+      fs.utimesSync(path.join(cwd, 'package.json'), later, later);
+      expect(await gatherCandidatesReused({cwd, type: 'doc'})).not.toBe(first);
+      fs.writeFileSync(
+        path.join(cwd, 'astryx.config.mjs'),
+        'export default {};\n',
+      );
+      expect(await gatherCandidatesReused({cwd, type: 'doc'})).not.toBe(first);
+    },
+    SLOW,
+  );
+
+  it(
+    'keeps one entry per folder and type',
+    async () => {
+      const cwd = scratchProject();
+      const docs = await gatherCandidatesReused({cwd, type: 'doc'});
+      const themes = await gatherCandidatesReused({cwd, type: 'theme'});
+      expect(themes).not.toBe(docs);
+      expect(themes.all.every(c => c.domain === 'theme')).toBe(true);
+      expect(
+        await gatherCandidatesReused({cwd: scratchProject(), type: 'doc'}),
+      ).not.toBe(docs);
+    },
+    SLOW,
+  );
+
+  it(
+    'gathers again when a file the gather read changes, in a linked package too',
+    async () => {
+      // A consumer whose integration is a symlink to a package folder
+      // elsewhere, as a workspace package is.
+      const dir = fs.mkdtempSync(
+        path.join(process.cwd(), '.astryx-gather-reuse-'),
+      );
+      const pkg = fs.mkdtempSync(
+        path.join(os.tmpdir(), 'astryx-linked-widgets-'),
+      );
+      try {
+        fs.writeFileSync(path.join(dir, 'package.json'), '{"name":"consumer"}');
+        fs.writeFileSync(
+          path.join(dir, 'astryx.config.mjs'),
+          "export default { integrations: ['@acme/widgets'] };\n",
+        );
+        fs.mkdirSync(
+          path.join(dir, 'node_modules', '@astryxdesign', 'core', 'src'),
+          {
+            recursive: true,
+          },
+        );
+        fs.mkdirSync(path.join(pkg, 'components'));
+        fs.writeFileSync(
+          path.join(pkg, 'package.json'),
+          '{"name":"@acme/widgets","version":"1.0.0"}',
+        );
+        fs.writeFileSync(
+          path.join(pkg, 'astryx.integration.mjs'),
+          "export default { components: './components' };\n",
+        );
+        fs.writeFileSync(
+          path.join(pkg, 'components', 'FancyGizmo.doc.mjs'),
+          `export default {
+            type: 'component',
+            name: 'FancyGizmo',
+            keywords: ['gizmo'],
+            usage: {description: 'A fancy part.'},
+            props: [],
+          };\n`,
+        );
+        fs.writeFileSync(
+          path.join(pkg, 'components', 'FancyGizmo.tsx'),
+          'export function FancyGizmo() { return null; }\n',
+        );
+        fs.mkdirSync(path.join(dir, 'node_modules', '@acme'));
+        fs.symlinkSync(
+          pkg,
+          path.join(dir, 'node_modules', '@acme', 'widgets'),
+          'dir',
+        );
+        const options = {cwd: dir, type: 'component'};
+        const first = await gatherCandidatesReused(options);
+        expect(first.all.map(c => c.name)).toContain('FancyGizmo');
+
+        // A new component in the linked package: its folder changed.
+        fs.writeFileSync(
+          path.join(pkg, 'components', 'TinySprocket.doc.mjs'),
+          `export default {
+            type: 'component',
+            name: 'TinySprocket',
+            keywords: ['sprocket'],
+            usage: {description: 'A small part.'},
+            props: [],
+          };\n`,
+        );
+        fs.writeFileSync(
+          path.join(pkg, 'components', 'TinySprocket.tsx'),
+          'export function TinySprocket() { return null; }\n',
+        );
+        const second = await gatherCandidatesReused(options);
+        expect(second).not.toBe(first);
+        expect(second.all.map(c => c.name)).toContain('TinySprocket');
+
+        // An edited file the gather read: the linked package's manifest.
+        const later = new Date(Date.now() + 5_000);
+        fs.utimesSync(path.join(pkg, 'package.json'), later, later);
+        expect(await gatherCandidatesReused(options)).not.toBe(second);
+      } finally {
+        fs.rmSync(dir, {recursive: true, force: true});
+        fs.rmSync(pkg, {recursive: true, force: true});
+      }
+    },
+    SLOW,
+  );
+});

package/api/search/_candidate-score.d.mts

@@ -0,0 +1,40 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * Score a single candidate against the search term across name, keywords,
+ * and prose signals. Returns the best (highest) score plus a human reason,
+ * or null if nothing matched above the floor.
+ *
+ * @param {string} term - Lowercased search term.
+ * @param {object} candidate
+ * @param {string} candidate.name - Primary identifier (component/hook name, topic, template name).
+ * @param {string[]} [candidate.aliases] - Other names, scored like the name (a theme's display name).
+ * @param {string} [candidate.domain] - A component, hook, or template name
+ *   also matches typed as words: `command palette` is CommandPalette.
+ * @param {string[]} [candidate.keywords] - Authored intent (componentsUsed, category words).
+ * @param {string[]} [candidate.weakKeywords] - Derived signal (components a page renders).
+ * @param {string} [candidate.description]
+ * @param {string[]} [candidate.prose] - Extra free-text blobs (doc section text, best practices).
+ * @param {string[]} [candidate.guidance] - Usage guidance (features, best practices) — scored a tier below description.
+ * @param {{fuzzy?: boolean, joins?: boolean}} [opts] - `fuzzy`: allow edit-distance (typo) matches. Default true; multi-word queries pass false. `joins`: read a spaced query joined into a name or keyword (default true); off when the query names an entry outright.
+ * @returns {{score: number, reason: string, alias?: 'partial' | 'keyword'} | null}
+ *   `alias` marks a hit that read the query's words joined.
+ */
+export function scoreCandidate(term: string, { name, aliases, domain, keywords, weakKeywords, description, prose, guidance, }: {
+    name: string;
+    aliases?: string[] | undefined;
+    domain?: string | undefined;
+    keywords?: string[] | undefined;
+    weakKeywords?: string[] | undefined;
+    description?: string | undefined;
+    prose?: string[] | undefined;
+    guidance?: string[] | undefined;
+}, { fuzzy, joins }?: {
+    fuzzy?: boolean;
+    joins?: boolean;
+}): {
+    score: number;
+    reason: string;
+    alias?: "partial" | "keyword";
+} | null;