@astryxdesign/cli 0.7.2 → 0.7.3-canary.06872b4

package/api/search/_features.mjs

@@ -0,0 +1,155 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file The plain signals search's learned searchers score with. Every one
+ * is a deterministic count over the query's words and each entry's own text:
+ * BM25 over the entry's name, its docs text, and its source-built card; how
+ * many query words it covers; whether the query holds its name; the lexical
+ * ranker's score and rank; and the router's guess of the domain the query
+ * wants. A searcher's weights file holds one number per signal (and per
+ * signal and domain); its score is their weighted sum.
+ * @input a query, the lexical ranking, the gathered candidates, static weights
+ * @output ranked entry ids, and the search response built from them
+ * @position api/search; the searchers and the root router import it
+ */
+
+import {DOMAINS, ROUTES, words, fold, queryWords} from './_query-words.mjs';
+import {entryOf} from './_entries.mjs';
+
+/** @typedef {import('./_entries.mjs').EntryTable} EntryTable */
+
+/** The signal names, in weights-file order (before the per-domain copies). */
+export const SIGNALS = [
+  's_score',
+  's_rr',
+  's_in',
+  's_top20',
+  'd_component',
+  'd_hook',
+  'd_doc',
+  'd_template',
+  'd_theme',
+  'bm_name',
+  'bm_desc',
+  'bm_card',
+  'cover',
+  'cover_all',
+  'head',
+  'r_dom',
+  'r_none',
+  'name_in_q',
+  'name_exact',
+];
+
+/** The signals that also get one copy per domain. */
+export const PER_DOMAIN = [
+  's_score',
+  's_rr',
+  'bm_name',
+  'bm_desc',
+  'bm_card',
+  'cover',
+  'r_dom',
+  'name_in_q',
+];
+
+/**
+ * Every entry's signals for one query, as rows of numbers.
+ * @param {EntryTable} table
+ * @param {string} query
+ * @param {{domain: string, name: string, score?: number}[]} lexical the lexical ranking, best first
+ * @param {number[] | null} route the router's class probabilities
+ * @returns {Float64Array[]}
+ */
+export function signals(table, query, lexical, route) {
+  const n = table.ids.length;
+  const S = SIGNALS.length;
+  const rows = table.ids.map(() => new Float64Array(S));
+  const seen = new Set();
+  let r = 0;
+  for (const res of lexical) {
+    const k = entryOf(table, res);
+    if (k === undefined || seen.has(k)) continue;
+    seen.add(k);
+    r++;
+    const x = rows[k];
+    x[0] = (res.score ?? 0) / 100;
+    x[1] = 1 / r;
+    x[2] = 1;
+    x[3] = r <= 20 ? 1 : 0;
+  }
+  for (let k = 0; k < n; k++) rows[k][4 + table.domain[k]] = 1;
+  const qw = [...new Set(queryWords(query))];
+  if (qw.length) {
+    const hits = qw.map(() => new Uint8Array(n));
+    /** @type {const} */ (['name', 'desc', 'card']).forEach((f, j) => {
+      const {post, idf, len, avg} = table.fields[f];
+      qw.forEach((w, a) => {
+        for (const [k, c] of post.get(w) ?? []) {
+          rows[k][9 + j] +=
+            ((idf.get(w) ?? 0) * c * 2.2) /
+            (c + 1.2 * (0.25 + (0.75 * len[k]) / avg));
+          hits[a][k] = 1;
+        }
+      });
+    });
+    for (let k = 0; k < n; k++) {
+      let cov = 0;
+      for (let a = 0; a < qw.length; a++) cov += hits[a][k];
+      rows[k][12] = cov / qw.length;
+      rows[k][13] = cov === qw.length && qw.length > 1 ? 1 : 0;
+      rows[k][14] = hits[qw.length - 1][k];
+    }
+  }
+  if (route) {
+    for (let k = 0; k < n; k++) {
+      rows[k][15] = route[table.domain[k]];
+      rows[k][16] = route[ROUTES.length - 1];
+    }
+  }
+  const all = words(query).map(fold);
+  const spaced = ` ${all.join(' ')} `;
+  const runs = new Set();
+  for (let len = 1; len <= 3; len++) {
+    for (let a = 0; a + len <= all.length; a++)
+      runs.add(all.slice(a, a + len).join(''));
+  }
+  const whole = all.join('');
+  for (let k = 0; k < n; k++) {
+    const nw = table.name[k];
+    if (!nw.length) continue;
+    const j = nw.join('');
+    if (spaced.includes(` ${nw.join(' ')} `) || runs.has(j)) {
+      rows[k][17] = 1;
+      if (j === whole) rows[k][18] = 1;
+    }
+  }
+  return rows;
+}
+
+/**
+ * Ranks the entries by a searcher's weights: the weighted sum of each entry's
+ * standardized signals and per-domain copies; ties go to the earlier id.
+ * @param {EntryTable} table
+ * @param {Float64Array[]} rows
+ * @param {{w: number[], mu: number[], sd: number[]}} weights
+ * @returns {number[]} entry positions, best first
+ */
+export function rank(table, rows, weights) {
+  const {w, mu, sd} = weights;
+  const S = SIGNALS.length;
+  const per = PER_DOMAIN.map(s => SIGNALS.indexOf(s));
+  const score = rows.map((x, k) => {
+    let s = 0;
+    for (let j = 0; j < S; j++) s += ((x[j] - mu[j]) / sd[j]) * w[j];
+    const d = table.domain[k];
+    per.forEach((j, p) => {
+      for (let dd = 0; dd < DOMAINS.length; dd++) {
+        const c = S + p * DOMAINS.length + dd;
+        s += (((dd === d ? x[j] : 0) - mu[c]) / sd[c]) * w[c];
+      }
+    });
+    return Math.round(s * 1e9) / 1e9;
+  });
+  return score.map((s, k) => k).sort((a, b) => score[b] - score[a] || a - b);
+}

package/api/search/_gather-components.d.mts

@@ -0,0 +1,26 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * Build component candidates: core's own tree plus every configured
+ * integration's components.
+ * @param {string} coreDir
+ * @param {string} cwd
+ * @returns {Promise<Candidate[]>}
+ */
+export function gatherComponents(coreDir: string, cwd: string): Promise<Candidate[]>;
+/**
+ * Every component the project can use, Core's and its integrations', with the
+ * keywords its own doc declares: the discovery and doc reads search's own
+ * candidates use, without the import paths and prose a result carries. `build`
+ * reads it to tell a part of a page from a page when its search was narrowed
+ * away from components.
+ * @param {string} coreDir
+ * @param {string} cwd
+ * @returns {Promise<{name: string, keywords: string[]}[]>}
+ */
+export function componentKeywords(coreDir: string, cwd: string): Promise<{
+    name: string;
+    keywords: string[];
+}[]>;
+export type Candidate = import("./_lexical.mjs").Candidate;

package/api/search/_gather-components.mjs

@@ -0,0 +1,284 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Search's component candidates: Core's components and the ones
+ * integrations contribute, read from their docs. Imported only by the adapter.
+ */
+
+import {discoverExternalPackages} from '../../foundation/fs/paths.mjs';
+import {
+  discoverComponents,
+  discoverValidIntegrationComponents,
+  docStem,
+  familyMember,
+  findComponentReadme,
+  forEachIntegrationDoc,
+  integrationMemberEntry,
+  integrationRecordHold,
+  isIntegrationMember,
+  resolveImportPath,
+  resolveIntegrationImportPath,
+  CORE_PACKAGE,
+} from '../../foundation/discovery/component-discovery.mjs';
+import {
+  loadComponentReplacements,
+  loadIntegrationsSafely,
+  resolveLegacyExternalDoc,
+} from '../component/_adapter.mjs';
+import {loadModuleDoc, guidanceFrom} from './_module-doc.mjs';
+
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
+
+/**
+ * Build component candidates from core's own tree: name + keywords +
+ * usage/description from the component's .doc.mjs.
+ * @param {string} coreDir
+ * @returns {Promise<Candidate[]>}
+ */
+async function gatherCoreComponents(coreDir) {
+  const grouped = discoverComponents(coreDir);
+  const names = Object.values(grouped).flat();
+  /** @type {Candidate[]} */
+  const candidates = [];
+  for (const comp of names) {
+    const readme = findComponentReadme(coreDir, comp);
+    /** @type {string[]} */
+    let keywords = [];
+    let description = '';
+    /** @type {string[]} */
+    let guidance = [];
+    if (readme && readme.endsWith('.doc.mjs')) {
+      const loaded = await loadModuleDoc(readme);
+      // A member documented in its family's doc (VStack in Stack's doc) is
+      // indexed from its own entry: the family's keywords and guidance
+      // describe the family, not the member.
+      const doc = familyMember(loaded, comp) ?? loaded;
+      if (doc) {
+        keywords = Array.isArray(doc.keywords) ? doc.keywords : [];
+        description = doc.usage?.description || doc.description || '';
+        guidance = guidanceFrom(doc);
+      }
+    }
+    candidates.push({
+      domain: 'component',
+      name: comp,
+      keywords,
+      description,
+      guidance,
+      _package: CORE_PACKAGE,
+      _import: resolveImportPath(coreDir, comp),
+    });
+  }
+  return candidates;
+}
+
+/**
+ * Build component candidates contributed by the project's configured
+ * integrations (astryx.config's `integrations`): name + keywords +
+ * usage/description from each component's .doc.mjs, same as core. Without
+ * this, an integration component is invisible to `search`/`build` even
+ * though `component --list`/`component <Name>` already resolve it — the two
+ * discovery paths silently disagreed. Also returns, per name, the docs that
+ * hold it by their file name and the loadable docs that only record it.
+ * @param {import('../../foundation/integrations/integrations.mjs').LoadedIntegration[]} loadedIntegrations
+ * @returns {Promise<{candidates: Candidate[], fileNamed: Map<string, Set<string>>, documented: Map<string, Set<string>>}>}
+ */
+async function gatherIntegrationComponents(loadedIntegrations) {
+  /** @type {Candidate[]} */
+  const candidates = [];
+  // What the owner rule reads: the docs that hold a name by their file name,
+  // loadable or not and hidden ones too, and the loadable docs that only
+  // record it.
+  /** @type {Map<string, Set<string>>} */
+  const fileNamed = new Map();
+  /** @type {Map<string, Set<string>>} */
+  const documented = new Map();
+  /** @param {Map<string, Set<string>>} map @param {string} name @param {string} docPath */
+  const hold = (map, name, docPath) =>
+    map.set(name, (map.get(name) ?? new Set()).add(docPath));
+  for (const integration of loadedIntegrations) {
+    const {components, discovered} =
+      await discoverValidIntegrationComponents(integration);
+    const renamed = new Set(
+      discovered
+        .filter(rec => integrationRecordHold(rec) === 'doc')
+        .filter(rec => !isIntegrationMember(rec))
+        .map(rec => rec.docPath),
+    );
+    forEachIntegrationDoc(integration.components, (docPath, stem) => {
+      if (!renamed.has(docPath)) hold(fileNamed, stem, docPath);
+    });
+    for (const rec of components) {
+      const loaded = await loadModuleDoc(rec.docPath);
+      // A family member is indexed from its own entry: the family's keywords
+      // and guidance describe the family.
+      const member = isIntegrationMember(rec);
+      const doc = member ? integrationMemberEntry(loaded, rec) : loaded;
+      // A doc that names another export of its file still answers to the
+      // file's name, as `component <file name>` does.
+      const fileName = docStem(rec.docPath);
+      if (integrationRecordHold(rec) === 'doc') {
+        hold(documented, rec.name, rec.docPath);
+      }
+      if (renamed.has(rec.docPath) && !member) {
+        hold(documented, fileName, rec.docPath);
+      }
+      candidates.push({
+        domain: 'component',
+        name: rec.name,
+        aliases: !member && fileName !== rec.name ? [fileName] : [],
+        _hold: integrationRecordHold(rec),
+        _docPath: rec.docPath,
+        keywords: doc && Array.isArray(doc.keywords) ? doc.keywords : [],
+        description: doc ? doc.usage?.description || doc.description || '' : '',
+        guidance: guidanceFrom(doc),
+        _package: rec.package ?? integration.name,
+        // Exactly what `component` reports: a doc may state its own specifier
+        // (one entry point exporting several components), and only when it
+        // does not do we resolve the subpath against the owning package's
+        // exports — read off the integration, which the loader already parsed.
+        // Reporting the bare package name here handed out a path that does not
+        // resolve, and disagreed with what `component <Name>` said about the
+        // very same component.
+        _import: resolveIntegrationImportPath(
+          {
+            exportsMap: integration.__packageExports,
+            packageDir: integration.__packageDir,
+            docPath: rec.docPath,
+            packageName: rec.package,
+          },
+          rec.name,
+          loaded?.import,
+        ),
+      });
+    }
+  }
+  return {candidates, fileNamed, documented};
+}
+
+/**
+ * Build component candidates: core's own tree plus every configured
+ * integration's components.
+ * @param {string} coreDir
+ * @param {string} cwd
+ * @returns {Promise<Candidate[]>}
+ */
+export async function gatherComponents(coreDir, cwd) {
+  const loadedIntegrations = await loadIntegrationsSafely(cwd);
+  const [core, gathered, replacements] = await Promise.all([
+    gatherCoreComponents(coreDir),
+    gatherIntegrationComponents(loadedIntegrations),
+    loadComponentReplacements(coreDir, loadedIntegrations),
+  ]);
+  const {candidates: integrations, fileNamed, documented} = gathered;
+  // An active replacement answers to the Core name it replaces, so a search
+  // for that name finds the replacement and not the Core original
+  // (spec:AST-035 FR11).
+  /** @type {Map<string, string>} */
+  const targetOf = new Map(
+    replacements.active.map(active => [
+      `${active.package}\0${active.name}`,
+      active.target,
+    ]),
+  );
+  // The owner rule, as component detail applies it: a candidate `component
+  // <name>` does not open names its package and follows the owner, and a
+  // file-name alias that opens another owner is dropped.
+  const coreNames = new Set(core.map(candidate => candidate.name));
+  /** @param {string} name @param {Candidate} candidate */
+  const opens = (name, candidate) =>
+    !coreNames.has(name) &&
+    findComponentReadme(coreDir, name) == null &&
+    [...(fileNamed.get(name) ?? [])].every(doc => doc === candidate._docPath) &&
+    [...(documented.get(name) ?? [])].every(doc => doc === candidate._docPath);
+  // A legacy docs package keeps an exact name over an export name a doc records.
+  /** @type {ReturnType<typeof discoverExternalPackages> | undefined} */
+  let externals;
+  /** @param {string} name */
+  const legacyDocuments = name =>
+    (externals ??= discoverExternalPackages(cwd)).some(
+      ext => resolveLegacyExternalDoc(ext, name) != null,
+    );
+  const ruled = integrations.map(candidate => {
+    const aliases = (candidate.aliases ?? []).filter(alias =>
+      opens(alias, candidate),
+    );
+    return candidate._hold === 'doc' &&
+      (!opens(candidate.name, candidate) || legacyDocuments(candidate.name))
+      ? {
+          ...candidate,
+          aliases,
+          _command: `astryx component ${candidate.name} --package ${candidate._package}`,
+        }
+      : {...candidate, aliases};
+  });
+  return [
+    ...core.filter(candidate => !replacements.forTarget(candidate.name)),
+    ...[
+      ...ruled.filter(candidate => !candidate._command),
+      ...ruled.filter(candidate => candidate._command),
+    ].map(candidate => {
+      const target = targetOf.get(`${candidate._package}\0${candidate.name}`);
+      if (target != null) {
+        return {...candidate, aliases: [...(candidate.aliases ?? []), target]};
+      }
+      // Another package's component named after a replaced Core component is
+      // shadowed for the bare name (FR14): its command names its package.
+      const shadowedBy = replacements.forTarget(candidate.name);
+      return shadowedBy && shadowedBy.package !== candidate._package
+        ? {
+            ...candidate,
+            _command: `astryx component ${candidate.name} --package ${candidate._package}`,
+          }
+        : candidate;
+    }),
+  ];
+}
+
+/**
+ * Every component the project can use, Core's and its integrations', with the
+ * keywords its own doc declares: the discovery and doc reads search's own
+ * candidates use, without the import paths and prose a result carries. `build`
+ * reads it to tell a part of a page from a page when its search was narrowed
+ * away from components.
+ * @param {string} coreDir
+ * @param {string} cwd
+ * @returns {Promise<{name: string, keywords: string[]}[]>}
+ */
+export async function componentKeywords(coreDir, cwd) {
+  /** @param {any} doc */
+  const keywordsOf = doc => (Array.isArray(doc?.keywords) ? doc.keywords : []);
+  const core = Object.values(discoverComponents(coreDir))
+    .flat()
+    .map(async name => {
+      const readme = findComponentReadme(coreDir, name);
+      const loaded =
+        readme && readme.endsWith('.doc.mjs')
+          ? await loadModuleDoc(readme)
+          : null;
+      return {name, keywords: keywordsOf(familyMember(loaded, name) ?? loaded)};
+    });
+  const loadedIntegrations = await loadIntegrationsSafely(cwd);
+  const replacements = await loadComponentReplacements(
+    coreDir,
+    loadedIntegrations,
+  );
+  const integrations = loadedIntegrations.map(async integration => {
+    const {components} = await discoverValidIntegrationComponents(integration);
+    return Promise.all(
+      components.map(async rec => {
+        const loaded = await loadModuleDoc(rec.docPath);
+        return {
+          name: rec.name,
+          keywords: keywordsOf(integrationMemberEntry(loaded, rec) ?? loaded),
+        };
+      }),
+    );
+  });
+  return [
+    ...(await Promise.all(core)).filter(
+      component => !replacements.forTarget(component.name),
+    ),
+    ...(await Promise.all(integrations)).flat(),
+  ];
+}

package/api/search/_gather-docs.d.mts

@@ -0,0 +1,20 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
+/**
+ * Build doc candidates at the grain a reader reads them: each section of a
+ * topic, whose command reads just that section; each topic as a whole, whose
+ * command lists its sections; and each docs-tree node by its route. The tree's
+ * guides split into sections like topics, and its typed docs also match by
+ * their own name, so `assertResponse` finds `cli/api/functions/assert-response`.
+ *
+ * Reads the project's catalog rather than the CLI's own docs directory, so a
+ * topic an integration contributed (or replaced) is searchable exactly like a
+ * built-in one — otherwise the replacement is served by `astryx docs` but
+ * invisible to the command whose job is finding it.
+ * @param {string} cwd
+ * @returns {Promise<Candidate[]>}
+ */
+export function gatherDocs(cwd: string): Promise<Candidate[]>;
+export type Candidate = import("./_lexical.mjs").Candidate;