@astryxdesign/cli 0.7.2 → 0.7.3-canary.06872b4

package/api/build/_adapter.d.mts

@@ -32,6 +32,27 @@
  * @returns {Promise<PageTemplate[]>}
  */
 export function loadPageTemplates(cwd: string): Promise<PageTemplate[]>;
+/**
+ * A block the kit can name, as search names a block result.
+ * @typedef {object} BlockTemplate
+ * @property {string} name The block's own id, as search reports it.
+ * @property {string} package The npm package that owns this block.
+ * @property {string} displayName Human-facing name.
+ * @property {string} description What the block is.
+ * @property {string} command `astryx template <id> --type block`, as search prints it.
+ */
+/**
+ * Every ready page template and every block the project can scaffold, from one
+ * discovery: pages as {@link loadPageTemplates} returns them, blocks as
+ * {@link BlockTemplate}. Discovery failures leave both empty.
+ *
+ * @param {string} cwd
+ * @returns {Promise<{pages: PageTemplate[], blocks: BlockTemplate[]}>}
+ */
+export function loadTemplates(cwd: string): Promise<{
+    pages: PageTemplate[];
+    blocks: BlockTemplate[];
+}>;
 /**
  * The components the project can use, Core's and its integrations', each with
  * the keywords its own doc declares: what the ranker reads to tell a part of a
@@ -48,6 +69,18 @@ export function loadComponents(cwd: string): Promise<ComponentWords[]>;
  * @returns {import('./kit/weights.mjs').WeightsFile | null}
  */
 export function loadWeights(): import("./kit/weights.mjs").WeightsFile | null;
+/**
+ * The classifier weights of build's root (tree.weights.json), read once.
+ * @returns {import('../../foundation/route/route.mjs').ClassifierWeights}
+ */
+export function loadTreeWeights(): import("../../foundation/route/route.mjs").ClassifierWeights;
+/**
+ * The block picker's weights (blocks/blocks.weights.json), read once; null
+ * when the file is absent, unreadable, or not the shape the picker reads, in
+ * which case build names its own blocks.
+ * @returns {import('./blocks/blocks.mjs').BlockPickerWeights | null}
+ */
+export function loadBlockPicker(): import("./blocks/blocks.mjs").BlockPickerWeights | null;
 /**
  * Whether a parsed weights file has the shape the kit reads: every row has a
  * weight per candidate, every bias a number per candidate, and three blend
@@ -125,3 +158,28 @@ export type ComponentWords = {
      */
     keywords: string[];
 };
+/**
+ * A block the kit can name, as search names a block result.
+ */
+export type BlockTemplate = {
+    /**
+     * The block's own id, as search reports it.
+     */
+    name: string;
+    /**
+     * The npm package that owns this block.
+     */
+    package: string;
+    /**
+     * Human-facing name.
+     */
+    displayName: string;
+    /**
+     * What the block is.
+     */
+    description: string;
+    /**
+     * `astryx template <id> --type block`, as search prints it.
+     */
+    command: string;
+};

package/api/build/_adapter.mjs

@@ -9,7 +9,9 @@
  *   contribute.
  * @output Ready page templates as `{name, displayName, description, category,
  *   keywords, command}`, where `command` is the `astryx template` command that
- *   selects exactly that template; components as `{name, keywords}`.
+ *   selects exactly that template; blocks as search names them; components as
+ *   `{name, keywords}`; the kit's matcher weights, the root's classifier
+ *   weights, and the block picker's weights.
  * @position Beside build.mjs (api/build/). The kit leaf reads templates and
  *   components only through here, because a subject's `_adapter.mjs` is its
  *   only environment access. This adds no discovery of its own: templates come
@@ -63,13 +65,35 @@ export {withListingCache} from '../../foundation/fs/listing-cache.mjs';
  * @returns {Promise<PageTemplate[]>}
  */
 export async function loadPageTemplates(cwd) {
+  return (await loadTemplates(cwd)).pages;
+}
+
+/**
+ * A block the kit can name, as search names a block result.
+ * @typedef {object} BlockTemplate
+ * @property {string} name The block's own id, as search reports it.
+ * @property {string} package The npm package that owns this block.
+ * @property {string} displayName Human-facing name.
+ * @property {string} description What the block is.
+ * @property {string} command `astryx template <id> --type block`, as search prints it.
+ */
+
+/**
+ * Every ready page template and every block the project can scaffold, from one
+ * discovery: pages as {@link loadPageTemplates} returns them, blocks as
+ * {@link BlockTemplate}. Discovery failures leave both empty.
+ *
+ * @param {string} cwd
+ * @returns {Promise<{pages: PageTemplate[], blocks: BlockTemplate[]}>}
+ */
+export async function loadTemplates(cwd) {
   let templates;
   try {
     templates = await discoverTemplates(cwd);
   } catch {
-    return [];
+    return {pages: [], blocks: []};
   }
-  return templates
+  const pages = templates
     .filter(t => t.type === 'page' && t.isReady !== false)
     .map(t => ({
       name: t.dirName,
@@ -83,6 +107,16 @@ export async function loadPageTemplates(cwd) {
       // owns the Core id it names, so that id selects it, not its own.
       command: `astryx template ${t.replaces ?? t.dirName} --type page`,
     }));
+  const blocks = templates
+    .filter(t => t.type === 'block')
+    .map(t => ({
+      name: t.dirName,
+      package: t.package ?? '@astryxdesign/core',
+      displayName: t.name,
+      description: t.description || '',
+      command: `astryx template ${t.replaces ?? t.dirName} --type block`,
+    }));
+  return {pages, blocks};
 }
 
 /**
@@ -126,6 +160,48 @@ export function loadWeights() {
   return weights ?? null;
 }
 
+/** @type {import('../../foundation/route/route.mjs').ClassifierWeights | undefined} */
+let treeWeights;
+
+/**
+ * The classifier weights of build's root (tree.weights.json), read once.
+ * @returns {import('../../foundation/route/route.mjs').ClassifierWeights}
+ */
+export function loadTreeWeights() {
+  treeWeights ??= JSON.parse(
+    fs.readFileSync(new URL('./tree.weights.json', import.meta.url), 'utf8'),
+  );
+  return /** @type {import('../../foundation/route/route.mjs').ClassifierWeights} */ (
+    treeWeights
+  );
+}
+
+/** @type {import('./blocks/blocks.mjs').BlockPickerWeights | null | undefined} */
+let blockPicker;
+
+/**
+ * The block picker's weights (blocks/blocks.weights.json), read once; null
+ * when the file is absent, unreadable, or not the shape the picker reads, in
+ * which case build names its own blocks.
+ * @returns {import('./blocks/blocks.mjs').BlockPickerWeights | null}
+ */
+export function loadBlockPicker() {
+  if (blockPicker === undefined) {
+    try {
+      const file = JSON.parse(
+        fs.readFileSync(
+          new URL('./blocks/blocks.weights.json', import.meta.url),
+          'utf8',
+        ),
+      );
+      blockPicker = isBlockPickerWeights(file) ? file : null;
+    } catch {
+      blockPicker = null;
+    }
+  }
+  return blockPicker ?? null;
+}
+
 /**
  * Whether a parsed weights file has the shape the kit reads: every row has a
  * weight per candidate, every bias a number per candidate, and three blend
@@ -159,6 +235,42 @@ export function isWeightsFile(file) {
   );
 }
 
+/**
+ * Whether a parsed block picker file has the shape the picker reads: a
+ * vocabulary weight per word, a bias per block, sparse rows of
+ * [block index, weight] pairs, and five combiner numbers.
+ * @param {any} w
+ * @returns {w is import('./blocks/blocks.mjs').BlockPickerWeights}
+ */
+function isBlockPickerWeights(w) {
+  const n = Array.isArray(w?.classes) ? w.classes.length : 0;
+  return (
+    n > 0 &&
+    w.classes.every((/** @type {unknown} */ c) => typeof c === 'string') &&
+    Array.isArray(w.bias) &&
+    w.bias.length === n &&
+    w.bias.every((/** @type {unknown} */ b) => Number.isFinite(b)) &&
+    !!w.vocab &&
+    typeof w.vocab === 'object' &&
+    Object.values(w.vocab).every(v => Number.isFinite(v)) &&
+    !!w.words &&
+    typeof w.words === 'object' &&
+    Object.values(w.words).every(
+      row =>
+        Array.isArray(row) &&
+        row.length % 2 === 0 &&
+        row.every(
+          (/** @type {number} */ x, /** @type {number} */ i) =>
+            Number.isFinite(x) &&
+            (i % 2 === 1 || (Number.isInteger(x) && x >= 0 && x < n)),
+        ),
+    ) &&
+    Array.isArray(w.theta) &&
+    w.theta.length === 5 &&
+    w.theta.every((/** @type {unknown} */ x) => Number.isFinite(x))
+  );
+}
+
 /**
  * Analyze what a start template needs that the project lacks. The kit leaf
  * calls this through the adapter rather than reading the file itself, because

package/api/build/blocks/blocks.d.mts

@@ -0,0 +1,42 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in api/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * The blocks build names for a request, re-ranked by the picker.
+ * @param {string} request
+ * @param {{kitBlocks: string[], searched: BlockScore[], foreign: string[]}} signals
+ *   build's signals: the kit's blocks, search's block results [id, score,
+ *   matched, total] in order, and the blocks whose package is not Core's
+ * @param {BlockPickerWeights} weights
+ * @returns {string[]} the kit's blocks in a new order: a block the request
+ *   names, one the weights do not know, or one an integration ships keeps its
+ *   slot; the others, best first, fill the remaining slots
+ */
+export function pickBlocks(request: string, signals: {
+    kitBlocks: string[];
+    searched: BlockScore[];
+    foreign: string[];
+}, weights: BlockPickerWeights): string[];
+export type BlockPickerWeights = {
+    /**
+     * word or word pair -> weight x1024
+     */
+    vocab: Record<string, number>;
+    /**
+     * the block ids the words score
+     */
+    classes: string[];
+    /**
+     * one per class, x64
+     */
+    bias: number[];
+    /**
+     * word -> [class index, weight x64, ...]
+     */
+    words: Record<string, number[]>;
+    /**
+     * the five combiner numbers
+     */
+    theta: number[];
+};
+export type BlockScore = import("../kit/signals.mjs").BlockScore;

package/api/build/blocks/blocks.mjs

@@ -0,0 +1,169 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file build's block picker: re-ranks the blocks build would name for a
+ * request with checked-in word weights, and keeps build's own count.
+ *
+ * @input The request, build's signals for it (`kitSignals`: the blocks the kit
+ *   names, search's block results with their scores and coverage), the block
+ *   ids the project can scaffold, and the picker weights (`blocks.weights.json`
+ *   beside this file, read by the adapter): a vocabulary of words and word
+ *   pairs with their weight (x1024), the block ids it scores, a bias per block
+ *   and a sparse row of weights per word (x64), and five combiner numbers.
+ * @output `pickBlocks(...)`: block ids, best first, as many as build's own list.
+ * @position api/build/blocks; tree.mjs's main searcher calls it between
+ *   kitSignals (kit/signals.mjs) and kitAnswer (kit/answer.mjs). Pure: no file, network, or environment access.
+ *
+ * How it scores. The request's words and adjacent word pairs, each weighted
+ * (1 + ln count) x its vocabulary weight and scaled to unit length, give every
+ * block a linear score (its bias plus the weighted words' rows). Only the
+ * blocks build already names are re-ranked: never another block, never more.
+ * Each one's final score combines five numbers: search's score / 100, the
+ * share of the request search matched, whether the kit names it, 1 / (1 + its
+ * place in build's list), and the block's log-share of the linear scores.
+ * Ties keep build's order. A block the request names (its name, or the
+ * component its name starts with, written out, joined, or plural), a block the
+ * weights do not know, and a block whose package is not Core's keep the slot
+ * build gave them; the picker re-orders the rest within the other slots.
+ */
+
+import {pluralOf} from '../../search/_query.mjs';
+import {wordsOf} from '../../search/_words.mjs';
+
+/**
+ * @typedef {object} BlockPickerWeights
+ * @property {Record<string, number>} vocab word or word pair -> weight x1024
+ * @property {string[]} classes the block ids the words score
+ * @property {number[]} bias one per class, x64
+ * @property {Record<string, number[]>} words word -> [class index, weight x64, ...]
+ * @property {number[]} theta the five combiner numbers
+ *
+ * @typedef {import('../kit/signals.mjs').BlockScore} BlockScore
+ */
+
+/** The linear score of a block the words do not score. */
+const UNSCORED = -30;
+
+/**
+ * The words and adjacent word pairs of a request, with their weights, in the
+ * order they first appear; only those in the vocabulary.
+ * @param {string} request
+ * @param {BlockPickerWeights} weights
+ * @returns {Map<string, number>}
+ */
+function features(request, weights) {
+  const tokens =
+    String(request ?? '')
+      .toLowerCase()
+      .match(/[a-z0-9]{2,}/g) ?? [];
+  const all = [...tokens];
+  for (let i = 0; i + 1 < tokens.length; i++)
+    all.push(`${tokens[i]} ${tokens[i + 1]}`);
+  /** @type {Map<string, number>} */
+  const counts = new Map();
+  for (const f of all) {
+    if (Object.hasOwn(weights.vocab, f))
+      counts.set(f, (counts.get(f) ?? 0) + 1);
+  }
+  /** @type {Map<string, number>} */
+  const x = new Map();
+  for (const [f, c] of counts)
+    x.set(f, ((1 + Math.log(c)) * weights.vocab[f]) / 1024);
+  return x;
+}
+
+/**
+ * Every class's linear score for a request, and the log of the normalizer.
+ * @param {string} request
+ * @param {BlockPickerWeights} weights
+ * @returns {{z: number[], lse: number}}
+ */
+function linearScores(request, weights) {
+  const x = features(request, weights);
+  let sq = 0;
+  for (const v of x.values()) sq += v * v;
+  const norm = Math.sqrt(sq);
+  const z = weights.bias.slice();
+  if (norm > 0) {
+    for (const [f, v] of x) {
+      const row = Object.hasOwn(weights.words, f) ? weights.words[f] : null;
+      if (!row) continue;
+      for (let k = 0; k < row.length; k += 2)
+        z[row[k]] += (v / norm) * row[k + 1];
+    }
+  }
+  for (let c = 0; c < z.length; c++) z[c] /= 64;
+  let max = -Infinity;
+  for (const v of z) if (v > max) max = v;
+  let sum = 0;
+  for (const v of z) sum += Math.exp(v - max);
+  return {z, lse: max + Math.log(sum)};
+}
+
+/**
+ * Whether a request names a block: a run of one to three adjacent words of
+ * the request, joined, is the block's name, the start of its name that names
+ * its component (at least four letters, whole words), or the plural of either.
+ * @param {string[]} words the request's lowercase words
+ * @param {string} block
+ * @returns {boolean}
+ */
+function names(words, block) {
+  const parts = wordsOf(block);
+  /** @type {Set<string>} */
+  const forms = new Set();
+  for (let n = 1; n <= parts.length; n++) {
+    const form = parts.slice(0, n).join('');
+    if (form.length >= 4) forms.add(form);
+  }
+  for (let i = 0; i < words.length; i++) {
+    let run = '';
+    for (let n = 0; n < 3 && i + n < words.length; n++) {
+      run += words[i + n];
+      for (const form of forms)
+        if (run === form || pluralOf(run, form)) return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * The blocks build names for a request, re-ranked by the picker.
+ * @param {string} request
+ * @param {{kitBlocks: string[], searched: BlockScore[], foreign: string[]}} signals
+ *   build's signals: the kit's blocks, search's block results [id, score,
+ *   matched, total] in order, and the blocks whose package is not Core's
+ * @param {BlockPickerWeights} weights
+ * @returns {string[]} the kit's blocks in a new order: a block the request
+ *   names, one the weights do not know, or one an integration ships keeps its
+ *   slot; the others, best first, fill the remaining slots
+ */
+export function pickBlocks(request, signals, weights) {
+  const own = signals.kitBlocks;
+  if (own.length === 0) return [];
+  const {z, lse} = linearScores(request, weights);
+  const classIndex = new Map(weights.classes.map((c, i) => [c, i]));
+  /** @type {Map<string, BlockScore>} */
+  const found = new Map();
+  for (const s of signals.searched) if (!found.has(s[0])) found.set(s[0], s);
+  const foreign = new Set(signals.foreign);
+  const words = wordsOf(request);
+  const keeps = own.map(
+    b => !classIndex.has(b) || foreign.has(b) || names(words, b),
+  );
+  const [t0, t1, t2, t3, t4] = weights.theta;
+  const scored = own
+    .map((b, pos) => ({b, pos}))
+    .filter((_, k) => !keeps[k])
+    .map(({b, pos}) => {
+      const s = found.get(b);
+      const c = classIndex.get(b);
+      const f0 = s ? s[1] / 100 : 0;
+      const f1 = s ? (s[2] ?? 0) / Math.max(s[3] ?? 0, 1) : 0;
+      const f4 = (c === undefined ? UNSCORED : z[c]) - lse;
+      return {b, pos, s: t0 * f0 + t1 * f1 + t2 + t3 / (1 + pos) + t4 * f4};
+    })
+    .sort((a, b) => b.s - a.s || a.pos - b.pos);
+  let next = 0;
+  return own.map((b, k) => (keeps[k] ? b : scored[next++].b));
+}

package/api/build/blocks/blocks.test.mjs

@@ -0,0 +1,108 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Unit tests for build's block picker on small hand-written weights: it
+ * only re-orders the blocks build names (never another block, never more), a
+ * block the request names keeps its slot, as does a block the weights do not
+ * know or an integration ships, and ties keep build's order.
+ */
+
+import {describe, expect, it} from 'vitest';
+import {pickBlocks} from './blocks.mjs';
+
+/** @type {import('./blocks.mjs').BlockPickerWeights} */
+const WORDS = {
+  vocab: {chart: 1024, table: 2048, sales: 1536, 'sales chart': 3072, row: 512},
+  classes: ['ChartBar', 'TableBasic', 'CardStat', 'ListPlain', 'CalendarBasic'],
+  bias: [10, -20, 0, -64, 0],
+  words: {
+    chart: [0, 90, 2, 30],
+    table: [1, 120, 3, -40],
+    'sales chart': [0, 60, 2, 50],
+    row: [2, 80],
+  },
+  theta: [0.5, 0.25, 0.5, 0.4, 1.5],
+};
+/** @type {[string, number, number, number][]} */
+const SEARCHED = [
+  ['TableBasic', 96, 1, 3],
+  ['ListPlain', 64, 1, 3],
+  ['ChartBar', 50, 1, 3],
+  ['CardStat', 40, 1, 3],
+];
+
+/** @type {import('./blocks.mjs').BlockPickerWeights} */
+const TIES = {
+  vocab: {alpha: 1024},
+  classes: ['X', 'Y', 'Z'],
+  bias: [0, 0, 0],
+  words: {},
+  theta: [0, 0, 0, 0, 1],
+};
+
+/**
+ * @param {string[]} kitBlocks
+ * @param {string[]} [foreign]
+ */
+const signals = (kitBlocks, foreign = []) => ({
+  kitBlocks,
+  searched: SEARCHED,
+  foreign,
+});
+
+describe('pickBlocks', () => {
+  it('re-orders only the blocks build names, never adding one', () => {
+    const own = ['TableBasic', 'ListPlain', 'CardStat'];
+    const picks = pickBlocks('sales chart with a row', signals(own), WORDS);
+    expect([...picks].sort()).toEqual([...own].sort());
+    expect(picks).toEqual(['CardStat', 'TableBasic', 'ListPlain']);
+  });
+
+  it('never names a block the words score highly that build did not name', () => {
+    const picks = pickBlocks(
+      'sales chart chart chart',
+      signals(['TableBasic', 'ListPlain']),
+      WORDS,
+    );
+    expect(picks).not.toContain('ChartBar');
+    expect(picks).toHaveLength(2);
+  });
+
+  it('keeps a block the request names in its slot', () => {
+    const own = ['TableBasic', 'CalendarBasic', 'CardStat'];
+    for (const request of [
+      'sales chart with a calendar',
+      'sales chart with two calendars',
+      'sales chart, calendar basic',
+    ]) {
+      expect(pickBlocks(request, signals(own), WORDS)[1]).toBe('CalendarBasic');
+    }
+  });
+
+  it('keeps a block the weights do not know or an integration ships in its slot', () => {
+    const own = ['TableBasic', 'AcmeCallout', 'CardStat', 'ListPlain'];
+    const picks = pickBlocks(
+      'sales chart with a row',
+      signals(own, ['ListPlain']),
+      WORDS,
+    );
+    expect(picks[1]).toBe('AcmeCallout');
+    expect(picks[3]).toBe('ListPlain');
+    expect([...picks].sort()).toEqual([...own].sort());
+  });
+
+  it('keeps build’s order on ties and when no word is known', () => {
+    const own = ['X', 'Y', 'Z'];
+    expect(
+      pickBlocks('??', {kitBlocks: own, searched: [], foreign: []}, TIES),
+    ).toEqual(own);
+  });
+
+  it('returns nothing when build names no block, and does not change its input', () => {
+    expect(pickBlocks('sales chart', signals([]), WORDS)).toEqual([]);
+    const own = ['TableBasic', 'ListPlain'];
+    const copy = [...own];
+    pickBlocks('sales chart', signals(own), WORDS);
+    expect(own).toEqual(copy);
+  });
+});