@astryxdesign/cli 0.6.3-canary.db4e378 → 0.6.3-canary.ddb63c3

package/foundation/doc-compiler/bundle.d.mts

@@ -0,0 +1,47 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in foundation/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * @typedef {import('./compile.mjs').CompiledDocNode} CompiledDocNode
+ * @typedef {import('./compile.mjs').CompiledReferenceNode} CompiledReferenceNode
+ * @typedef {import('./diagnostics.mjs').CompilerDiagnostic} CompilerDiagnostic
+ */
+/**
+ * @typedef {object} CompiledDocsBundle
+ * @property {number} schemaVersion
+ * @property {string | null} lang
+ * @property {Array<CompiledDocNode | CompiledReferenceNode>} nodes in the order
+ *   the project reads its roots
+ * @property {CompilerDiagnostic[]} diagnostics sorted
+ */
+/**
+ * Compile every descriptor the project reads.
+ *
+ * A problem with one descriptor withdraws that node and is reported; it never
+ * stops the rest, so every node in the bundle reads back through the sealed
+ * parser. A token reference that finds nothing is a warning: readers
+ * print a placeholder for it, as they always have.
+ *
+ * @param {import('../config/project.mjs').Project} project
+ * @param {{lang?: string | null}} [options]
+ * @returns {Promise<CompiledDocsBundle>}
+ */
+export function compileDocs(project: import("../config/project.mjs").Project, { lang }?: {
+    lang?: string | null;
+}): Promise<CompiledDocsBundle>;
+export type CompiledDocNode = import("./compile.mjs").CompiledDocNode;
+export type CompiledReferenceNode = import("./compile.mjs").CompiledReferenceNode;
+export type CompilerDiagnostic = import("./diagnostics.mjs").CompilerDiagnostic;
+export type CompiledDocsBundle = {
+    schemaVersion: number;
+    lang: string | null;
+    /**
+     * in the order
+     * the project reads its roots
+     */
+    nodes: Array<CompiledDocNode | CompiledReferenceNode>;
+    /**
+     * sorted
+     */
+    diagnostics: CompilerDiagnostic[];
+};

package/foundation/doc-compiler/bundle.mjs

@@ -0,0 +1,218 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Compiled docs bundle — every descriptor a project reads, compiled at
+ *   once.
+ *
+ * @input A loaded Project and a reading language.
+ * @output A plain-JSON bundle: the schema version, the language, one compiled
+ *   node per descriptor (a doc topic with its extensions is one linked node),
+ *   and every compiler diagnostic, in one deterministic order.
+ * @position The whole-project view of the compiler, for checks that need every
+ *   node at once; today only tests call it. Single reads stay lazy:
+ *   ./read.mjs and api/docs/_adapter.mjs compile only what a read touches.
+ *   Internal to the CLI.
+ */
+
+import {
+  readThemeDescriptorValue,
+  themeDescriptorLabel,
+} from '../discovery/theme-discovery.mjs';
+import {
+  COMPILED_DOC_SCHEMA_VERSION,
+  linkReferenceTopic,
+  lowerReferenceTopic,
+} from './compile.mjs';
+import {diagnostic as rawDiagnostic, sortDiagnostics} from './diagnostics.mjs';
+import {collectDocInputs} from './inputs.mjs';
+import {compileDocFile, loadTopicInput} from './read.mjs';
+import {packageSource, scrubPaths} from './source.mjs';
+
+/**
+ * A compiler diagnostic whose message names files by package, never by their
+ * location on this machine.
+ * @param {string} code
+ * @param {Parameters<typeof rawDiagnostic>[1]} at
+ */
+function diagnostic(code, at) {
+  return rawDiagnostic(code, {
+    ...at,
+    message: scrubPaths(at.message) || '(the error had no message)',
+  });
+}
+
+/**
+ * @typedef {import('./compile.mjs').CompiledDocNode} CompiledDocNode
+ * @typedef {import('./compile.mjs').CompiledReferenceNode} CompiledReferenceNode
+ * @typedef {import('./diagnostics.mjs').CompilerDiagnostic} CompilerDiagnostic
+ */
+
+/**
+ * @typedef {object} CompiledDocsBundle
+ * @property {number} schemaVersion
+ * @property {string | null} lang
+ * @property {Array<CompiledDocNode | CompiledReferenceNode>} nodes in the order
+ *   the project reads its roots
+ * @property {CompilerDiagnostic[]} diagnostics sorted
+ */
+
+/**
+ * Compile every descriptor the project reads.
+ *
+ * A problem with one descriptor withdraws that node and is reported; it never
+ * stops the rest, so every node in the bundle reads back through the sealed
+ * parser. A token reference that finds nothing is a warning: readers
+ * print a placeholder for it, as they always have.
+ *
+ * @param {import('../config/project.mjs').Project} project
+ * @param {{lang?: string | null}} [options]
+ * @returns {Promise<CompiledDocsBundle>}
+ */
+export async function compileDocs(project, {lang = null} = {}) {
+  const {inputs, problems} = await collectDocInputs(project);
+  /** @type {CompilerDiagnostic[]} */
+  const diagnostics = problems.map(problem =>
+    diagnostic(problem.code, {
+      provider: problem.provider,
+      source: problem.source,
+      message: problem.message,
+    }),
+  );
+  const catalog = await project.docs();
+
+  /** @type {Map<string, CompiledReferenceNode>} lowered topics by name */
+  const topics = new Map();
+  for (const entry of catalog.entries()) {
+    const input = await loadTopicInput(entry, lang);
+    const failed = [input.base, ...input.extensions].filter(
+      file => 'error' in file || 'overlayError' in file,
+    );
+    if (failed.length > 0) {
+      for (const file of failed) {
+        const fromExtension = input.extensions.find(ext => ext === file);
+        const path = fromExtension
+          ? entry.extensions[input.extensions.indexOf(fromExtension)].path
+          : entry.path;
+        diagnostics.push(
+          diagnostic('error' in file ? 'load_failed' : 'overlay_failed', {
+            provider: fromExtension ? fromExtension.provider : entry.package,
+            source: packageSource(path),
+            message: `${file.file}: ${messageOf('error' in file ? file.error : file.overlayError)}`,
+          }),
+        );
+      }
+      continue;
+    }
+    try {
+      topics.set(entry.name.toLowerCase(), lowerReferenceTopic(input));
+    } catch (error) {
+      diagnostics.push(
+        diagnostic(
+          errorCode(error) === 'not_json' ? 'not_json' : 'invalid_topic',
+          {
+            provider: entry.package,
+            source: packageSource(entry.path),
+            message: messageOf(error),
+          },
+        ),
+      );
+    }
+  }
+  /** @param {string} name */
+  const lowerTarget = async name => {
+    const target = catalog.resolve(name);
+    return target ? (topics.get(target.name.toLowerCase()) ?? null) : null;
+  };
+
+  /** @type {Array<CompiledDocNode | CompiledReferenceNode>} */
+  const nodes = [];
+  for (const input of inputs) {
+    if (input.root === 'docs') {
+      if (input.role !== 'base') continue;
+      const lowered = topics.get(input.name.toLowerCase());
+      if (!lowered) continue;
+      const linked = await linkReferenceTopic(lowered, lowerTarget);
+      diagnostics.push(...unresolvedReferences(linked, input));
+      nodes.push(linked);
+      continue;
+    }
+    const result = await compileDocFile(
+      input.file,
+      {
+        root: input.root,
+        provider: input.owner,
+        id: input.id,
+        lang:
+          input.root === 'components' || input.root === 'hooks' ? lang : null,
+        check: true,
+        ...(input.root === 'templates' ? {loader: 'template'} : {}),
+        ...(input.root === 'themes'
+          ? {
+              label: themeDescriptorLabel(input.file, input.owner),
+              readStatic: () =>
+                readThemeDescriptorValue(
+                  input.file,
+                  themeDescriptorLabel(input.file, input.owner),
+                ),
+            }
+          : {}),
+      },
+      {node: true},
+    );
+    diagnostics.push(...result.diagnostics);
+    // A descriptor with a problem of its own withdraws its node.
+    if (result.node && !result.diagnostics.some(d => d.severity === 'error')) {
+      nodes.push(result.node);
+    }
+  }
+
+  return {
+    schemaVersion: COMPILED_DOC_SCHEMA_VERSION,
+    lang,
+    nodes,
+    diagnostics: sortDiagnostics(diagnostics),
+  };
+}
+
+/**
+ * A warning for each token reference in a linked topic that finds no topic or
+ * no section.
+ * @param {CompiledReferenceNode} node
+ * @param {import('./inputs.mjs').DocInput} input
+ * @returns {CompilerDiagnostic[]}
+ */
+function unresolvedReferences(node, input) {
+  /** @type {CompilerDiagnostic[]} */
+  const found = [];
+  for (const section of node.doc.sections) {
+    for (const block of section.content) {
+      if (block?.type !== 'token-ref') continue;
+      const status = block.resolved?.status;
+      if (status === 'resolved') continue;
+      found.push(
+        diagnostic('unresolved_reference', {
+          provider: input.owner,
+          source: input.source,
+          field: `sections.${section.id}`,
+          message:
+            status === 'unknown-topic'
+              ? `"${node.id}" section "${section.id}" refers to topic "${block.topic}", which this project does not have.`
+              : `"${node.id}" section "${section.id}" refers to section "${block.section}" of "${block.topic}", which that topic does not have.`,
+        }),
+      );
+    }
+  }
+  return found;
+}
+
+/** @param {unknown} error */
+function messageOf(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** @param {unknown} error */
+function errorCode(error) {
+  return error && typeof error === 'object' && 'compilerCode' in error
+    ? error.compilerCode
+    : undefined;
+}

package/foundation/doc-compiler/bundle.test.mjs

@@ -0,0 +1,255 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file compileDocs: every descriptor a project reads compiles to one node or
+ *   a diagnostic, and the bundle reads back through its sealed parser.
+ */
+
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {Project} from '../config/project.mjs';
+import {CLI_ROOT} from '../fs/paths.mjs';
+import {compileDocs} from './bundle.mjs';
+import {COMPILED_DOC_KINDS} from './compile.mjs';
+import {collectDocInputs} from './inputs.mjs';
+import {parseCompiledDocsBundle} from './ir.mjs';
+
+const REPO_ROOT = path.resolve(CLI_ROOT, '..', '..');
+
+let tmpDir;
+
+beforeEach(() => {
+  // Inside the repo, so the project finds this checkout's Core.
+  tmpDir = fs.mkdtempSync(path.join(REPO_ROOT, '.astryx-doc-bundle-'));
+  fs.writeFileSync(
+    path.join(tmpDir, 'package.json'),
+    JSON.stringify({name: 'consumer', version: '1.0.0'}),
+  );
+});
+
+afterEach(() => {
+  fs.rmSync(tmpDir, {recursive: true, force: true});
+});
+
+describe('compileDocs over what this repo ships', () => {
+  it('compiles every descriptor, in every language, with nothing to report', async () => {
+    const project = await Project.load(tmpDir);
+    const {inputs} = await collectDocInputs(project);
+    // One node per input, except that a topic and its extensions are one node.
+    const expected = inputs.filter(
+      input => input.root !== 'docs' || input.role === 'base',
+    ).length;
+    for (const lang of [null, 'zh', 'dense']) {
+      const bundle = await compileDocs(project, {lang});
+      expect(bundle.diagnostics).toEqual([]);
+      expect(bundle.nodes).toHaveLength(expected);
+      expect(new Set(bundle.nodes.map(node => node.kind))).toEqual(
+        new Set(COMPILED_DOC_KINDS),
+      );
+      // Plain JSON that reads back through the sealed parser, as written.
+      const copy = JSON.parse(JSON.stringify(bundle));
+      expect(parseCompiledDocsBundle(copy)).toEqual(bundle);
+    }
+  }, 180_000);
+
+  it('is deterministic', async () => {
+    const one = await compileDocs(await Project.load(tmpDir));
+    const two = await compileDocs(await Project.load(tmpDir));
+    expect(JSON.stringify(two)).toBe(JSON.stringify(one));
+  }, 120_000);
+
+  it('never carries a machine path', async () => {
+    const bundle = await compileDocs(await Project.load(tmpDir));
+    const text = JSON.stringify(bundle);
+    expect(text.includes(REPO_ROOT)).toBe(false);
+    expect(text.includes(tmpDir)).toBe(false);
+  }, 120_000);
+});
+
+describe('compileDocs over a broken integration', () => {
+  /**
+   * @param {string} rel
+   * @param {string} text
+   */
+  function write(rel, text) {
+    const file = path.join(tmpDir, 'node_modules', '@acme', 'kit', rel);
+    fs.mkdirSync(path.dirname(file), {recursive: true});
+    fs.writeFileSync(file, text);
+  }
+
+  /** @param {object} fields */
+  const topic = fields =>
+    `export default ${JSON.stringify({
+      type: 'generic',
+      title: 'Acme',
+      description: 'Acme notes.',
+      sections: [{title: 'Acme', content: [{type: 'prose', text: 'Hi.'}]}],
+      ...fields,
+    })};\n`;
+
+  beforeEach(() => {
+    fs.writeFileSync(
+      path.join(tmpDir, 'package.json'),
+      JSON.stringify({
+        name: 'consumer',
+        version: '1.0.0',
+        dependencies: {'@acme/kit': '^1.0.0'},
+      }),
+    );
+    write(
+      'package.json',
+      JSON.stringify({name: '@acme/kit', version: '1.0.0'}),
+    );
+    write(
+      'astryx.integration.mjs',
+      "export default {components: './components', docs: './docs'};\n",
+    );
+    write(
+      'components/AcmeCard.tsx',
+      'export function AcmeCard() { return null; }\n',
+    );
+    write(
+      'components/AcmeCard.doc.mjs',
+      "export default {type: 'component', name: 'AcmeCard', displayName: 'Acme Card', usage: {description: 'A card.'}, props: []};\n",
+    );
+    write('docs/acme-guide.doc.mjs', topic({name: 'acme-guide'}));
+  });
+
+  it('warns on a token reference that finds nothing, and still compiles the topic', async () => {
+    write(
+      'docs/acme-guide.doc.mjs',
+      topic({
+        name: 'acme-guide',
+        sections: [
+          {
+            title: 'Colors',
+            content: [
+              {type: 'token-ref', topic: 'no-such-topic', section: 'x'},
+              {type: 'token-ref', topic: 'tokens', section: 'no-such-section'},
+            ],
+          },
+        ],
+      }),
+    );
+    const bundle = await compileDocs(await Project.load(tmpDir));
+    const warnings = bundle.diagnostics.filter(d => d.provider === '@acme/kit');
+    expect(warnings.map(d => [d.code, d.severity, d.field])).toEqual([
+      ['unresolved_reference', 'warning', 'sections.colors'],
+      ['unresolved_reference', 'warning', 'sections.colors'],
+    ]);
+    expect(
+      warnings.every(d => d.source === '@acme/kit/docs/acme-guide.doc.mjs'),
+    ).toBe(true);
+    expect(bundle.nodes.some(node => node.id === 'acme-guide')).toBe(true);
+  });
+
+  it('reports a component doc that fails to load, and compiles the rest', async () => {
+    write(
+      'components/AcmeCard.doc.mjs',
+      "throw new Error('boom');\nexport default {};\n",
+    );
+    const project = await Project.load(tmpDir);
+    const bundle = await compileDocs(project);
+    const found = bundle.diagnostics.filter(d => d.provider === '@acme/kit');
+    // Discovery lists the doc; compiling it fails at load, and only it.
+    expect(found.map(d => [d.code, d.source])).toEqual([
+      ['load_failed', '@acme/kit/components/AcmeCard.doc.mjs'],
+    ]);
+    expect(found[0].message).toMatch(/boom/);
+    expect(bundle.nodes.some(node => node.kind === 'reference')).toBe(true);
+  });
+
+  it('withdraws a descriptor that fails its parser, and the bundle still reads back', async () => {
+    write(
+      'components/AcmeCard.doc.mjs',
+      "export default {type: 'component', displayName: 'No name'};\n",
+    );
+    const bundle = await compileDocs(await Project.load(tmpDir));
+    const found = bundle.diagnostics.filter(d => d.provider === '@acme/kit');
+    expect(found.map(d => d.code)).toEqual(['invalid_doc']);
+    expect(
+      bundle.nodes.some(node => node.id === '@acme/kit:components:AcmeCard'),
+    ).toBe(false);
+    expect(() =>
+      parseCompiledDocsBundle(JSON.parse(JSON.stringify(bundle))),
+    ).not.toThrow();
+  });
+
+  it('reports a topic value JSON cannot hold with the same code as any other doc', async () => {
+    write(
+      'docs/acme-guide.doc.mjs',
+      topic({name: 'acme-guide'}).replace('};', ', big: 1n};'),
+    );
+    const bundle = await compileDocs(await Project.load(tmpDir));
+    const found = bundle.diagnostics.filter(d => d.provider === '@acme/kit');
+    expect(found.map(d => d.code)).toContain('not_json');
+    expect(found.map(d => d.code)).not.toContain('invalid_topic');
+    // The compiler's code never reaches the error a reader throws.
+    const {lowerTopic} = await import('../../api/docs/_adapter.mjs');
+    const catalog = await (await Project.load(tmpDir)).docs();
+    const entry = /** @type {any} */ (catalog.resolve('acme-guide'));
+    const thrown = await lowerTopic(catalog, entry).catch(error => error);
+    expect(thrown).toBeInstanceOf(Error);
+    expect(/** @type {any} */ (thrown).code).toBeUndefined();
+  });
+
+  it('gives every extension file its own input, even two from one package', async () => {
+    write('docs/theme-a.doc.mjs', topic({name: 'theme-a', extends: 'theme'}));
+    write('docs/theme-b.doc.mjs', topic({name: 'theme-b', extends: 'theme'}));
+    const project = await Project.load(tmpDir);
+    const {problems} = await collectDocInputs(project);
+    expect(problems.filter(p => p.code === 'duplicate_id')).toEqual([]);
+  });
+
+  it('names a package that has no name by its folder, never by its path', async () => {
+    fs.writeFileSync(
+      path.join(tmpDir, 'package.json'),
+      JSON.stringify({version: '1.0.0'}),
+    );
+    fs.writeFileSync(
+      path.join(tmpDir, 'astryx.integration.mjs'),
+      "export default {components: './components'};\n",
+    );
+    fs.mkdirSync(path.join(tmpDir, 'components', 'Local'), {recursive: true});
+    fs.writeFileSync(
+      path.join(tmpDir, 'components', 'Local', 'Local.tsx'),
+      'export {};\n',
+    );
+    fs.writeFileSync(
+      path.join(tmpDir, 'components', 'Local', 'Local.doc.mjs'),
+      "export default {type: 'component', name: 'Local', displayName: 'Local', usage: {description: 'd'}, props: []};\n",
+    );
+    const bundle = await compileDocs(await Project.load(tmpDir));
+    const text = JSON.stringify(bundle);
+    expect(text.includes(tmpDir)).toBe(false);
+    expect(() => parseCompiledDocsBundle(JSON.parse(text))).not.toThrow();
+  });
+
+  it('reports an extension that fails to load against the extension, not its base', async () => {
+    write('docs/theme-notes.doc.mjs', "throw new Error('bad extension');\n");
+    // Discovery refuses an unloadable topic file, so the catalog never merges
+    // it; compile the entry as the catalog would have if it had loaded.
+    const project = await Project.load(tmpDir);
+    const catalog = await project.docs();
+    const theme = /** @type {any} */ (catalog.resolve('theme'));
+    theme.extensions.push({
+      package: '@acme/kit',
+      path: path.join(
+        tmpDir,
+        'node_modules',
+        '@acme',
+        'kit',
+        'docs',
+        'theme-notes.doc.mjs',
+      ),
+    });
+    const bundle = await compileDocs(project);
+    const found = bundle.diagnostics.filter(d => d.code === 'load_failed');
+    expect(found.map(d => [d.provider, d.source])).toEqual([
+      ['@acme/kit', '@acme/kit/docs/theme-notes.doc.mjs'],
+    ]);
+    // The broken extension withdraws the topic it extends, as a docs read does.
+    expect(bundle.nodes.some(node => node.id === 'theme')).toBe(false);
+  });
+});