@astryxdesign/cli 0.7.2-canary.fe2c655 → 0.7.2

package/foundation/discovery/doc-fluff.mjs

@@ -0,0 +1,328 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * Fluff in docs: padding phrases that add words and no meaning, and guides
+ * whose prose is mostly function words. Docs are read by people and agents
+ * with a budget, so every word should carry something.
+ *
+ * Only prose is read (the same strings copied passages compare), and inline
+ * code and quoted text are set aside, so a phrase inside a command, or one
+ * quoted as an example, never counts.
+ */
+
+import fs from 'node:fs';
+
+import {builtinGuideFiles} from './docs-discovery.mjs';
+import {
+  contentWords,
+  fileWords,
+  lineAtOffset,
+  locateWords,
+  loadQuietly,
+  proseStrings,
+  passageWords,
+  textWords,
+} from './copied-passages.mjs';
+
+/**
+ * Padding phrases: each can be cut, or replaced by a shorter word, without
+ * losing meaning. Only phrases that are padding in every use are listed;
+ * words such as "just", "simply" or "really", and phrases such as "whether or
+ * not" or "keep in mind", which carry meaning in a correct sentence ("not
+ * simply a wrapper", "regardless of whether"), are not.
+ */
+export const FILLER_PHRASES = [
+  "it's important to note",
+  'it is important to note',
+  'important to note that',
+  "it's worth noting",
+  'it is worth noting',
+  'worth noting that',
+  'it should be noted',
+  'please note',
+  'note that',
+  'as mentioned above',
+  'as mentioned earlier',
+  'as mentioned before',
+  'as noted above',
+  'as you can see',
+  'needless to say',
+  'in order to',
+  'due to the fact that',
+  'for the purpose of',
+  'at this point in time',
+  'in other words',
+  'basically',
+  'essentially',
+  'obviously',
+];
+
+/** What to write instead, where a phrase is not simply cut. */
+const INSTEAD = {
+  'in order to': 'to',
+  'due to the fact that': 'because',
+  'for the purpose of': 'to or for',
+  'at this point in time': 'now',
+};
+
+/** A guide shorter than this many prose words is not scored for density. */
+export const DENSITY_MIN_WORDS = 80;
+
+/**
+ * A guide whose share of content words is below this is mostly glue. Core's
+ * own guides score from about 0.51 up, with a median near 0.62.
+ */
+export const MIN_DENSITY = 0.5;
+
+/**
+ * @param {string} phrase
+ * @returns {RegExp}
+ */
+function phrasePattern(phrase) {
+  const escaped = phrase.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  // "note that" is padding as a sentence opener ("Note that X"), not inside
+  // one ("a note that explains").
+  return phrase === 'note that'
+    ? new RegExp(`(?:^|[.;:!?]\\s+)${escaped}\\b`, 'gi')
+    : // A hyphen joins words ("non-obviously"), so it bounds no phrase.
+      new RegExp(`(?<![a-z'-])${escaped}(?![a-z-])`, 'gi');
+}
+
+const PATTERNS = FILLER_PHRASES.map(phrase => ({
+  phrase,
+  pattern: phrasePattern(phrase),
+}));
+
+/**
+ * A prose string with what is not prose blanked out, length kept: `{@link}`
+ * pointers, inline code, and quoted text. A quoted phrase is mentioned, not
+ * used ("cut 'in order to'").
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+export function maskProse(text) {
+  // A quoted phrase is mentioned, not used ("cut 'in order to'"), so quotes
+  // are set aside along with inline code. Each mask keeps the text's length,
+  // so an offset in the masked text is an offset in the text.
+  const blank = (/** @type {string} */ span) => ' '.repeat(span.length);
+  return (
+    text
+      // A pointer names a doc, and its target is no prose ("{@link
+      // generic:obviously-fast}").
+      .replace(/\{@link\s[^}]*\}/g, blank)
+      .replace(/`[^`]*`/g, blank)
+      .replace(/"[^"]*"|“[^”]*”|‘[^’]*’/g, blank)
+      .replace(/’/g, "'")
+      // A single-quoted span opens and closes beside a non-letter, so the
+      // apostrophe in "don't" or "it's" opens nothing.
+      .replace(/(?<![A-Za-z])'[^']*'(?![A-Za-z])/g, blank)
+  );
+}
+
+/**
+ * Each padding phrase in a text, outside inline code and quotes, in order,
+ * with its offset in the text.
+ *
+ * `thatAfter` is true when the phrase is followed by a "that" a reader cuts
+ * with it ("It is worth noting that the cache is shared").
+ *
+ * @param {string} text
+ * @returns {{phrase: string, at: number, thatAfter: boolean}[]}
+ */
+export function fillerAt(text) {
+  const prose = maskProse(text);
+  /** @type {{phrase: string, at: number, thatAfter: boolean}[]} */
+  const found = [];
+  for (const {phrase, pattern} of PATTERNS) {
+    for (const match of prose.matchAll(pattern)) {
+      // A phrase inside a longer listed one ("note that" in "worth noting
+      // that") is the longer one's.
+      const at = match.index + match[0].toLowerCase().indexOf(phrase);
+      if (
+        !found.some(each => at >= each.at && at < each.at + each.phrase.length)
+      ) {
+        found.push({
+          phrase,
+          at,
+          // Only a phrase that ends in note or noting leads into a "that"
+          // clause the reader cuts with it; "Obviously that works" and "As
+          // noted above that flag" keep their "that".
+          thatAfter:
+            /\bnot(?:e|ed|ing)$/.test(phrase) &&
+            !phrase.endsWith('that') &&
+            /^\s+that\b/i.test(prose.slice(at + phrase.length)),
+        });
+      }
+    }
+  }
+  return found.sort((a, b) => a.at - b.at);
+}
+
+/**
+ * Each padding phrase in a text, outside inline code and quotes, in order.
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+export function fillerIn(text) {
+  return fillerAt(text).map(each => each.phrase);
+}
+
+/**
+ * The share of a doc's prose words that are content words.
+ *
+ * @param {{word: string}[]} words
+ * @returns {number}
+ */
+export function density(words) {
+  if (words.length === 0) return 1;
+  let content = 0;
+  for (let i = 0; i < words.length; i++) {
+    if (contentWords(words, i, 1) === 1) content++;
+  }
+  return content / words.length;
+}
+
+/**
+ * A number rounded down to `places` decimals, so a share under the floor
+ * never reads as the floor.
+ *
+ * @param {number} value
+ * @param {number} places
+ * @returns {number}
+ */
+export function floorTo(value, places) {
+  const scale = 10 ** places;
+  return Math.floor(value * scale + 1e-9) / scale;
+}
+
+/**
+ * A share as a percent with one decimal, rounded down ("49.5%").
+ *
+ * @param {number} share
+ * @returns {string}
+ */
+export function sharePercent(share) {
+  return `${floorTo(share * 100, 1).toFixed(1)}%`;
+}
+
+/**
+ * The words a topic's density is measured on: its prose, without what the
+ * phrase check also leaves unread (links, inline code, quoted text).
+ *
+ * @param {unknown} doc
+ * @returns {{word: string}[]}
+ */
+export function densityWords(doc) {
+  return proseStrings(doc).flatMap(({text}) =>
+    passageWords(maskProse(text)).map(word => ({word})),
+  );
+}
+
+/**
+ * The median density of Core's own guides of at least
+ * {@link DENSITY_MIN_WORDS} words.
+ *
+ * @returns {Promise<number>}
+ */
+export async function coreDensityMedian() {
+  // Core's guides are measured the way a package's topics are.
+  /** @type {number[]} */
+  const scores = [];
+  for (const file of builtinGuideFiles()) {
+    const doc = await loadQuietly(file);
+    if (typeof doc?.name !== 'string') continue;
+    const words = densityWords(doc);
+    if (words.length >= DENSITY_MIN_WORDS) scores.push(density(words));
+  }
+  scores.sort((a, b) => a - b);
+  return scores.length === 0 ? 0 : scores[Math.floor(scores.length / 2)];
+}
+
+/**
+ * Fluff in a package's topics: every padding phrase, with its section and
+ * file line, and every topic of at least {@link DENSITY_MIN_WORDS} words
+ * whose density is below {@link MIN_DENSITY}.
+ *
+ * @param {{records: {name: string, path: string}[], guides?: {name: string, ref?: {topicFile?: string}}[]}} discovered
+ *   the package's docs, as discoverIntegrationDocs returns them
+ * @returns {Promise<import('../../api/integration/authoring-checks.type.mjs').DocFluff[]>}
+ */
+export async function findFluff(discovered) {
+  const topics = [
+    ...discovered.records.map(record => ({
+      name: record.name,
+      file: record.path,
+    })),
+    ...(discovered.guides ?? []).map(guide => ({
+      name: guide.name,
+      file: guide.ref?.topicFile,
+    })),
+  ];
+  /** @type {import('../../api/integration/authoring-checks.type.mjs').DocFluff[]} */
+  const found = [];
+  /** @type {number | null} */
+  let median = null;
+  for (const {name, file} of topics) {
+    const doc = file ? await loadQuietly(file) : null;
+    if (!doc || !file) continue;
+    /** @type {{word: string, line: number}[]} */
+    let source = [];
+    try {
+      source = fileWords(fs.readFileSync(file, 'utf8'));
+    } catch {
+      // No line numbers; each finding still names its section.
+    }
+    // Each string is found by its whole run of words, from where the last
+    // one was found, so two strings that open alike keep their own lines.
+    let cursor = 0;
+    for (const {text, section} of proseStrings(doc)) {
+      const words = textWords(text);
+      let index = locateWords(source, words, cursor);
+      if (index < 0) index = locateWords(source, words);
+      if (index >= 0) cursor = index + words.length;
+      for (const {phrase, at, thatAfter} of fillerAt(text)) {
+        const line = index < 0 ? null : lineAtOffset(source, index, words, at);
+        const where = section ? ` (section "${section}")` : '';
+        const instead = /** @type {Record<string, string>} */ (INSTEAD)[phrase];
+        found.push({
+          topic: name,
+          kind: 'phrase',
+          phrase,
+          section,
+          file,
+          line,
+          density: null,
+          message:
+            `"${name}"${where} pads its prose with "${phrase}"` +
+            `${line == null ? '' : ` (line ${line})`}. ` +
+            (instead
+              ? `Write "${instead}" instead.`
+              : thatAfter
+                ? 'Cut it and the "that" after it; the sentence says the same without them.'
+                : 'Cut it; the sentence says the same without it.'),
+        });
+      }
+    }
+    const words = densityWords(doc);
+    if (words.length < DENSITY_MIN_WORDS) continue;
+    const score = density(words);
+    if (score >= MIN_DENSITY) continue;
+    median ??= await coreDensityMedian();
+    found.push({
+      topic: name,
+      kind: 'density',
+      phrase: null,
+      section: null,
+      file,
+      line: null,
+      density: floorTo(score, 3),
+      message:
+        `"${name}" is mostly glue: ${sharePercent(score)} of its ${words.length} prose words carry content, ` +
+        `under the ${Math.round(MIN_DENSITY * 100)}% floor (Core's guides have a median of ${Math.round(median * 100)}%). ` +
+        'Cut filler and restated context until each sentence states one thing.',
+    });
+  }
+  return found;
+}

package/foundation/discovery/doc-fluff.test.mjs

@@ -0,0 +1,142 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {describe, expect, it} from 'vitest';
+
+import {
+  density,
+  densityWords,
+  fillerAt,
+  fillerIn,
+  floorTo,
+  sharePercent,
+} from './doc-fluff.mjs';
+import {passageWords} from './copied-passages.mjs';
+
+const words = (/** @type {string} */ text) =>
+  passageWords(text).map(word => ({word}));
+
+describe('fillerIn', () => {
+  it('finds padding phrases in order, the longest one only', () => {
+    expect(
+      fillerIn(
+        'In order to start, it is worth noting that you basically run it. Note that it builds.',
+      ),
+    ).toEqual(['in order to', 'it is worth noting', 'basically', 'note that']);
+  });
+
+  it('leaves inline code, quotes, a noun "note", and ambiguous words alone', () => {
+    expect(
+      fillerIn(
+        'Run `basically --in order to`. Cut "in order to" and “basically”. Add a note that explains it. Copy just this one file; it really is the source.',
+      ),
+    ).toEqual([]);
+  });
+
+  it('sets aside single-quoted text, but not an apostrophe', () => {
+    expect(
+      fillerIn(
+        "Avoid ‘basically’ and ‘obviously’. Don't write 'it is worth noting that' in a guide.",
+      ),
+    ).toEqual([]);
+    expect(fillerIn("Don't pad: it's important to note the order.")).toEqual([
+      "it's important to note",
+    ]);
+  });
+
+  it('lists no phrase that carries meaning in a correct sentence', () => {
+    expect(
+      fillerIn(
+        'It is not simply a wrapper. Every app gets a theme whether or not anyone picks one. Keep in mind the order. Feel free to skip it.',
+      ),
+    ).toEqual([]);
+  });
+
+  it('reads a curly apostrophe like a straight one', () => {
+    expect(fillerIn('It’s important to note the order.')).toEqual([
+      "it's important to note",
+    ]);
+  });
+});
+
+describe('fillerAt', () => {
+  it('reads no phrase inside a link target or a hyphenated word', () => {
+    expect(
+      fillerIn(
+        'Read {@link generic:obviously-fast} before you tune the build.',
+      ),
+    ).toEqual([]);
+    expect(fillerIn('A non-obviously slow build still passes.')).toEqual([]);
+    expect(fillerIn('It is obviously slow.')).toEqual(['obviously']);
+  });
+
+  it('marks a "that" the reader cuts with the phrase', () => {
+    expect(
+      fillerAt('It is worth noting that the cache is shared.').map(
+        each => each.thatAfter,
+      ),
+    ).toEqual([true]);
+    expect(
+      fillerAt('Basically, the cache is shared.').map(each => each.thatAfter),
+    ).toEqual([false]);
+    // Only a note or noting phrase leads a clause the reader cuts.
+    expect(
+      fillerAt('Obviously that works only in a browser.').map(
+        each => each.thatAfter,
+      ),
+    ).toEqual([false]);
+    expect(
+      fillerAt('As noted above that flag is required.').map(
+        each => each.thatAfter,
+      ),
+    ).toEqual([false]);
+  });
+});
+
+describe('densityWords', () => {
+  it('measures prose only, not quoted text, inline code or links', () => {
+    const doc = {
+      sections: [
+        {
+          title: 'Overview',
+          content: [
+            {
+              type: 'prose',
+              text: 'Theme tokens set colors. Write "it is in the" or `of the` as {@link generic:theme} shows.',
+            },
+          ],
+        },
+      ],
+    };
+    expect(densityWords(doc).map(each => each.word)).toEqual([
+      'theme',
+      'tokens',
+      'set',
+      'colors.',
+      'write',
+      'or',
+      'as',
+      'shows.',
+    ]);
+  });
+});
+
+describe('share rounding', () => {
+  it('rounds a share under the floor down, never up to it', () => {
+    expect(floorTo(0.4995, 3)).toBe(0.499);
+    expect(sharePercent(0.4995)).toBe('49.9%');
+    expect(sharePercent(0.5)).toBe('50.0%');
+  });
+});
+
+describe('density', () => {
+  it('scores a padded sentence well below a dense one', () => {
+    const padded = words(
+      'So that is why it is the one that you would want to use if you can.',
+    );
+    const dense = words(
+      'Name each codemod folder after the Core version whose upgrade runs it.',
+    );
+    expect(density(padded)).toBeLessThan(0.3);
+    expect(density(dense)).toBeGreaterThan(0.6);
+  });
+});

package/foundation/discovery/doc-types.d.mts

@@ -0,0 +1,157 @@
+// @generated by scripts/sync-api-types.mjs from the JSDoc in foundation/**/*.mjs.
+// DO NOT EDIT — run `pnpm sync:api-types` to regenerate.
+
+/**
+ * Check each doc file's values against the published authoring types.
+ * @param {DocFile[]} files
+ * @param {{packageDir: string, resolveFrom: Array<string | null | undefined>, core?: boolean}} options
+ *   `core`: the docs are Core's own, so templates are held to TemplateDoc
+ * @returns {Promise<DocTypesReport>}
+ */
+export function checkDocTypes(files: DocFile[], options: {
+    packageDir: string;
+    resolveFrom: Array<string | null | undefined>;
+    core?: boolean;
+}): Promise<DocTypesReport>;
+/**
+ * @typedef {'components' | 'hooks' | 'templates' | 'docs' | 'docs-overlay' | 'themes'} DocRoot
+ */
+/**
+ * @typedef {object} DocFile
+ * @property {string} path absolute path to the doc file
+ * @property {DocRoot} root the contribution root it was found in
+ */
+/**
+ * @typedef {'unknown_field' | 'missing_field' | 'wrong_type'} DocTypeProblem
+ */
+/**
+ * @typedef {object} DocTypeFinding
+ * @property {string} doc the doc file, relative to the package root
+ * @property {number} line 1-based line of the field (or of the doc value)
+ * @property {string} export the export checked (`default`, `docs`, `docsZh`, …)
+ * @property {string} type the authoring type it was checked against
+ * @property {string} field the field's path in the doc, such as
+ *   `usage.bestPractices[2].guidance`; empty for the doc value as a whole
+ * @property {DocTypeProblem} problem
+ * @property {'warning'} severity
+ * @property {string} message what TypeScript reports, shortened
+ * @property {string} fix what to change
+ */
+/**
+ * @typedef {object} DocTypesUnchecked
+ * @property {string} doc the doc file, relative to the package root
+ * @property {string} reason why it was not checked
+ */
+/**
+ * @typedef {object} DocTypesReport
+ * @property {'checked' | 'skipped'} status `skipped` means nothing was checked,
+ *   and `reason` says why
+ * @property {string} [reason]
+ * @property {string} [typescript] the version of the compiler that checked
+ * @property {number} docs how many doc values were checked
+ * @property {DocTypeFinding[]} findings
+ * @property {DocTypesUnchecked[]} unchecked doc files with nothing to check,
+ *   or not fully checked
+ */
+/** The line both surfaces print when the compiler is missing. */
+export const DOC_TYPES_TYPESCRIPT_MISSING_REASON: "TypeScript is not installed in this project, so docs were not checked against the authoring types. On TypeScript 7, add its TypeScript 6 API package (`npm install -D @typescript/typescript6`); otherwise install TypeScript 5.0 through 6 (`npm install -D typescript@6`). Then run this check again.";
+export type DocRoot = "components" | "hooks" | "templates" | "docs" | "docs-overlay" | "themes";
+export type DocFile = {
+    /**
+     * absolute path to the doc file
+     */
+    path: string;
+    /**
+     * the contribution root it was found in
+     */
+    root: DocRoot;
+};
+export type DocTypeProblem = "unknown_field" | "missing_field" | "wrong_type";
+export type DocTypeFinding = {
+    /**
+     * the doc file, relative to the package root
+     */
+    doc: string;
+    /**
+     * 1-based line of the field (or of the doc value)
+     */
+    line: number;
+    /**
+     * the export checked (`default`, `docs`, `docsZh`, …)
+     */
+    export: string;
+    /**
+     * the authoring type it was checked against
+     */
+    type: string;
+    /**
+     * the field's path in the doc, such as
+     * `usage.bestPractices[2].guidance`; empty for the doc value as a whole
+     */
+    field: string;
+    problem: DocTypeProblem;
+    severity: "warning";
+    /**
+     * what TypeScript reports, shortened
+     */
+    message: string;
+    /**
+     * what to change
+     */
+    fix: string;
+};
+export type DocTypesUnchecked = {
+    /**
+     * the doc file, relative to the package root
+     */
+    doc: string;
+    /**
+     * why it was not checked
+     */
+    reason: string;
+};
+export type DocTypesReport = {
+    /**
+     * `skipped` means nothing was checked,
+     * and `reason` says why
+     */
+    status: "checked" | "skipped";
+    reason?: string | undefined;
+    /**
+     * the version of the compiler that checked
+     */
+    typescript?: string | undefined;
+    /**
+     * how many doc values were checked
+     */
+    docs: number;
+    findings: DocTypeFinding[];
+    /**
+     * doc files with nothing to check,
+     * or not fully checked
+     */
+    unchecked: DocTypesUnchecked[];
+};
+export type Target = {
+    exportName: string;
+    /**
+     * the doc value's expression
+     */
+    node: any;
+    /**
+     * the authoring type, as a type expression
+     */
+    typeExpr: string;
+    /**
+     * how findings name that type
+     */
+    typeName: string;
+    /**
+     * nodes whose leading JSDoc annotations are blanked
+     */
+    blank: any[];
+    /**
+     * source ranges to blank
+     */
+    blankRanges: Array<[number, number]>;
+};