@markuplint/spec-generator 4.6.19 → 4.8.0

package/CHANGELOG.md

@@ -3,6 +3,32 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [4.8.0](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.7.0...@markuplint/spec-generator@4.8.0) (2025-08-24)
+
+
+### Bug Fixes
+
+* **spec-generator:** remove debug log ([a00691b](https://github.com/markuplint/markuplint/commit/a00691bc9874ba6a8adb5bf6217e7d6c79959660))
+
+
+### Features
+
+* **spec-generator:** update MDN scraping logic ([45889a9](https://github.com/markuplint/markuplint/commit/45889a9a308c48ccd43df6837463d0ecabd547a8))
+
+
+
+
+
+# [4.7.0](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.6.19...@markuplint/spec-generator@4.7.0) (2025-08-13)
+
+### Bug Fixes
+
+- ensure that each `clean` command correctly removes build files ([110b78e](https://github.com/markuplint/markuplint/commit/110b78e85379d29a84ca68325127344a87a570b6))
+
+### Features
+
+- **spec-generator:** update ARIA spec scraping for ARIA 1.3 page structure ([f614b54](https://github.com/markuplint/markuplint/commit/f614b54b6e495fd52783ac3fa5e5ca199f5632a3))
+
 ## [4.6.19](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.6.18...@markuplint/spec-generator@4.6.19) (2025-04-13)
 
 **Note:** Version bump only for package @markuplint/spec-generator

package/README.md

@@ -1,3 +1,82 @@
 # @markuplint/spec-generator
 
-This is private package for generating `@markuplint/html-spec`
+Private builder used to generate `@markuplint/html-spec`.
+
+It assembles an Extended Spec JSON from the HTML element source files and external references
+(MDN, WAI‑ARIA, HTML‑ARIA), then writes `index.json` in `@markuplint/html-spec`.
+
+## How it is invoked
+
+Called from `packages/@markuplint/html-spec/build.mjs`:
+
+```ts
+await main({
+  outputFilePath: 'index.json',
+  htmlFilePattern: 'src/spec.*.json',
+  commonAttrsFilePath: 'src/spec-common.attributes.json',
+  commonContentsFilePath: 'src/spec-common.contents.json',
+});
+```
+
+You normally don't run this directly; use:
+
+- From repo root: `yarn up:gen`
+- Only html-spec: `yarn workspace @markuplint/html-spec run gen`
+
+## What it does
+
+1. Read element sources
+
+- Load every `src/spec.*.json` and infer the element name from the filename (e.g. `spec.a.json` → `a`).
+
+2. Enrich from MDN
+
+- Fetch the MDN element page and populate missing metadata:
+  - `cite`, `description`, `categories`, `omission`, attribute flags
+  - Existing fields in `src/spec.*.json` take precedence over scraped values
+  - Attributes are merged name-by-name; manual entries win
+
+3. Add obsolete elements
+
+- Inject HTML obsolete elements (WHATWG list) and some deprecated SVG elements if not present.
+
+4. Load shared data
+
+- `def['#globalAttrs']` from `src/spec-common.attributes.json`
+- `def['#contentModels']` from `src/spec-common.contents.json` (`models` key)
+
+5. Build ARIA definitions
+
+- Scrape WAI‑ARIA (1.1/1.2/1.3) and Graphics‑ARIA, plus HTML‑ARIA cross‑refs, to produce
+  `def['#aria']` (roles, properties, synonyms, defaults, and equivalent HTML attrs).
+
+6. Emit Extended Spec JSON
+
+- `{ cites, def: { #globalAttrs, #aria, #contentModels }, specs: [...] }` → `index.json`
+  (Pretty‑printed by the caller)
+
+## Source of truth vs. generated data
+
+- Source of truth for element specs is in `@markuplint/html-spec/src/`.
+- This generator is purely a build step; do not edit the output `index.json` by hand.
+
+## Precedence rules (important)
+
+- Manual data in `src/spec.*.json` overrides MDN‑scraped values on conflict.
+- Attribute objects are merged per name; manual keys win, MDN may fill missing flags.
+- Shared files under `src/spec-common.*.json` are imported as‑is.
+
+## Network and caching
+
+- Uses live HTTP fetch against MDN/W3C specs. There is an in‑process cache for the current run only.
+- If a fetch fails, the entry may be left empty; re‑run later or edit your manual source to cover it.
+
+## When to change this package
+
+- Only when the scraping targets change (DOM structure/URLs), or when the Extended Spec shape evolves
+  in `@markuplint/ml-spec`.
+
+## See also
+
+- `@markuplint/html-spec` README — how to edit the element sources.
+- `@markuplint/ml-spec` README — schema shapes, generation, and spec merging.

package/lib/aria.js

@@ -1,4 +1,3 @@
-/* global cheerio */
 import { fetch } from './fetch.js';
 import { arrayUnique, nameCompare } from './utils.js';
 export async function getAria() {
@@ -80,7 +79,7 @@ async function getRoles(version, graphicsAria = false) {
         /\[deprecated in aria 1\.\d]/i) === -1
             ? undefined
             : true;
-        const $features = $el.find('.role-features tr');
+        const $features = $el.find('.role-features tr, table.def');
         const generalization = $features
             .find('.role-parent a')
             .toArray()
@@ -193,8 +192,8 @@ async function getProps(version, roles) {
         .toArray()
         .map(el => {
         const href = $(el).prop('href');
-        const hashIndex = href.indexOf('#');
-        const hash = hashIndex === -1 ? undefined : href.slice(hashIndex);
+        const hashIndex = href?.indexOf('#');
+        const hash = hashIndex === -1 ? undefined : href?.slice(hashIndex);
         return hash?.slice(1);
     })
         .filter((s) => !!s));
@@ -203,9 +202,9 @@ async function getProps(version, roles) {
         const className = $section.attr('class');
         const type = className && /property/i.test(className) ? 'property' : 'state';
         const deprecated = (className && /deprecated/i.test(className)) || undefined;
-        const $value = $section.find(`table.${type}-features .${type}-value, .state-features .property-value`);
+        const $value = $section.find(`table .${type}-value, table .property-value, .state-features .property-value`);
         const value = $value.text().trim();
-        const $valueDescriptions = $section.find('table.value-descriptions tbody tr');
+        const $valueDescriptions = $section.find('table:is(.value-descriptions, .def:has(.value-description)) tbody tr');
         const valueDescriptions = {};
         $valueDescriptions.each((_, $tr) => {
             const name = $($tr)
@@ -213,7 +212,7 @@ async function getProps(version, roles) {
                 .text()
                 .replaceAll(/\(default\)\s*:?/gi, '')
                 .trim();
-            const desc = $($tr).find('.value-description').text().trim();
+            const desc = $($tr).find('.value-description').text().trim().replaceAll(/\s+/g, ' ');
             valueDescriptions[name] = desc;
         });
         const enumValues = [];
@@ -227,7 +226,7 @@ async function getProps(version, roles) {
                 .trim());
             enumValues.push(...values);
         }
-        const $defaultValue = $section.find('table.value-descriptions .value-name .default');
+        const $defaultValue = $section.find('table:is(.value-descriptions, .def:has(.value-description)) .value-name .default');
         const defaultValue = $defaultValue
             .text()
             .replaceAll(/\(default\)/gi, '')

package/lib/fetch.d.ts

@@ -1,3 +1,4 @@
-export declare function fetch(url: string): Promise<cheerio.Root>;
+import * as cheerio from 'cheerio';
+export declare function fetch(url: string): Promise<cheerio.CheerioAPI>;
 export declare function fetchText(url: string): Promise<string>;
 export declare function getReferences(): string[];

package/lib/html-elements.js

@@ -50,13 +50,16 @@ export async function getElements(filePattern) {
     specs.push(...obsoleteElements);
     specs = await Promise.all(specs.map(async (el) => {
         const { localName, namespace, ml } = getName(el.name);
-        const cite = `https://developer.mozilla.org/en-US/docs/Web/${ml}/Element/${localName}`;
+        const urlTagName = /^h[1-6]$/i.test(localName) ? 'Heading_Elements' : localName;
+        // https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/a
+        // https://developer.mozilla.org/en-US/docs/Web/SVG/Reference/Element/a
+        const cite = `https://developer.mozilla.org/en-US/docs/Web/${ml}/Reference/Element${ml === 'HTML' ? 's' : ''}/${urlTagName}`;
         const mdnData = await fetchHTMLElement(cite);
         // @ts-ignore
         delete el.name;
         // @ts-ignore
         delete el.namespace;
-        return {
+        const spec = {
             // @ts-ignore
             name: namespace === 'http://www.w3.org/2000/svg' ? `svg:${localName}` : localName,
             namespace,
@@ -99,9 +102,9 @@ export async function getElements(filePattern) {
                 return attrs;
             })()),
         };
+        return spec;
     }));
     return specs
         .sort(nameCompare)
-        .sort((a, b) => (a.namespace == b.namespace ? 0 : a.namespace === 'http://www.w3.org/2000/svg' ? 1 : -1))
-        .filter(spec => spec.name !== 'h1-h6');
+        .sort((a, b) => (a.namespace == b.namespace ? 0 : a.namespace === 'http://www.w3.org/2000/svg' ? 1 : -1));
 }

package/lib/scraping.d.ts

@@ -1,4 +1,3 @@
 import type { ExtendedElementSpec } from '@markuplint/ml-spec';
-export declare function fetchHTMLElementLinks(): Promise<string[]>;
 export declare function fetchObsoleteElements(obsoleteList: readonly string[], specs: readonly ExtendedElementSpec[]): ExtendedElementSpec[];
 export declare function fetchHTMLElement(link: string): Promise<ExtendedElementSpec>;

package/lib/scraping.js

@@ -1,19 +1,6 @@
-/* global cheerio */
 import { fetch } from './fetch.js';
-import { getThisOutline, sortObjectByKey } from './utils.js';
-const MAIN_ARTICLE_SELECTOR = 'article.main-page-content, article.article';
-export async function fetchHTMLElementLinks() {
-    const $ = await fetch('https://developer.mozilla.org/en-US/docs/Web/HTML/Element');
-    const $listHeading = $($('#sidebar-quicklinks summary')
-        .toArray()
-        .find(el => /html elements/i.test($(el).text())));
-    const $list = $listHeading.siblings('ol,ul');
-    const lists = $list
-        .find('li a')
-        .toArray()
-        .map(el => `https://developer.mozilla.org${$(el).attr('href')}`);
-    return lists;
-}
+import { sortObjectByKey } from './utils.js';
+const MAIN_ARTICLE_SELECTOR = 'main#content';
 export function fetchObsoleteElements(obsoleteList, specs) {
     return obsoleteList
         .map(name => {
@@ -42,16 +29,13 @@ export function fetchObsoleteElements(obsoleteList, specs) {
 }
 export async function fetchHTMLElement(link) {
     const $ = await fetch(link);
-    let name = link.replace(/.+\/([\w-]+)$/, '$1').toLowerCase();
-    if (name === 'heading_elements') {
-        name = 'h1-h6';
-    }
+    const name = link.replace(/.+\/([\w-]+)$/, '$1').toLowerCase();
     const $article = $(MAIN_ARTICLE_SELECTOR);
-    $article.find('p:empty').remove();
-    const description = $article.find('h2#summary').next('div').find('> p:first-of-type').text().trim().replaceAll(/\s+/g, ' ') ||
-        $article.find('.seoSummary').closest('p').text().trim().replaceAll(/\s+/g, ' ') ||
-        $article.find('h1').next('div').find('> p:first-of-type').text().trim().replaceAll(/\s+/g, ' ') ||
-        $article.find('.section-content:eq(0)').find('> p:eq(0)').text().trim().replaceAll(/\s+/g, ' ');
+    const description = $article
+        .find('.reference-layout__header .content-section')
+        .text()
+        .trim()
+        .replaceAll(/\s+/g, ' ');
     const $bcTable = $article.find('.bc-table');
     const $bcTableFirstRow = $bcTable.find('tbody tr:first-child th');
     const isBcTableIsAvailable = $bcTableFirstRow.find('code').text().trim() === name;
@@ -99,11 +83,11 @@ export async function fetchHTMLElement(link) {
         categories.push('#palpable');
     if (/script-supporting/i.test(cat))
         categories.push('#script-supporting');
-    let { attributes } = getAttributes($, '#attributes', name);
-    const { attributes: deprecatedAttributes } = getAttributes($, '#deprecated_attributes', name);
-    const { attributes: individualAttributes } = getAttributes($, '#individual_attributes', name);
-    const { attributes: nonStandardAttributes } = getAttributes($, '#non-standard_attributes', name);
-    const { attributes: obsoleteAttributes } = getAttributes($, '#obsolete_attributes', name);
+    let { attributes } = getAttributes($, 'attributes');
+    const { attributes: deprecatedAttributes } = getAttributes($, 'deprecated_attributes');
+    const { attributes: individualAttributes } = getAttributes($, 'individual_attributes');
+    const { attributes: nonStandardAttributes } = getAttributes($, 'non-standard_attributes');
+    const { attributes: obsoleteAttributes } = getAttributes($, 'obsolete_attributes');
     attributes = sortObjectByKey({
         ...attributes,
         ...deprecatedAttributes,
@@ -135,7 +119,7 @@ export async function fetchHTMLElement(link) {
 function getProperty(
 // eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
 $, prop) {
-    const $tr = $(MAIN_ARTICLE_SELECTOR).find('table.properties tr') ?? $('#Technical_summary').next('table tr');
+    const $tr = $('#technical_summary ~ figure.table-container > table tr');
     const $th = $($tr
         .find('th')
         .toArray()
@@ -144,11 +128,10 @@ $, prop) {
 }
 function getAttributes(
 // eslint-disable-next-line @typescript-eslint/prefer-readonly-parameter-types
-$, heading, tagName) {
-    const $heading = $(heading);
-    const $outline = getThisOutline($, $heading);
+$, id) {
+    const $section = $(`.content-section[aria-labelledby="${id}"]`);
     const attributes = {};
-    for (const dt of $outline.find('> div > dl > dt').toArray()) {
+    for (const dt of $section.find('> dl > dt').toArray()) {
         const $dt = $(dt);
         const name = $dt.find('code').text().trim();
         if (!name) {

package/lib/utils.d.ts

@@ -1,10 +1,12 @@
+import type * as cheerio from 'cheerio';
+import type { AnyNode } from 'domhandler';
 type HasName = {
     readonly name: string;
 };
 export declare function nameCompare(a: HasName | string, b: HasName | string): 1 | 0 | -1;
 export declare function sortObjectByKey<T>(o: T): T;
 export declare function arrayUnique<T extends HasName>(array: readonly T[]): T[];
-export declare function getThisOutline($: cheerio.Root, $start: cheerio.Cheerio): cheerio.Cheerio;
+export declare function getThisOutline($: cheerio.CheerioAPI, $start: cheerio.Cheerio<AnyNode>): cheerio.Cheerio<AnyNode>;
 export declare function mergeAttributes<T>(fromDocs: T, fromJSON: T): T;
 export declare function keys<T, K = keyof T>(object: T): K[];
 export declare function getName(origin: string): {

package/lib/utils.js

@@ -1,4 +1,3 @@
-/* global cheerio */
 export function nameCompare(a, b) {
     const nameA = typeof a === 'string' ? a : (a.name?.toUpperCase() ?? String(a));
     const nameB = typeof b === 'string' ? b : (b.name?.toUpperCase() ?? String(b));

package/package.json

@@ -1,11 +1,10 @@
 {
 	"name": "@markuplint/spec-generator",
-	"version": "4.6.19",
+	"version": "4.8.0",
 	"description": "Generates @markuplint/html-spec",
 	"repository": "git@github.com:markuplint/markuplint.git",
 	"author": "Yusuke Hirao <yusukehirao@me.com>",
 	"license": "MIT",
-	"private": false,
 	"type": "module",
 	"exports": {
 		".": {
@@ -19,22 +18,22 @@
 	"scripts": {
 		"build": "tsc --project tsconfig.build.json",
 		"dev": "tsc --watch --project tsconfig.build.json",
-		"clean": "tsc --build --clean"
+		"clean": "tsc --build --clean tsconfig.build.json"
 	},
 	"dependencies": {
-		"@types/cheerio": "0.22.35",
+		"@types/cheerio": "1.0.0",
 		"ajv": "8.17.1",
-		"cheerio": "1.0.0",
+		"cheerio": "1.1.2",
 		"cli-progress": "3.12.0",
-		"fast-xml-parser": "5.2.0",
-		"glob": "11.0.1",
-		"strip-json-comments": "5.0.1"
+		"fast-xml-parser": "5.2.5",
+		"glob": "11.0.3",
+		"strip-json-comments": "5.0.3"
 	},
 	"devDependencies": {
-		"@markuplint/ml-spec": "4.9.6",
-		"@markuplint/test-tools": "4.5.19",
+		"@markuplint/ml-spec": "4.10.0",
+		"@markuplint/test-tools": "4.5.21",
 		"@types/cli-progress": "3.11.6",
-		"type-fest": "4.39.1"
+		"type-fest": "4.41.0"
 	},
-	"gitHead": "eb36d59f7e13d4e59ff3f3c4eabb5ec06c070eb0"
+	"gitHead": "ae97eb2d31ecedf4f0800fbbf18588aad4ebca04"
 }