@markuplint/spec-generator 4.6.19-dev.110 → 4.7.0

package/CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [4.7.0](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.6.19...@markuplint/spec-generator@4.7.0) (2025-08-13)
+
+### Bug Fixes
+
+- ensure that each `clean` command correctly removes build files ([110b78e](https://github.com/markuplint/markuplint/commit/110b78e85379d29a84ca68325127344a87a570b6))
+
+### Features
+
+- **spec-generator:** update ARIA spec scraping for ARIA 1.3 page structure ([f614b54](https://github.com/markuplint/markuplint/commit/f614b54b6e495fd52783ac3fa5e5ca199f5632a3))
+
+## [4.6.19](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.6.18...@markuplint/spec-generator@4.6.19) (2025-04-13)
+
+**Note:** Version bump only for package @markuplint/spec-generator
+
 ## [4.6.18](https://github.com/markuplint/markuplint/compare/@markuplint/spec-generator@4.6.17...@markuplint/spec-generator@4.6.18) (2025-03-09)
 
 **Note:** Version bump only for package @markuplint/spec-generator

package/README.md

@@ -1,3 +1,82 @@
 # @markuplint/spec-generator
 
-This is private package for generating `@markuplint/html-spec`
+Private builder used to generate `@markuplint/html-spec`.
+
+It assembles an Extended Spec JSON from the HTML element source files and external references
+(MDN, WAI‑ARIA, HTML‑ARIA), then writes `index.json` in `@markuplint/html-spec`.
+
+## How it is invoked
+
+Called from `packages/@markuplint/html-spec/build.mjs`:
+
+```ts
+await main({
+  outputFilePath: 'index.json',
+  htmlFilePattern: 'src/spec.*.json',
+  commonAttrsFilePath: 'src/spec-common.attributes.json',
+  commonContentsFilePath: 'src/spec-common.contents.json',
+});
+```
+
+You normally don't run this directly; use:
+
+- From repo root: `yarn up:gen`
+- Only html-spec: `yarn workspace @markuplint/html-spec run gen`
+
+## What it does
+
+1. Read element sources
+
+- Load every `src/spec.*.json` and infer the element name from the filename (e.g. `spec.a.json` → `a`).
+
+2. Enrich from MDN
+
+- Fetch the MDN element page and populate missing metadata:
+  - `cite`, `description`, `categories`, `omission`, attribute flags
+  - Existing fields in `src/spec.*.json` take precedence over scraped values
+  - Attributes are merged name-by-name; manual entries win
+
+3. Add obsolete elements
+
+- Inject HTML obsolete elements (WHATWG list) and some deprecated SVG elements if not present.
+
+4. Load shared data
+
+- `def['#globalAttrs']` from `src/spec-common.attributes.json`
+- `def['#contentModels']` from `src/spec-common.contents.json` (`models` key)
+
+5. Build ARIA definitions
+
+- Scrape WAI‑ARIA (1.1/1.2/1.3) and Graphics‑ARIA, plus HTML‑ARIA cross‑refs, to produce
+  `def['#aria']` (roles, properties, synonyms, defaults, and equivalent HTML attrs).
+
+6. Emit Extended Spec JSON
+
+- `{ cites, def: { #globalAttrs, #aria, #contentModels }, specs: [...] }` → `index.json`
+  (Pretty‑printed by the caller)
+
+## Source of truth vs. generated data
+
+- Source of truth for element specs is in `@markuplint/html-spec/src/`.
+- This generator is purely a build step; do not edit the output `index.json` by hand.
+
+## Precedence rules (important)
+
+- Manual data in `src/spec.*.json` overrides MDN‑scraped values on conflict.
+- Attribute objects are merged per name; manual keys win, MDN may fill missing flags.
+- Shared files under `src/spec-common.*.json` are imported as‑is.
+
+## Network and caching
+
+- Uses live HTTP fetch against MDN/W3C specs. There is an in‑process cache for the current run only.
+- If a fetch fails, the entry may be left empty; re‑run later or edit your manual source to cover it.
+
+## When to change this package
+
+- Only when the scraping targets change (DOM structure/URLs), or when the Extended Spec shape evolves
+  in `@markuplint/ml-spec`.
+
+## See also
+
+- `@markuplint/html-spec` README — how to edit the element sources.
+- `@markuplint/ml-spec` README — schema shapes, generation, and spec merging.

package/lib/aria.js

@@ -1,4 +1,3 @@
-/* global cheerio */
 import { fetch } from './fetch.js';
 import { arrayUnique, nameCompare } from './utils.js';
 export async function getAria() {
@@ -80,7 +79,7 @@ async function getRoles(version, graphicsAria = false) {
         /\[deprecated in aria 1\.\d]/i) === -1
             ? undefined
             : true;
-        const $features = $el.find('.role-features tr');
+        const $features = $el.find('.role-features tr, table.def');
         const generalization = $features
             .find('.role-parent a')
             .toArray()
@@ -193,8 +192,8 @@ async function getProps(version, roles) {
         .toArray()
         .map(el => {
         const href = $(el).prop('href');
-        const hashIndex = href.indexOf('#');
-        const hash = hashIndex === -1 ? undefined : href.slice(hashIndex);
+        const hashIndex = href?.indexOf('#');
+        const hash = hashIndex === -1 ? undefined : href?.slice(hashIndex);
         return hash?.slice(1);
     })
         .filter((s) => !!s));
@@ -203,9 +202,9 @@ async function getProps(version, roles) {
         const className = $section.attr('class');
         const type = className && /property/i.test(className) ? 'property' : 'state';
         const deprecated = (className && /deprecated/i.test(className)) || undefined;
-        const $value = $section.find(`table.${type}-features .${type}-value, .state-features .property-value`);
+        const $value = $section.find(`table .${type}-value, table .property-value, .state-features .property-value`);
         const value = $value.text().trim();
-        const $valueDescriptions = $section.find('table.value-descriptions tbody tr');
+        const $valueDescriptions = $section.find('table:is(.value-descriptions, .def:has(.value-description)) tbody tr');
         const valueDescriptions = {};
         $valueDescriptions.each((_, $tr) => {
             const name = $($tr)
@@ -213,7 +212,7 @@ async function getProps(version, roles) {
                 .text()
                 .replaceAll(/\(default\)\s*:?/gi, '')
                 .trim();
-            const desc = $($tr).find('.value-description').text().trim();
+            const desc = $($tr).find('.value-description').text().trim().replaceAll(/\s+/g, ' ');
             valueDescriptions[name] = desc;
         });
         const enumValues = [];
@@ -227,7 +226,7 @@ async function getProps(version, roles) {
                 .trim());
             enumValues.push(...values);
         }
-        const $defaultValue = $section.find('table.value-descriptions .value-name .default');
+        const $defaultValue = $section.find('table:is(.value-descriptions, .def:has(.value-description)) .value-name .default');
         const defaultValue = $defaultValue
             .text()
             .replaceAll(/\(default\)/gi, '')

package/lib/fetch.d.ts

@@ -1,3 +1,4 @@
-export declare function fetch(url: string): Promise<cheerio.Root>;
+import * as cheerio from 'cheerio';
+export declare function fetch(url: string): Promise<cheerio.CheerioAPI>;
 export declare function fetchText(url: string): Promise<string>;
 export declare function getReferences(): string[];

package/lib/scraping.js

@@ -1,4 +1,3 @@
-/* global cheerio */
 import { fetch } from './fetch.js';
 import { getThisOutline, sortObjectByKey } from './utils.js';
 const MAIN_ARTICLE_SELECTOR = 'article.main-page-content, article.article';

package/lib/utils.d.ts

@@ -1,10 +1,12 @@
+import type * as cheerio from 'cheerio';
+import type { AnyNode } from 'domhandler';
 type HasName = {
     readonly name: string;
 };
 export declare function nameCompare(a: HasName | string, b: HasName | string): 1 | 0 | -1;
 export declare function sortObjectByKey<T>(o: T): T;
 export declare function arrayUnique<T extends HasName>(array: readonly T[]): T[];
-export declare function getThisOutline($: cheerio.Root, $start: cheerio.Cheerio): cheerio.Cheerio;
+export declare function getThisOutline($: cheerio.CheerioAPI, $start: cheerio.Cheerio<AnyNode>): cheerio.Cheerio<AnyNode>;
 export declare function mergeAttributes<T>(fromDocs: T, fromJSON: T): T;
 export declare function keys<T, K = keyof T>(object: T): K[];
 export declare function getName(origin: string): {

package/lib/utils.js

@@ -1,4 +1,3 @@
-/* global cheerio */
 export function nameCompare(a, b) {
     const nameA = typeof a === 'string' ? a : (a.name?.toUpperCase() ?? String(a));
     const nameB = typeof b === 'string' ? b : (b.name?.toUpperCase() ?? String(b));

package/package.json

@@ -1,11 +1,10 @@
 {
 	"name": "@markuplint/spec-generator",
-	"version": "4.6.19-dev.110+16e148e6e",
+	"version": "4.7.0",
 	"description": "Generates @markuplint/html-spec",
 	"repository": "git@github.com:markuplint/markuplint.git",
 	"author": "Yusuke Hirao <yusukehirao@me.com>",
 	"license": "MIT",
-	"private": false,
 	"type": "module",
 	"exports": {
 		".": {
@@ -19,22 +18,22 @@
 	"scripts": {
 		"build": "tsc --project tsconfig.build.json",
 		"dev": "tsc --watch --project tsconfig.build.json",
-		"clean": "tsc --build --clean"
+		"clean": "tsc --build --clean tsconfig.build.json"
 	},
 	"dependencies": {
-		"@types/cheerio": "0.22.35",
+		"@types/cheerio": "1.0.0",
 		"ajv": "8.17.1",
-		"cheerio": "1.0.0",
+		"cheerio": "1.1.2",
 		"cli-progress": "3.12.0",
-		"fast-xml-parser": "5.2.0",
-		"glob": "11.0.1",
-		"strip-json-comments": "5.0.1"
+		"fast-xml-parser": "5.2.5",
+		"glob": "11.0.3",
+		"strip-json-comments": "5.0.3"
 	},
 	"devDependencies": {
-		"@markuplint/ml-spec": "4.9.6-dev.110+16e148e6e",
-		"@markuplint/test-tools": "4.5.18",
+		"@markuplint/ml-spec": "4.9.7",
+		"@markuplint/test-tools": "4.5.20",
 		"@types/cli-progress": "3.11.6",
-		"type-fest": "4.39.1"
+		"type-fest": "4.41.0"
 	},
-	"gitHead": "16e148e6ec2abac673c5ea1e42bbb666d421f357"
+	"gitHead": "acbf53f7e30d7a59f850a0f279b617383266dab3"
 }