@mui/internal-code-infra 0.0.4-canary.3 → 0.0.4-canary.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -8
- package/build/babel-config.d.mts +11 -3
- package/build/brokenLinksChecker/crawlWorker.d.mts +1 -0
- package/build/brokenLinksChecker/index.d.mts +35 -2
- package/build/changelog/types.d.ts +1 -1
- package/build/cli/cmdArgosPush.d.mts +2 -2
- package/build/cli/cmdBuild.d.mts +2 -2
- package/build/cli/cmdCopyFiles.d.mts +2 -2
- package/build/cli/cmdExtractErrorCodes.d.mts +2 -2
- package/build/cli/cmdGenerateChangelog.d.mts +2 -2
- package/build/cli/cmdGithubAuth.d.mts +2 -2
- package/build/cli/cmdListWorkspaces.d.mts +4 -2
- package/build/cli/cmdNetlifyIgnore.d.mts +2 -2
- package/build/cli/cmdPublish.d.mts +4 -2
- package/build/cli/cmdPublishCanary.d.mts +3 -2
- package/build/cli/cmdPublishNewPackage.d.mts +4 -2
- package/build/cli/cmdSetVersionOverrides.d.mts +2 -2
- package/build/cli/cmdVale.d.mts +46 -0
- package/build/cli/cmdValidateBuiltTypes.d.mts +2 -2
- package/build/eslint/mui/rules/disallow-react-api-in-server-components.d.mts +2 -2
- package/build/eslint/mui/rules/docgen-ignore-before-comment.d.mts +2 -2
- package/build/eslint/mui/rules/no-restricted-resolved-imports.d.mts +2 -2
- package/build/markdownlint/duplicate-h1.d.mts +1 -1
- package/build/markdownlint/git-diff.d.mts +1 -1
- package/build/markdownlint/index.d.mts +1 -1
- package/build/markdownlint/straight-quotes.d.mts +1 -1
- package/build/markdownlint/table-alignment.d.mts +1 -1
- package/build/markdownlint/terminal-language.d.mts +1 -1
- package/build/utils/build.d.mts +3 -3
- package/build/utils/github.d.mts +1 -1
- package/build/utils/pnpm.d.mts +68 -2
- package/build/utils/testUtils.d.mts +7 -0
- package/package.json +38 -31
- package/src/babel-config.mjs +9 -3
- package/src/brokenLinksChecker/__fixtures__/static-site/index.html +1 -0
- package/src/brokenLinksChecker/__fixtures__/static-site/invalid-html.html +15 -0
- package/src/brokenLinksChecker/crawlWorker.mjs +173 -0
- package/src/brokenLinksChecker/index.mjs +177 -164
- package/src/brokenLinksChecker/index.test.ts +55 -13
- package/src/build-env.d.ts +13 -0
- package/src/changelog/fetchChangelogs.mjs +6 -2
- package/src/changelog/types.ts +1 -1
- package/src/cli/cmdListWorkspaces.mjs +9 -2
- package/src/cli/cmdNetlifyIgnore.mjs +4 -88
- package/src/cli/cmdPublish.mjs +51 -14
- package/src/cli/cmdPublishCanary.mjs +139 -107
- package/src/cli/cmdPublishNewPackage.mjs +27 -6
- package/src/cli/cmdVale.mjs +513 -0
- package/src/cli/cmdVale.test.mjs +644 -0
- package/src/cli/index.mjs +2 -0
- package/src/eslint/baseConfig.mjs +2 -1
- package/src/eslint/docsConfig.mjs +2 -1
- package/src/eslint/jsonConfig.mjs +2 -1
- package/src/eslint/mui/config.mjs +11 -1
- package/src/eslint/testConfig.mjs +2 -1
- package/src/estree-typescript.d.ts +1 -1
- package/src/untyped-plugins.d.ts +11 -11
- package/src/utils/build.test.mjs +546 -575
- package/src/utils/pnpm.mjs +192 -3
- package/src/utils/pnpm.test.mjs +580 -0
- package/src/utils/testUtils.mjs +18 -0
- package/src/utils/typescript.test.mjs +249 -272
- package/vale/.vale.ini +1 -0
- package/vale/styles/MUI/CorrectReferenceAllCases.yml +43 -0
- package/vale/styles/MUI/CorrectRererenceCased.yml +14 -0
- package/vale/styles/MUI/GoogleLatin.yml +11 -0
- package/vale/styles/MUI/MuiBrandName.yml +22 -0
- package/vale/styles/MUI/NoBritish.yml +112 -0
- package/vale/styles/MUI/NoCompanyName.yml +17 -0
|
@@ -1,21 +1,17 @@
|
|
|
1
1
|
/* eslint-disable no-console */
|
|
2
2
|
import { execaCommand } from 'execa';
|
|
3
3
|
import timers from 'node:timers/promises';
|
|
4
|
-
import { parse } from 'node-html-parser';
|
|
5
4
|
import * as fs from 'node:fs/promises';
|
|
6
5
|
import * as path from 'node:path';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
7
|
import chalk from 'chalk';
|
|
8
8
|
import { Transform } from 'node:stream';
|
|
9
|
-
import
|
|
10
|
-
import { unified } from 'unified';
|
|
11
|
-
import remarkParse from 'remark-parse';
|
|
12
|
-
import remarkGfm from 'remark-gfm';
|
|
13
|
-
import remarkRehype from 'remark-rehype';
|
|
14
|
-
import rehypeSlug from 'rehype-slug';
|
|
15
|
-
import rehypeStringify from 'rehype-stringify';
|
|
9
|
+
import { Worker } from 'node:worker_threads';
|
|
16
10
|
|
|
17
11
|
const DEFAULT_CONCURRENCY = 4;
|
|
18
12
|
|
|
13
|
+
const crawlWorkerUrl = new URL('./crawlWorker.mjs', import.meta.url);
|
|
14
|
+
|
|
19
15
|
/**
|
|
20
16
|
* Creates a Transform stream that prefixes each line with a given string.
|
|
21
17
|
* Useful for distinguishing server logs from other output.
|
|
@@ -105,6 +101,30 @@ function deserializeLinkStructure(data) {
|
|
|
105
101
|
return linkStructure;
|
|
106
102
|
}
|
|
107
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Input data passed to the crawl worker via workerData.
|
|
106
|
+
* @typedef {Object} CrawlWorkerInput
|
|
107
|
+
* @property {string} pageUrl - The page URL to crawl
|
|
108
|
+
* @property {ResolvedCrawlOptions} options - Fully resolved crawl options
|
|
109
|
+
*/
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Serialized page data returned by the crawl worker (uses arrays instead of Sets for structured clone).
|
|
113
|
+
* @typedef {Object} CrawlWorkerPageData
|
|
114
|
+
* @property {string} url - The normalized page URL
|
|
115
|
+
* @property {number} status - HTTP status code
|
|
116
|
+
* @property {string[]} targets - Array of anchor targets (e.g., '#intro')
|
|
117
|
+
* @property {string} contentType - Content-type of the page
|
|
118
|
+
*/
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Output message posted by the crawl worker.
|
|
122
|
+
* @typedef {Object} CrawlWorkerOutput
|
|
123
|
+
* @property {CrawlWorkerPageData} pageData - Serialized page data
|
|
124
|
+
* @property {Link[]} links - Links discovered on the page
|
|
125
|
+
* @property {{ pageUrl: string, results: import('html-validate').Result[] } | null} htmlValidateResults - HTML validation results, or null if validation was skipped/passed
|
|
126
|
+
*/
|
|
127
|
+
|
|
108
128
|
/**
|
|
109
129
|
* Data about a crawled page including its URL, HTTP status, and available link targets.
|
|
110
130
|
* @typedef {Object} PageData
|
|
@@ -131,77 +151,6 @@ async function writePagesToFile(pages, outPath) {
|
|
|
131
151
|
await fs.writeFile(outPath, JSON.stringify(fileContent, null, 2), 'utf-8');
|
|
132
152
|
}
|
|
133
153
|
|
|
134
|
-
/**
|
|
135
|
-
* Computes the accessible name of an element according to ARIA rules.
|
|
136
|
-
* Polyfill for `node.computedName` available only in Chrome v112+.
|
|
137
|
-
* Checks in order: aria-label, aria-labelledby, label[for], img alt, innerText.
|
|
138
|
-
* @param {import('node-html-parser').HTMLElement | null} elm - Element to compute name for
|
|
139
|
-
* @param {import('node-html-parser').HTMLElement} ownerDocument - Document containing the element
|
|
140
|
-
* @returns {string} The computed accessible name, or empty string if none found
|
|
141
|
-
*/
|
|
142
|
-
function getAccessibleName(elm, ownerDocument) {
|
|
143
|
-
if (!elm) {
|
|
144
|
-
return '';
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// 1. aria-label
|
|
148
|
-
const ariaLabel = elm.getAttribute('aria-label')?.trim();
|
|
149
|
-
if (ariaLabel) {
|
|
150
|
-
return ariaLabel;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// 2. aria-labelledby
|
|
154
|
-
const labelledby = elm.getAttribute('aria-labelledby');
|
|
155
|
-
if (labelledby) {
|
|
156
|
-
const labels = [];
|
|
157
|
-
for (const id of labelledby.split(/\s+/)) {
|
|
158
|
-
const label = getAccessibleName(ownerDocument.getElementById(id), ownerDocument);
|
|
159
|
-
if (label) {
|
|
160
|
-
labels.push(label);
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
const label = labels.join(' ').trim();
|
|
164
|
-
if (label) {
|
|
165
|
-
return label;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
// 3. <label for="id">
|
|
170
|
-
if (elm.id) {
|
|
171
|
-
const label = ownerDocument.querySelector(`label[for="${elm.id}"]`);
|
|
172
|
-
if (label) {
|
|
173
|
-
return getAccessibleName(label, ownerDocument);
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
// 4. <img alt="">
|
|
178
|
-
if (elm.tagName === 'IMG') {
|
|
179
|
-
const alt = elm.getAttribute('alt')?.trim();
|
|
180
|
-
if (alt) {
|
|
181
|
-
return alt;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
// 5. Fallback: visible text
|
|
186
|
-
return elm.innerText.trim();
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
/**
|
|
190
|
-
* Converts markdown content to HTML using unified pipeline.
|
|
191
|
-
* @param {string} markdown - Raw markdown content
|
|
192
|
-
* @returns {Promise<string>} Converted HTML string
|
|
193
|
-
*/
|
|
194
|
-
async function markdownToHtml(markdown) {
|
|
195
|
-
const result = await unified()
|
|
196
|
-
.use(remarkParse)
|
|
197
|
-
.use(remarkGfm)
|
|
198
|
-
.use(remarkRehype)
|
|
199
|
-
.use(rehypeSlug)
|
|
200
|
-
.use(rehypeStringify)
|
|
201
|
-
.process(markdown);
|
|
202
|
-
return String(result);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
154
|
/**
|
|
206
155
|
* Generic concurrent task queue with configurable concurrency limit.
|
|
207
156
|
* Processes tasks in FIFO order with a maximum number of concurrent workers.
|
|
@@ -402,11 +351,12 @@ function shouldIgnoreLink(link, ignores) {
|
|
|
402
351
|
* @property {number} [concurrency] - Number of concurrent page fetches (defaults to 4)
|
|
403
352
|
* @property {string[]} [seedUrls] - Starting URLs for the crawl (defaults to ['/'])
|
|
404
353
|
* @property {IgnoreRule[]} [ignores] - Rules to ignore broken links. Each rule can have path, href, contentType, and/or has properties. All specified properties must match (AND logic). Within a property, multiple values use OR logic.
|
|
354
|
+
* @property {boolean | import('html-validate').ConfigData} [htmlValidate] - Enable HTML validation on crawled pages. `false` (default): disabled. `true`: validate with recommended rules. Object: use as html-validate config (supports `extends: ['mui:recommended']` to reference the default config).
|
|
405
355
|
*/
|
|
406
356
|
|
|
407
357
|
/**
|
|
408
358
|
* Fully resolved configuration with all optional fields filled with defaults.
|
|
409
|
-
* @typedef {Omit<Required<CrawlOptions>, 'ignores'> & { ignores: NormalizedIgnoreRule[] }} ResolvedCrawlOptions
|
|
359
|
+
* @typedef {Omit<Required<CrawlOptions>, 'ignores' | 'htmlValidate'> & { ignores: NormalizedIgnoreRule[], htmlValidate: import('html-validate').ConfigData | null }} ResolvedCrawlOptions
|
|
410
360
|
*/
|
|
411
361
|
|
|
412
362
|
/**
|
|
@@ -422,6 +372,21 @@ function validateIgnoreRule(rule) {
|
|
|
422
372
|
}
|
|
423
373
|
}
|
|
424
374
|
|
|
375
|
+
/**
|
|
376
|
+
* Resolves the htmlValidate option into an html-validate config object or null.
|
|
377
|
+
* @param {boolean | import('html-validate').ConfigData | undefined} option
|
|
378
|
+
* @returns {import('html-validate').ConfigData | null}
|
|
379
|
+
*/
|
|
380
|
+
function resolveHtmlValidateConfig(option) {
|
|
381
|
+
if (!option) {
|
|
382
|
+
return null;
|
|
383
|
+
}
|
|
384
|
+
if (option === true) {
|
|
385
|
+
return { extends: ['mui:recommended'] };
|
|
386
|
+
}
|
|
387
|
+
return option;
|
|
388
|
+
}
|
|
389
|
+
|
|
425
390
|
/**
|
|
426
391
|
* Resolves partial crawl options by filling in defaults for all optional fields.
|
|
427
392
|
* @param {CrawlOptions} rawOptions - Partial options from user
|
|
@@ -447,6 +412,7 @@ function resolveOptions(rawOptions) {
|
|
|
447
412
|
concurrency: rawOptions.concurrency ?? DEFAULT_CONCURRENCY,
|
|
448
413
|
seedUrls: rawOptions.seedUrls ?? ['/'],
|
|
449
414
|
ignores: normalizedIgnores,
|
|
415
|
+
htmlValidate: resolveHtmlValidateConfig(rawOptions.htmlValidate),
|
|
450
416
|
};
|
|
451
417
|
}
|
|
452
418
|
|
|
@@ -506,25 +472,42 @@ async function resolveKnownTargets(options) {
|
|
|
506
472
|
|
|
507
473
|
/**
|
|
508
474
|
* Represents a broken link or broken link target discovered during crawling.
|
|
509
|
-
* @typedef {Object}
|
|
475
|
+
* @typedef {Object} BrokenLinkIssue
|
|
510
476
|
* @property {'broken-link' | 'broken-target'} type - Type of issue: 'broken-link' for 404 pages, 'broken-target' for missing anchors
|
|
511
477
|
* @property {string} message - Human-readable description of the issue (e.g., 'Target not found', 'Page returned error 404')
|
|
512
478
|
* @property {Link} link - The link object that has the issue
|
|
513
479
|
*/
|
|
514
480
|
|
|
481
|
+
/**
|
|
482
|
+
* Represents an HTML validation issue found on a crawled page.
|
|
483
|
+
* @typedef {Object} HtmlValidateIssue
|
|
484
|
+
* @property {'html-validate'} type - Issue type discriminator
|
|
485
|
+
* @property {string} message - Human-readable description of the issue
|
|
486
|
+
* @property {string} pageUrl - The page URL where the issue was found
|
|
487
|
+
* @property {string} ruleId - The html-validate rule that triggered this issue (e.g., 'no-dup-id')
|
|
488
|
+
* @property {number} severity - Severity level (1 = warning, 2 = error)
|
|
489
|
+
* @property {{ line: number, column: number }} location - Source location of the issue
|
|
490
|
+
* @property {string | null} selector - DOM selector for the element, or null
|
|
491
|
+
*/
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Any issue discovered during crawling.
|
|
495
|
+
* @typedef {BrokenLinkIssue | HtmlValidateIssue} Issue
|
|
496
|
+
*/
|
|
497
|
+
|
|
515
498
|
/**
|
|
516
499
|
* Results from a complete crawl operation.
|
|
517
500
|
* @typedef {Object} CrawlResult
|
|
518
501
|
* @property {Set<Link>} links - All links discovered during the crawl
|
|
519
502
|
* @property {Map<string, PageData>} pages - All pages crawled, keyed by normalized URL
|
|
520
|
-
* @property {Issue[]} issues - All broken links
|
|
503
|
+
* @property {Issue[]} issues - All issues found (broken links, broken targets, and HTML validation issues)
|
|
521
504
|
*/
|
|
522
505
|
|
|
523
506
|
/**
|
|
524
507
|
* Reports broken links to stderr, grouped by source page for better readability.
|
|
525
|
-
* @param {
|
|
508
|
+
* @param {BrokenLinkIssue[]} issuesList - Array of broken link issues to report
|
|
526
509
|
*/
|
|
527
|
-
function
|
|
510
|
+
function reportBrokenLinks(issuesList) {
|
|
528
511
|
if (issuesList.length === 0) {
|
|
529
512
|
return;
|
|
530
513
|
}
|
|
@@ -532,7 +515,7 @@ function reportIssues(issuesList) {
|
|
|
532
515
|
console.error('\nBroken links found:\n');
|
|
533
516
|
|
|
534
517
|
// Group issues by source URL
|
|
535
|
-
/** @type {Map<string,
|
|
518
|
+
/** @type {Map<string, BrokenLinkIssue[]>} */
|
|
536
519
|
const issuesBySource = new Map();
|
|
537
520
|
for (const issue of issuesList) {
|
|
538
521
|
const sourceUrl = issue.link.src ?? '(unknown)';
|
|
@@ -553,6 +536,39 @@ function reportIssues(issuesList) {
|
|
|
553
536
|
}
|
|
554
537
|
}
|
|
555
538
|
|
|
539
|
+
/**
|
|
540
|
+
* Reports HTML validation issues to stderr, grouped by page URL.
|
|
541
|
+
* @param {HtmlValidateIssue[]} htmlIssues - Array of HTML validation issues to report
|
|
542
|
+
*/
|
|
543
|
+
function reportHtmlValidation(htmlIssues) {
|
|
544
|
+
if (htmlIssues.length === 0) {
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
console.error('\nHTML validation issues:\n');
|
|
549
|
+
|
|
550
|
+
// Group by page URL
|
|
551
|
+
/** @type {Map<string, HtmlValidateIssue[]>} */
|
|
552
|
+
const issuesByPage = new Map();
|
|
553
|
+
for (const issue of htmlIssues) {
|
|
554
|
+
const pageIssues = issuesByPage.get(issue.pageUrl) ?? [];
|
|
555
|
+
if (pageIssues.length === 0) {
|
|
556
|
+
issuesByPage.set(issue.pageUrl, pageIssues);
|
|
557
|
+
}
|
|
558
|
+
pageIssues.push(issue);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
for (const [pageUrl, pageIssues] of issuesByPage.entries()) {
|
|
562
|
+
console.error(`Page ${chalk.cyan(pageUrl)}:`);
|
|
563
|
+
for (const issue of pageIssues) {
|
|
564
|
+
const severityLabel = issue.severity === 2 ? chalk.red('error') : chalk.yellow('warning');
|
|
565
|
+
console.error(
|
|
566
|
+
` ${issue.location.line}:${issue.location.column} ${severityLabel} ${issue.message} ${chalk.gray(issue.ruleId)}`,
|
|
567
|
+
);
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
556
572
|
/**
|
|
557
573
|
* Crawls a website starting from seed URLs, discovering all internal links and checking for broken links/targets.
|
|
558
574
|
* @param {CrawlOptions} rawOptions - Configuration options for the crawl
|
|
@@ -594,6 +610,35 @@ export async function crawl(rawOptions) {
|
|
|
594
610
|
const crawledPages = new Map();
|
|
595
611
|
/** @type {Set<Link>} */
|
|
596
612
|
const crawledLinks = new Set();
|
|
613
|
+
/** @type {Issue[]} */
|
|
614
|
+
const issues = [];
|
|
615
|
+
/**
|
|
616
|
+
* Spawns a crawl worker for a page URL.
|
|
617
|
+
* @param {string} pageUrl - The page URL to crawl
|
|
618
|
+
* @returns {Promise<{ pageData: PageData, links: Link[], htmlValidateResults: CrawlWorkerOutput['htmlValidateResults'] }>}
|
|
619
|
+
*/
|
|
620
|
+
function crawlInWorker(pageUrl) {
|
|
621
|
+
return new Promise((resolve, reject) => {
|
|
622
|
+
/** @type {CrawlWorkerInput} */
|
|
623
|
+
const input = { pageUrl, options };
|
|
624
|
+
const worker = new Worker(crawlWorkerUrl, {
|
|
625
|
+
workerData: input,
|
|
626
|
+
});
|
|
627
|
+
worker.on('message', (/** @type {CrawlWorkerOutput} */ msg) => {
|
|
628
|
+
resolve({
|
|
629
|
+
pageData: {
|
|
630
|
+
url: msg.pageData.url,
|
|
631
|
+
status: msg.pageData.status,
|
|
632
|
+
targets: new Set(msg.pageData.targets),
|
|
633
|
+
contentType: msg.pageData.contentType,
|
|
634
|
+
},
|
|
635
|
+
links: msg.links,
|
|
636
|
+
htmlValidateResults: msg.htmlValidateResults,
|
|
637
|
+
});
|
|
638
|
+
});
|
|
639
|
+
worker.on('error', (err) => reject(err));
|
|
640
|
+
});
|
|
641
|
+
}
|
|
597
642
|
|
|
598
643
|
const queue = new Queue(async (/** @type {Link} */ link) => {
|
|
599
644
|
crawledLinks.add(link);
|
|
@@ -611,78 +656,30 @@ export async function crawl(rawOptions) {
|
|
|
611
656
|
return;
|
|
612
657
|
}
|
|
613
658
|
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
/** @type {PageData} */
|
|
633
|
-
const pageData = {
|
|
634
|
-
url: pageUrl,
|
|
635
|
-
status: res.status,
|
|
636
|
-
targets: new Set(),
|
|
637
|
-
contentType: type,
|
|
638
|
-
};
|
|
639
|
-
|
|
640
|
-
if (pageData.status < 200 || pageData.status >= 400) {
|
|
641
|
-
console.warn(chalk.yellow(`Warning: ${pageUrl} returned status ${pageData.status}`));
|
|
642
|
-
return pageData;
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
if (type.startsWith('image/')) {
|
|
646
|
-
// Skip images
|
|
647
|
-
return pageData;
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
if (type !== 'text/html' && type !== 'text/markdown') {
|
|
651
|
-
console.warn(chalk.yellow(`Warning: ${pageUrl} returned non-HTML content-type: ${type}`));
|
|
652
|
-
return pageData;
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
const rawContent = await res.text();
|
|
656
|
-
const content = type === 'text/markdown' ? await markdownToHtml(rawContent) : rawContent;
|
|
657
|
-
|
|
658
|
-
const dom = parse(content, { parseNoneClosedTags: true });
|
|
659
|
-
|
|
660
|
-
let ignoredSelector = ':not(*)'; // matches nothing
|
|
661
|
-
if (options.ignoredContent.length > 0) {
|
|
662
|
-
ignoredSelector = Array.from(options.ignoredContent)
|
|
663
|
-
.flatMap((selector) => [selector, `${selector} *`])
|
|
664
|
-
.join(',');
|
|
665
|
-
}
|
|
666
|
-
const linksSelector = `a[href]:not(${ignoredSelector})`;
|
|
667
|
-
|
|
668
|
-
const pageLinks = dom.querySelectorAll(linksSelector).map((a) => ({
|
|
669
|
-
src: pageUrl,
|
|
670
|
-
text: getAccessibleName(a, dom),
|
|
671
|
-
href: a.getAttribute('href') ?? '',
|
|
672
|
-
contentType: type,
|
|
673
|
-
}));
|
|
674
|
-
|
|
675
|
-
for (const target of dom.querySelectorAll('*[id]')) {
|
|
676
|
-
if (!options.ignoredTargets.has(target.id)) {
|
|
677
|
-
pageData.targets.add(`#${target.id}`);
|
|
659
|
+
console.log(`Crawling ${chalk.cyan(pageUrl)}...`);
|
|
660
|
+
const workerPromise = crawlInWorker(pageUrl);
|
|
661
|
+
const pagePromise = workerPromise.then((result) => {
|
|
662
|
+
if (result.htmlValidateResults) {
|
|
663
|
+
for (const validationResult of result.htmlValidateResults.results) {
|
|
664
|
+
for (const msg of validationResult.messages) {
|
|
665
|
+
issues.push({
|
|
666
|
+
type: 'html-validate',
|
|
667
|
+
message: msg.message,
|
|
668
|
+
pageUrl: result.htmlValidateResults.pageUrl,
|
|
669
|
+
ruleId: msg.ruleId,
|
|
670
|
+
severity: msg.severity,
|
|
671
|
+
location: { line: msg.line, column: msg.column },
|
|
672
|
+
selector: msg.selector,
|
|
673
|
+
});
|
|
674
|
+
}
|
|
678
675
|
}
|
|
679
676
|
}
|
|
680
677
|
|
|
681
|
-
for (const
|
|
682
|
-
queue.add(
|
|
678
|
+
for (const discoveredLink of result.links) {
|
|
679
|
+
queue.add(discoveredLink);
|
|
683
680
|
}
|
|
684
681
|
|
|
685
|
-
return pageData;
|
|
682
|
+
return result.pageData;
|
|
686
683
|
});
|
|
687
684
|
|
|
688
685
|
crawledPages.set(pageUrl, pagePromise);
|
|
@@ -711,10 +708,6 @@ export async function crawl(rawOptions) {
|
|
|
711
708
|
await writePagesToFile(results, options.outPath);
|
|
712
709
|
}
|
|
713
710
|
|
|
714
|
-
/** Array to collect all issues found during validation */
|
|
715
|
-
/** @type {Issue[]} */
|
|
716
|
-
const issues = [];
|
|
717
|
-
|
|
718
711
|
/** Count of links ignored due to ignores configuration */
|
|
719
712
|
let ignoredCount = 0;
|
|
720
713
|
|
|
@@ -771,11 +764,24 @@ export async function crawl(rawOptions) {
|
|
|
771
764
|
}
|
|
772
765
|
}
|
|
773
766
|
|
|
774
|
-
|
|
767
|
+
// Split issues by type for reporting
|
|
768
|
+
/** @type {BrokenLinkIssue[]} */
|
|
769
|
+
const brokenLinkIssues = /** @type {BrokenLinkIssue[]} */ (
|
|
770
|
+
issues.filter((issue) => issue.type === 'broken-link' || issue.type === 'broken-target')
|
|
771
|
+
);
|
|
772
|
+
/** @type {HtmlValidateIssue[]} */
|
|
773
|
+
const htmlValidateIssues = /** @type {HtmlValidateIssue[]} */ (
|
|
774
|
+
issues.filter((issue) => issue.type === 'html-validate')
|
|
775
|
+
);
|
|
776
|
+
|
|
777
|
+
reportBrokenLinks(brokenLinkIssues);
|
|
778
|
+
reportHtmlValidation(htmlValidateIssues);
|
|
775
779
|
|
|
776
780
|
// Derive counts from issues
|
|
777
|
-
const brokenLinks =
|
|
778
|
-
const brokenLinkTargets =
|
|
781
|
+
const brokenLinks = brokenLinkIssues.filter((issue) => issue.type === 'broken-link').length;
|
|
782
|
+
const brokenLinkTargets = brokenLinkIssues.filter(
|
|
783
|
+
(issue) => issue.type === 'broken-target',
|
|
784
|
+
).length;
|
|
779
785
|
|
|
780
786
|
const endTime = Date.now();
|
|
781
787
|
const durationSeconds = (endTime - startTime) / 1000;
|
|
@@ -784,14 +790,21 @@ export async function crawl(rawOptions) {
|
|
|
784
790
|
unit: 'second',
|
|
785
791
|
maximumFractionDigits: 2,
|
|
786
792
|
}).format(durationSeconds);
|
|
793
|
+
const fmt = new Intl.NumberFormat('en-US').format;
|
|
787
794
|
console.log(chalk.blue(`\nCrawl completed in ${duration}`));
|
|
788
|
-
console.log(` Total links found: ${chalk.cyan(crawledLinks.size)}`);
|
|
789
|
-
console.log(` Total broken links: ${chalk.cyan(brokenLinks)}`);
|
|
790
|
-
console.log(` Total broken link targets: ${chalk.cyan(brokenLinkTargets)}`);
|
|
791
|
-
console.log(` Total ignored: ${chalk.cyan(ignoredCount)}`);
|
|
795
|
+
console.log(` Total links found: ${chalk.cyan(fmt(crawledLinks.size))}`);
|
|
796
|
+
console.log(` Total broken links: ${chalk.cyan(fmt(brokenLinks))}`);
|
|
797
|
+
console.log(` Total broken link targets: ${chalk.cyan(fmt(brokenLinkTargets))}`);
|
|
798
|
+
console.log(` Total ignored: ${chalk.cyan(fmt(ignoredCount))}`);
|
|
799
|
+
if (options.htmlValidate) {
|
|
800
|
+
const pagesWithHtmlIssues = new Set(htmlValidateIssues.map((issue) => issue.pageUrl)).size;
|
|
801
|
+
console.log(
|
|
802
|
+
` HTML validation issues: ${chalk.cyan(fmt(htmlValidateIssues.length))} across ${chalk.cyan(fmt(pagesWithHtmlIssues))} ${pagesWithHtmlIssues === 1 ? 'page' : 'pages'}`,
|
|
803
|
+
);
|
|
804
|
+
}
|
|
792
805
|
|
|
793
806
|
if (options.outPath) {
|
|
794
|
-
console.log(chalk.blue(`Output written to: ${options.outPath}`));
|
|
807
|
+
console.log(chalk.blue(`Output written to: ${pathToFileURL(options.outPath)}`));
|
|
795
808
|
}
|
|
796
809
|
|
|
797
810
|
return { links: crawledLinks, pages: results, issues };
|
|
@@ -2,12 +2,18 @@ import path from 'node:path';
|
|
|
2
2
|
import getPort from 'get-port';
|
|
3
3
|
import { describe, expect, it } from 'vitest';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
type
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
import {
|
|
6
|
+
crawl,
|
|
7
|
+
type BrokenLinkIssue,
|
|
8
|
+
type HtmlValidateIssue,
|
|
9
|
+
type Issue,
|
|
10
|
+
type Link,
|
|
11
|
+
// eslint-disable-next-line import/extensions
|
|
12
|
+
} from './index.mjs';
|
|
13
|
+
|
|
14
|
+
type ExpectedBrokenLinkIssue = Omit<Partial<BrokenLinkIssue>, 'link'> & { link?: Partial<Link> };
|
|
15
|
+
|
|
16
|
+
function objectMatchingIssue(expectedIssue: ExpectedBrokenLinkIssue) {
|
|
11
17
|
return expect.objectContaining({
|
|
12
18
|
...expectedIssue,
|
|
13
19
|
...(expectedIssue.link ? { link: expect.objectContaining(expectedIssue.link) } : {}),
|
|
@@ -15,16 +21,16 @@ function objectMatchingIssue(expectedIssue: ExpectedIssue) {
|
|
|
15
21
|
}
|
|
16
22
|
|
|
17
23
|
/**
|
|
18
|
-
* Helper to assert that
|
|
24
|
+
* Helper to assert that a broken link issue with matching properties exists in the issues array
|
|
19
25
|
*/
|
|
20
|
-
function expectIssue(issues: Issue[], expectedIssue:
|
|
26
|
+
function expectIssue(issues: Issue[], expectedIssue: ExpectedBrokenLinkIssue) {
|
|
21
27
|
expect(issues).toEqual(expect.arrayContaining([objectMatchingIssue(expectedIssue)]));
|
|
22
28
|
}
|
|
23
29
|
|
|
24
30
|
/**
|
|
25
|
-
* Helper to assert that no issue with matching properties exists in the issues array
|
|
31
|
+
* Helper to assert that no broken link issue with matching properties exists in the issues array
|
|
26
32
|
*/
|
|
27
|
-
function expectNotIssue(issues: Issue[], notExpectedIssue:
|
|
33
|
+
function expectNotIssue(issues: Issue[], notExpectedIssue: ExpectedBrokenLinkIssue) {
|
|
28
34
|
expect(issues).not.toEqual(expect.arrayContaining([objectMatchingIssue(notExpectedIssue)]));
|
|
29
35
|
}
|
|
30
36
|
|
|
@@ -56,12 +62,21 @@ describe('Broken Links Checker', () => {
|
|
|
56
62
|
// Test href-only rule (matches from any page) - note: matches the actual href value
|
|
57
63
|
{ href: 'broken-relative.html' },
|
|
58
64
|
],
|
|
65
|
+
htmlValidate: {
|
|
66
|
+
extends: ['mui:recommended'],
|
|
67
|
+
rules: {
|
|
68
|
+
'no-raw-characters': 'off',
|
|
69
|
+
},
|
|
70
|
+
},
|
|
59
71
|
});
|
|
60
72
|
|
|
61
|
-
expect(result.links).toHaveLength(
|
|
62
|
-
//
|
|
73
|
+
expect(result.links).toHaveLength(67);
|
|
74
|
+
// Broken link issue count: original 11, minus ignored ones (broken-from-markdown via contentType,
|
|
63
75
|
// broken-relative via href-only rule)
|
|
64
|
-
|
|
76
|
+
const brokenLinkIssues = result.issues.filter(
|
|
77
|
+
(issue) => issue.type === 'broken-link' || issue.type === 'broken-target',
|
|
78
|
+
);
|
|
79
|
+
expect(brokenLinkIssues).toHaveLength(9);
|
|
65
80
|
|
|
66
81
|
// Test ignores: these broken links should be ignored (not in issues)
|
|
67
82
|
expectNotIssue(result.issues, {
|
|
@@ -257,5 +272,32 @@ describe('Broken Links Checker', () => {
|
|
|
257
272
|
// Test contentType is stored on pageData
|
|
258
273
|
expect(result.pages.get('/example.md')?.contentType).toBe('text/markdown');
|
|
259
274
|
expect(result.pages.get('/')?.contentType).toBe('text/html');
|
|
275
|
+
|
|
276
|
+
// Test htmlValidate: invalid-html.html has duplicate IDs which should be reported
|
|
277
|
+
const htmlValidateIssues = result.issues.filter(
|
|
278
|
+
(issue): issue is HtmlValidateIssue => issue.type === 'html-validate',
|
|
279
|
+
);
|
|
280
|
+
const invalidHtmlIssues = htmlValidateIssues.filter(
|
|
281
|
+
(issue) => issue.pageUrl === '/invalid-html.html',
|
|
282
|
+
);
|
|
283
|
+
expect(invalidHtmlIssues.length).toBeGreaterThan(0);
|
|
284
|
+
expect(invalidHtmlIssues).toEqual(
|
|
285
|
+
expect.arrayContaining([
|
|
286
|
+
expect.objectContaining({
|
|
287
|
+
type: 'html-validate',
|
|
288
|
+
pageUrl: '/invalid-html.html',
|
|
289
|
+
ruleId: 'no-dup-id',
|
|
290
|
+
}),
|
|
291
|
+
]),
|
|
292
|
+
);
|
|
293
|
+
|
|
294
|
+
// Test htmlValidate override: no-raw-characters is off, so raw & should NOT be reported
|
|
295
|
+
expect(invalidHtmlIssues).not.toEqual(
|
|
296
|
+
expect.arrayContaining([
|
|
297
|
+
expect.objectContaining({
|
|
298
|
+
ruleId: 'no-raw-characters',
|
|
299
|
+
}),
|
|
300
|
+
]),
|
|
301
|
+
);
|
|
260
302
|
}, 30000);
|
|
261
303
|
});
|
|
@@ -93,8 +93,12 @@ async function fetchCommitsRest({ octokit, repo, lastRelease, release, org = 'mu
|
|
|
93
93
|
}
|
|
94
94
|
|
|
95
95
|
const promises = results.map(async (commit) => {
|
|
96
|
-
const
|
|
97
|
-
|
|
96
|
+
const matches = [...commit.commit.message.matchAll(/#(\d+)/g)];
|
|
97
|
+
// The PR number is always the last match.
|
|
98
|
+
// Sometimes the PR titles include an issue number like this:
|
|
99
|
+
// [tag] PR title (#00001) (#00002)
|
|
100
|
+
const prMatch = matches.at(-1);
|
|
101
|
+
if (!prMatch) {
|
|
98
102
|
return null;
|
|
99
103
|
}
|
|
100
104
|
|
package/src/changelog/types.ts
CHANGED
|
@@ -233,7 +233,7 @@ export interface IntroConfig {
|
|
|
233
233
|
* - {{teamCount}}: Number of team members
|
|
234
234
|
* - {{communityCount}}: Number of community contributors
|
|
235
235
|
*
|
|
236
|
-
* Example: "
|
|
236
|
+
* Example: "A big thanks to the {{contributorCount}} contributors who made this release possible."
|
|
237
237
|
*
|
|
238
238
|
* Set to `false` or omit to disable the thank you message.
|
|
239
239
|
*/
|
|
@@ -15,6 +15,7 @@ import { getWorkspacePackages } from '../utils/pnpm.mjs';
|
|
|
15
15
|
* @property {boolean} [publicOnly] - Whether to filter to only public packages
|
|
16
16
|
* @property {'json'|'path'|'name'|'publish-dir'} [output] - Output format (name, path, or json)
|
|
17
17
|
* @property {string} [sinceRef] - Git reference to filter changes since
|
|
18
|
+
* @property {string[]} [filter] - Same as filtering packages with --filter in pnpm. Only include packages matching the filter. See https://pnpm.io/filtering.
|
|
18
19
|
*/
|
|
19
20
|
|
|
20
21
|
export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({
|
|
@@ -37,13 +38,19 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({
|
|
|
37
38
|
.option('since-ref', {
|
|
38
39
|
type: 'string',
|
|
39
40
|
description: 'Filter packages changed since git reference',
|
|
41
|
+
})
|
|
42
|
+
.option('filter', {
|
|
43
|
+
type: 'string',
|
|
44
|
+
array: true,
|
|
45
|
+
description:
|
|
46
|
+
'Same as filtering packages with --filter in pnpm. Only include packages matching the filter. See https://pnpm.io/filtering.',
|
|
40
47
|
});
|
|
41
48
|
},
|
|
42
49
|
handler: async (argv) => {
|
|
43
|
-
const { publicOnly = false, output = 'name', sinceRef } = argv;
|
|
50
|
+
const { publicOnly = false, output = 'name', sinceRef, filter = [] } = argv;
|
|
44
51
|
|
|
45
52
|
// Get packages using our helper function
|
|
46
|
-
const packages = await getWorkspacePackages({ sinceRef, publicOnly });
|
|
53
|
+
const packages = await getWorkspacePackages({ sinceRef, publicOnly, filter });
|
|
47
54
|
|
|
48
55
|
if (output === 'json') {
|
|
49
56
|
// Serialize packages to JSON
|