lint-wiki-dumps 0.0.0 → 0.1.0

package/README.md

@@ -23,15 +23,15 @@ npm i vscode-css-languageservice
 ## Usage
 
 ```sh
-npx lint-wiki-dumps <language> <path to download directory>
+npx lint-wiki-dumps <language> <path to download> [path to HTML output]
 # For example:
 npx lint-wiki-dumps zh-yue ~/Downloads/dumps
 ```
 
-or execute the Bash script `scan.sh` directly:
+or execute the Bash script `scan.sh` (single thread) or `scan-parallel.sh` (multi-core cluster) directly:
 
 ```sh
-bash scan.sh <language> <path to download>
+bash scan.sh <language> <path to download> [path to HTML output]
 # For example:
 bash scan.sh zh-yue ~/Downloads/dumps
 ```
@@ -49,11 +49,11 @@ node parser.js zh-yue ~/Downloads/dumps/zh-yuewiki-lastest-pages-articles.xml.bz
 To generate HTML reports, you can use the following command:
 
 ```sh
-node report.js <language>
+node report.js <language> [path to HTML output]
 # For example:
 node report.js zh-yue
 ```
 
 ## Report
 
-The tool will generate reports in two formats: JSON and HTML. The JSON report will be saved in the `results` folder, while the HTML report will be available at `reports/index.html`.
+The tool will generate reports in two formats: JSON and HTML. The JSON report will be saved in the `results` folder, while the HTML report will be available at `reports/index.html` or the specified path.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lint-wiki-dumps",
-	"version": "0.0.0",
+	"version": "0.1.0",
 	"description": "Lint Wikipedia dumps",
 	"keywords": [
 		"lint",
@@ -20,7 +20,7 @@
 		"reports/*.css"
 	],
 	"bin": {
-		"lint-wiki-dumps": "scan.sh"
+		"lint-wiki-dumps": "scan-parallel.sh"
 	},
 	"repository": {
 		"type": "git",
@@ -32,10 +32,10 @@
 		"lint": "tsc --noEmit && tsc --project reports/tsconfig.json --noEmit && eslint --cache ."
 	},
 	"dependencies": {
-		"@bhsd/common": "^0.9.0",
+		"@bhsd/common": "^0.9.1",
 		"chalk": "^4.1.2",
 		"unbzip2-stream": "^1.4.3",
-		"wikilint": "^2.18.3",
+		"wikilint": "^2.18.4",
 		"xml-stream": "^0.4.5"
 	},
 	"optionalDependencies": {

package/parser-parallel.js

@@ -0,0 +1,90 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cluster_1 = __importDefault(require("cluster"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const chalk_1 = __importDefault(require("chalk"));
+const common_1 = require("@bhsd/common");
+const processor_1 = require("./processor");
+const [, , site, dir] = process.argv, target = `${site}wiki`;
+if (cluster_1.default.isPrimary) {
+    (0, processor_1.init)();
+    const dumpDir = dir.replace(/^~/u, os_1.default.homedir()), prefix = target.replaceAll('-', '_'), files = fs_1.default.readdirSync(dumpDir).filter(file => file.endsWith('.bz2') && file.startsWith(prefix))
+        .map(file => {
+        const filePath = path_1.default.join(dumpDir, file);
+        return [filePath, fs_1.default.statSync(filePath).size];
+    })
+        .sort(([, a], [, b]) => b - a), 
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
+    workers = new Array(Math.min(os_1.default.availableParallelism(), files.length)).fill(undefined)
+        .map(() => cluster_1.default.fork());
+    let i = 0, n = 0;
+    console.time('parse');
+    for (; i < workers.length; i++) {
+        const worker = workers[i];
+        worker.on('message', count => {
+            n += count;
+            if (i < files.length) {
+                worker.send([files[i], i]);
+                i++;
+            }
+            else {
+                worker.disconnect();
+            }
+        }).send([files[i], i]);
+    }
+    process.on('exit', () => {
+        console.timeEnd('parse');
+        console.log(chalk_1.default.green(`Parsed ${n} pages in total`));
+    });
+}
+else {
+    process.on('message', ([[file], j]) => {
+        const results = fs_1.default.createWriteStream(path_1.default.join(processor_1.resultDir, `${site}-${j}.json`)), processor = new processor_1.Processor(site, results);
+        let i = 0;
+        results.write('{');
+        results.on('close', () => {
+            process.send(i);
+        });
+        const stop = () => {
+            processor.stop(`parse ${file}`, `Parsed ${i} pages from ${file}`);
+        };
+        const lint = ($text, ns, title, date, retry = 0) => {
+            try {
+                processor.lint($text, ns, title, date);
+                return true;
+            }
+            catch (e) {
+                if (e instanceof RangeError && e.message === 'Maximum heap size exceeded') {
+                    if (retry === 0) {
+                        stream.pause();
+                    }
+                    else if (retry > 5) {
+                        processor.error(e, title);
+                        return true;
+                    }
+                    setTimeout(() => {
+                        if (lint($text, ns, title, date, retry + 1)) {
+                            stream.resume();
+                        }
+                    }, 1e4);
+                    return false;
+                }
+                throw e;
+            }
+        };
+        console.time(`parse ${file}`);
+        const stream = (0, processor_1.getXmlStream)(file);
+        stream.on('endElement: page', ({ title, ns, revision: { model, timestamp, text: { $text } } }) => {
+            if (model === 'wikitext' && $text && ns === '0') {
+                (0, common_1.refreshStdout)(`${i++} ${title}`);
+                lint($text, ns, title, new Date(timestamp));
+            }
+        });
+        stream.on('end', stop);
+    });
+}

package/parser.js

@@ -6,26 +6,30 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const os_1 = __importDefault(require("os"));
-const perf_hooks_1 = require("perf_hooks");
-const chalk_1 = __importDefault(require("chalk"));
-const unbzip2_stream_1 = __importDefault(require("unbzip2-stream"));
-const xml_stream_1 = __importDefault(require("xml-stream"));
-const wikilint_1 = __importDefault(require("wikilint"));
 const common_1 = require("@bhsd/common");
-const n = Number(process.argv[4]) || Infinity, [, , site, file, , restart] = process.argv;
-wikilint_1.default.config = `${site}wiki`;
-if (!fs_1.default.existsSync('results')) {
-    fs_1.default.mkdirSync('results');
-}
-const stream = new xml_stream_1.default(fs_1.default.createReadStream(file.replace(/^~/u, os_1.default.homedir())).pipe((0, unbzip2_stream_1.default)())), output = path_1.default.join('results', `${site}.json`);
-let old;
-try {
-    old = require(`./${output}`); // eslint-disable-line @typescript-eslint/no-require-imports
-}
-catch { }
-const time = old?.['#timestamp'], last = time && new Date(time), results = fs_1.default.createWriteStream(output, { flags: restart ? 'a' : 'w' }), ignore = new Set(['no-arg', 'url-encoding', 'h1', 'var-anchor']);
-let i = 0, latest = last, failed = 0, comma = restart ? ',' : '', stopping = false, restarted = !restart, worst;
-stream.preserve('text', true);
+const processor_1 = require("./processor");
+const n = Number(process.argv[4]) || Infinity, [, , site, file, , restart] = process.argv, filePath = path_1.default.join(processor_1.resultDir, `${site}.json`), data = fs_1.default.existsSync(filePath) && fs_1.default.readFileSync(filePath, 'utf8');
+const getTimestamp = () => {
+    if (!data) {
+        return undefined;
+    }
+    const i = data.indexOf('"#timestamp": "') + 15;
+    return data.slice(i, data.indexOf('"', i));
+};
+const getErrors = (page) => {
+    if (!data) {
+        return undefined;
+    }
+    const str = JSON.stringify(page), i = data.indexOf(`${str}: [`);
+    if (i === -1) {
+        return undefined;
+    }
+    const j = i + str.length + 2;
+    return JSON.parse(data.slice(j, data.indexOf('\n]', j) + 2));
+};
+(0, processor_1.init)();
+const time = getTimestamp(), last = time && new Date(time), results = fs_1.default.createWriteStream(path_1.default.join(processor_1.resultDir, `${site}.json`), { flags: restart ? 'a' : 'w' }), processor = new processor_1.Processor(site, results, last);
+let i = 0, stopping = false, restarted = !restart;
 if (!restart) {
     results.write('{');
 }
@@ -34,22 +38,10 @@ results.on('close', () => {
 });
 const stop = () => {
     stopping = true;
-    console.timeEnd('parse');
-    console.log(chalk_1.default.green(`Parsed ${i} pages`));
-    if (failed) {
-        console.error(chalk_1.default.red(`${failed} pages failed to parse`));
-    }
-    if (worst) {
-        console.info(chalk_1.default.yellow(`Worst page: ${worst.title} (${worst.duration.toFixed(3)} ms)`));
-    }
-    results.write(`${comma}\n"#timestamp": ${JSON.stringify(latest)}\n}`);
-    results.close();
-};
-const newEntry = (title, errors) => {
-    results.write(`${comma}\n${JSON.stringify(title)}: ${JSON.stringify(errors, null, '\t')}`);
-    comma ||= ',';
+    processor.stop('parse', `Parsed ${i} pages`);
 };
 console.time('parse');
+const stream = (0, processor_1.getXmlStream)(file.replace(/^~/u, os_1.default.homedir()));
 stream.on('endElement: page', ({ title, ns, revision: { model, timestamp, text: { $text } } }) => {
     if (i === n) {
         if (!stopping) {
@@ -60,37 +52,13 @@ stream.on('endElement: page', ({ title, ns, revision: { model, timestamp, text:
         (0, common_1.refreshStdout)(`${i++} ${title}`);
         const date = new Date(timestamp);
         if (last && date <= last) {
-            const previous = old[title];
+            const previous = getErrors(title);
             if (previous) {
-                newEntry(title, previous);
+                processor.newEntry(title, previous);
             }
         }
         else {
-            latest = !latest || date > latest ? date : latest;
-            try {
-                const start = perf_hooks_1.performance.now(), errors = wikilint_1.default.parse($text).lint()
-                    .filter(({ severity, rule }) => severity === 'error' && !ignore.has(rule)), duration = perf_hooks_1.performance.now() - start;
-                if (errors.length > 0) {
-                    newEntry(title, errors.map(({ severity, suggestions, fix, ...e }) => ({
-                        ...e,
-                        ...suggestions && {
-                            suggestions: suggestions.map(action => ({
-                                ...action,
-                                original: $text.slice(...action.range),
-                            })),
-                        },
-                        ...fix && { fix: { ...fix, original: $text.slice(...fix.range) } },
-                        excerpt: $text.slice(e.startIndex, e.endIndex),
-                    })));
-                }
-                if (!worst || duration > worst.duration) {
-                    worst = { title, duration };
-                }
-            }
-            catch (e) {
-                console.error(chalk_1.default.red(`Error parsing ${title}`), e);
-                failed++;
-            }
+            processor.lint($text, ns, title, date);
         }
     }
     else if (title === restart) {

package/processor.js

@@ -0,0 +1,117 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Processor = exports.getXmlStream = exports.init = exports.resultDir = exports.MAX = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const perf_hooks_1 = require("perf_hooks");
+const cluster_1 = __importDefault(require("cluster"));
+const chalk_1 = __importDefault(require("chalk"));
+const unbzip2_stream_1 = __importDefault(require("unbzip2-stream"));
+const xml_stream_1 = __importDefault(require("xml-stream"));
+const wikilint_1 = __importDefault(require("wikilint"));
+exports.MAX = 100, exports.resultDir = path_1.default.join(__dirname, 'results');
+const ignore = new Set(['no-arg', 'url-encoding', 'h1', 'var-anchor']);
+const init = () => {
+    if (!fs_1.default.existsSync(exports.resultDir)) {
+        fs_1.default.mkdirSync(exports.resultDir);
+    }
+};
+exports.init = init;
+const getXmlStream = (file) => {
+    const stream = new xml_stream_1.default(fs_1.default.createReadStream(file).pipe((0, unbzip2_stream_1.default)()));
+    stream.preserve('text', true);
+    return stream;
+};
+exports.getXmlStream = getXmlStream;
+class Processor {
+    #failed = 0;
+    #comma = '';
+    #worst;
+    #results;
+    #latest;
+    /** @param site site nickname */
+    constructor(site, results, latest) {
+        wikilint_1.default.config = `${site}wiki`;
+        this.#results = results;
+        this.#latest = latest;
+    }
+    /**
+     * Stop the processing and log the results.
+     * @param timer timer name
+     * @param msg message to log
+     */
+    stop(timer, msg) {
+        console.log();
+        console.timeEnd(timer);
+        console.log(chalk_1.default.green(msg));
+        if (this.#failed) {
+            console.error(chalk_1.default.red(`${this.#failed} pages failed to parse`));
+        }
+        if (this.#worst) {
+            console.info(chalk_1.default.yellow(`Worst page: ${this.#worst.title} (${this.#worst.duration.toFixed(3)} ms)`));
+        }
+        this.#results.write(`${this.#comma}\n"#timestamp": ${JSON.stringify(this.#latest)}\n}`);
+        this.#results.end();
+    }
+    /**
+     * Write a new entry to the results file.
+     * @param title page title
+     * @param errors lint errors
+     */
+    newEntry(title, errors) {
+        this.#results.write(`${this.#comma}\n${JSON.stringify(title)}: ${JSON.stringify(errors, null, '\t')}`);
+        this.#comma ||= ',';
+    }
+    /**
+     * Parse a page and lint it.
+     * @param $text page text
+     * @param ns page namespace
+     * @param title page title
+     * @param date page revision date
+     * @throws `RangeError` maximum heap size exceeded
+     */
+    lint($text, ns, title, date) {
+        if (!this.#latest || date > this.#latest) {
+            this.#latest = date;
+        }
+        try {
+            const start = perf_hooks_1.performance.now(), errors = wikilint_1.default.parse($text, ns === '828').lint()
+                .filter(({ severity, rule }) => severity === 'error' && !ignore.has(rule)), duration = perf_hooks_1.performance.now() - start;
+            if (errors.length > 0) {
+                this.newEntry(title, errors.map(({ severity, suggestions, fix, ...e }) => ({
+                    ...e,
+                    ...suggestions && {
+                        suggestions: suggestions.map(action => ({
+                            ...action,
+                            original: $text.slice(...action.range),
+                        })),
+                    },
+                    ...fix && { fix: { ...fix, original: $text.slice(...fix.range) } },
+                    excerpt: $text.slice(e.startIndex, e.endIndex).slice(0, exports.MAX),
+                })));
+            }
+            if (!this.#worst || duration > this.#worst.duration) {
+                this.#worst = { title, duration };
+            }
+        }
+        catch (e) {
+            if (cluster_1.default.isWorker && e instanceof RangeError && e.message === 'Maximum heap size exceeded') {
+                throw e;
+            }
+            this.error(e, title);
+        }
+    }
+    /**
+     * Log an error message.
+     * @param e error object
+     * @param title page title
+     */
+    error(e, title) {
+        console.error(chalk_1.default.red(`Error parsing ${title}`), e);
+        this.#failed++;
+    }
+}
+exports.Processor = Processor;

package/report.js

@@ -7,65 +7,105 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const crypto_1 = require("crypto");
 const chalk_1 = __importDefault(require("chalk"));
-const [, , lang] = process.argv;
-const mkdir = (dir) => {
-    if (!fs_1.default.existsSync(dir)) {
-        fs_1.default.mkdirSync(dir);
+const processor_1 = require("./processor");
+const { argv } = process, [, , lang] = argv, defaultOurDir = path_1.default.join(__dirname, 'reports');
+let [, , , outDir] = argv;
+const mkdir = (dir, empty) => {
+    if (fs_1.default.existsSync(dir)) {
+        if (!empty) {
+            return;
+        }
+        fs_1.default.rmSync(dir, { recursive: true });
     }
+    fs_1.default.mkdirSync(dir);
 };
-const dataDir = path_1.default.join('reports', 'data');
+if (outDir) {
+    mkdir(outDir);
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
+    fs_1.default.cpSync(defaultOurDir, outDir, { recursive: true, force: true });
+}
+else {
+    outDir = defaultOurDir;
+}
+const dataDir = path_1.default.join(outDir, 'data');
 mkdir(dataDir);
 const writeJS = (data, file) => {
     fs_1.default.writeFileSync(path_1.default.join(dataDir, `${file}.js`), `window.data=${JSON.stringify(data)}`);
 };
-const dir = fs_1.default.readdirSync('results'), summary = [], MAX = 100;
+const initJS = (file) => {
+    const stream = fs_1.default.createWriteStream(`${file}.js`);
+    stream.write('window.data={"articles":[');
+    return stream;
+};
+const compare = (a, b) => a.localeCompare(b);
+const resultDir = path_1.default.join(__dirname, 'results'), dir = fs_1.default.readdirSync(resultDir), summary = new Set(), ruleRecords = new Map(), wiki = {}, siteDir = path_1.default.join(dataDir, lang), articlesDir = path_1.default.join(siteDir, 'pages');
+let latest;
+mkdir(siteDir, true);
+mkdir(articlesDir);
 for (const file of dir) {
-    try {
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-        const data = require(`./results/${file}`);
-        const site = file.slice(0, -5);
-        summary.push(site);
-        delete data['#timestamp'];
-        if (!lang || lang === site) {
-            const siteDir = path_1.default.join(dataDir, site);
-            mkdir(siteDir);
+    if (!file.endsWith('.json')) {
+        continue;
+    }
+    const fileDir = path_1.default.join(resultDir, file);
+    if (!fs_1.default.existsSync(fileDir)) {
+        console.error(chalk_1.default.red(`Failed to read ${file}`));
+        continue;
+    }
+    const k = file.search(/-\d+\.json$/u), site = k === -1 ? file.slice(0, -5) : file.slice(0, k);
+    summary.add(site);
+    if (lang !== site) {
+        continue;
+    }
+    const data = fs_1.default.readFileSync(fileDir, 'utf8'), date = new Date(data.substr(data.indexOf('\n"#timestamp": "') + 16, 10));
+    latest = !latest || date > latest ? date : latest;
+    for (const mt of data.matchAll(/^(".+"): \[$/gmu)) {
+        const page = JSON.parse(mt[1]), hash = (0, crypto_1.createHash)('sha256').update(page).digest('hex')
+            .slice(0, 8), errors = JSON.parse(data.slice(mt.index + mt[0].length - 1, data.indexOf('\n]', mt.index) + 2)), rules = new Set(), info = [];
+        for (const { rule, startLine, startCol, message, excerpt } of errors) {
+            // article
+            const line = startLine + 1, col = startCol + 1;
+            info.push([rule, line, col, message, excerpt]);
             // wiki
-            const values = Object.values(data), rules = [...new Set(values.flat().map(({ rule }) => rule))].sort((a, b) => a.localeCompare(b)), wiki = [];
-            for (const rule of rules) {
-                wiki.push([rule, values.filter(errors => errors.some(({ rule: r }) => r === rule)).length]);
-                // rule
-                const articles = Object.entries(data).filter(([, errors]) => errors.some(({ rule: r }) => r === rule))
-                    .sort(([a], [b]) => a.localeCompare(b))
-                    .map(([page, errors]) => {
-                    const { startLine, startCol, message, excerpt } = errors.find(({ rule: r }) => r === rule);
-                    return [page, startLine + 1, startCol + 1, message, excerpt.slice(0, MAX * 0.8)];
-                }), batches = Math.ceil(articles.length / 200);
-                for (let i = 0; i < batches; i++) {
-                    writeJS({
-                        articles: articles.slice(i * 200, (i + 1) * 200),
-                        batches,
-                    }, path_1.default.join(site, `${rule}-${i}`));
-                }
+            if (!(rule in wiki)) {
+                wiki[rule] = 0;
             }
-            writeJS(wiki, path_1.default.join(site, 'index'));
-            // article
-            const articlesDir = path_1.default.join(siteDir, 'pages');
-            mkdir(articlesDir);
-            for (const [page, errors] of Object.entries(data)) {
-                const hash = (0, crypto_1.createHash)('sha256').update(page).digest('hex')
-                    .slice(0, 8), info = errors.map(({ startLine, startCol, rule, message, excerpt, }) => [
-                    rule,
-                    startLine + 1,
-                    startCol + 1,
-                    message,
-                    excerpt.slice(0, MAX),
-                ]);
-                writeJS(info, path_1.default.join(site, 'pages', hash));
+            // rule
+            if (!rules.has(rule)) {
+                rules.add(rule);
+                wiki[rule]++;
+                let ruleRecord;
+                if (ruleRecords.has(rule)) {
+                    ruleRecord = ruleRecords.get(rule);
+                }
+                else {
+                    ruleRecord = ['', []];
+                    ruleRecords.set(rule, ruleRecord);
+                }
+                ruleRecord[1].push(page);
+                ruleRecord[0] += `${JSON.stringify([page, line, col, message, excerpt.slice(0, processor_1.MAX * 0.8)], null, '\t')},`;
             }
         }
+        writeJS(info, path_1.default.join(site, 'pages', hash));
     }
-    catch {
-        console.error(chalk_1.default.red(`Failed to read ${file}`));
+}
+const timestamp = latest.toISOString().slice(0, 10);
+// rule
+for (const [rule, [str, pages]] of ruleRecords) {
+    const batches = Math.ceil(pages.length / 200);
+    pages.sort(compare);
+    for (let i = 0; i < batches; i++) {
+        const stream = initJS(path_1.default.join(siteDir, `${rule}-${i}`));
+        for (let j = i * 200; j < (i + 1) * 200; j++) {
+            const page = pages[j];
+            if (!page) {
+                break;
+            }
+            const index = str.indexOf(`[\n\t${JSON.stringify(page)}`);
+            stream.write(str.slice(index, str.indexOf('\n]', index) + 3));
+        }
+        stream.write(`],batches:${batches},timestamp:"${timestamp}"}`);
+        stream.end();
     }
 }
-writeJS(summary, 'index');
+writeJS([...summary].sort(compare), 'index');
+writeJS([...Object.entries(wiki).sort(([a], [b]) => a.localeCompare(b)), timestamp], path_1.default.join(lang, 'index'));

package/reports/dist/rule.js

@@ -1,7 +1,6 @@
 "use strict";
 (() => {
-    const search = new URLSearchParams(location.search), lang = search.get('lang'), rule = search.get('rule'), batch = Math.floor(Number(search.get('start') || 0) / 200), endStr = String((batch + 1) * 200), prev = document.getElementById('prev'), next = document.getElementById('next'), start = document.getElementById('start'), end = document.getElementById('end'), title = document.querySelector('title'), h2 = document.querySelector('h2'), wiki = document.getElementById('wiki'), tbody = document.querySelector('tbody'), script = document.createElement('script');
-    h2.textContent = h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org: ${rule}`);
+    const search = new URLSearchParams(location.search), lang = search.get('lang'), rule = search.get('rule'), batch = Math.floor(Number(search.get('start') || 0) / 200), endStr = String((batch + 1) * 200), nav = document.getElementById('nav'), prev = document.getElementById('prev'), next = document.getElementById('next'), start = document.getElementById('start'), end = document.getElementById('end'), title = document.querySelector('title'), h2 = document.querySelector('h2'), wiki = document.getElementById('wiki'), table = document.querySelector('table'), tbody = document.querySelector('tbody'), script = document.createElement('script');
     title.textContent = title.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
     wiki.textContent = `${lang}wiki`;
     wiki.href += `?lang=${lang}`;
@@ -18,6 +17,7 @@
     next.href = `${location.pathname}?${search}`;
     script.src = `./data/${lang}/${rule}-${batch}.js`;
     script.addEventListener('load', () => {
+        h2.textContent = `${h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org: ${rule}`)} (${data.timestamp})`;
         if (data.batches === batch + 1) {
             next.removeAttribute('href');
             end.textContent = String(batch * 200 + data.articles.length);
@@ -43,6 +43,7 @@
             tr.append(article, edit, line, column, detail, notice, more);
             tbody.append(tr);
         }
+        table.after(nav.cloneNode(true));
     });
     document.head.append(script);
 })();

package/reports/dist/wiki.js

@@ -1,11 +1,11 @@
 "use strict";
 (() => {
     const lang = new URLSearchParams(location.search).get('lang'), script = document.createElement('script'), title = document.querySelector('title'), h2 = document.querySelector('h2'), tbody = document.querySelector('tbody');
-    h2.textContent = h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
     title.textContent = title.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
     script.src = `./data/${lang}/index.js`;
     script.addEventListener('load', () => {
-        for (const [rule, count] of data) {
+        h2.textContent = `${h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org`)} (${data.slice(-1)[0]})`;
+        for (const [rule, count] of data.slice(0, -1)) {
             const tr = document.createElement('tr'), description = document.createElement('td'), pages = document.createElement('td'), a = document.createElement('a');
             a.textContent = rule;
             a.href = `./rule.html?lang=${lang}&rule=${rule}`;

package/scan-parallel.sh

@@ -0,0 +1,23 @@
+#!/usr/local/bin/bash
+if (( $# < 2 ))
+then
+	echo 'Usage: npx lint-wiki-dumps <language> <path to download> [path to HTML output]'
+	echo 'Example: npx lint-wiki-dumps zh-yue ~/Downloads/dumps'
+	exit 1
+fi
+site="${1}wiki" # example: zh-yuewiki
+target="${1//-/_}wiki" # example: zh_yuewiki
+files=$( \
+	curl -s "https://dumps.wikimedia.org/$target/latest/" \
+	| grep -o "href=\"$target-latest-pages-articles[0-9].*\.bz2\">" \
+	| gsed "s|href=\"|https://dumps.wikimedia.org/$target/latest/|;s|\">||" \
+)
+if (( ${#files} < 2 ))
+then
+	echo 'Switching to single-threaded mode'
+	bash scan.sh "$1" "$2" "$3"
+else
+	curl --output-dir "$2" --remote-name-all $files
+	npx getParserConfig "$site" "https://$1.wikipedia.org/w/"
+	node parser-parallel.js "$1" "$2" && node report.js "$1" "$3"
+fi

package/scan.sh

@@ -1,16 +1,16 @@
 #!/usr/local/bin/bash
 if (( $# < 2 ))
 then
-	echo 'Usage: npx lint-wiki-dumps <language> <path to download>'
+	echo 'Usage: npx lint-wiki-dumps <language> <path to download> [path to HTML output]'
 	echo 'Example: npx lint-wiki-dumps zh-yue ~/Downloads/dumps'
 	exit 1
 fi
 site="${1}wiki" # example: zh-yuewiki
 target="${1//-/_}wiki" # example: zh_yuewiki
-file="${target}-latest-pages-articles.xml.bz2"
+file="$target-latest-pages-articles.xml.bz2"
 if (( $# < 3 ))
 then
 	curl --output-dir "$2" -O "https://dumps.wikimedia.org/$target/latest/$file"
 	npx getParserConfig "$site" "https://$1.wikipedia.org/w/"
 fi
-node parser.js "$1" "$2/$file" "$3" "$4" && node report.js "$1"
+node parser.js "$1" "$2/$file" "$4" "$5" && node report.js "$1" "$3"