lint-wiki-dumps 0.0.0 → 0.2.0

package/README.md

@@ -23,7 +23,7 @@ npm i vscode-css-languageservice
 ## Usage
 
 ```sh
-npx lint-wiki-dumps <language> <path to download directory>
+npx lint-wiki-dumps <language> <path to download> [path to HTML output]
 # For example:
 npx lint-wiki-dumps zh-yue ~/Downloads/dumps
 ```
@@ -31,7 +31,7 @@ npx lint-wiki-dumps zh-yue ~/Downloads/dumps
 or execute the Bash script `scan.sh` directly:
 
 ```sh
-bash scan.sh <language> <path to download>
+bash scan.sh <language> <path to download> [path to HTML output]
 # For example:
 bash scan.sh zh-yue ~/Downloads/dumps
 ```
@@ -49,11 +49,11 @@ node parser.js zh-yue ~/Downloads/dumps/zh-yuewiki-lastest-pages-articles.xml.bz
 To generate HTML reports, you can use the following command:
 
 ```sh
-node report.js <language>
+node report.js <language> [path to HTML output]
 # For example:
 node report.js zh-yue
 ```
 
 ## Report
 
-The tool will generate reports in two formats: JSON and HTML. The JSON report will be saved in the `results` folder, while the HTML report will be available at `reports/index.html`.
+The tool will generate reports in two formats: JSON and HTML. The JSON report will be saved in the `results` folder, while the HTML report will be available at `reports/index.html` or the specified path.

package/download.sh

@@ -0,0 +1,16 @@
+#!/usr/local/bin/bash
+path="https://dumps.wikimedia.org/$1/latest/"
+files=$( \
+	curl -s "$path" \
+	| grep -o "href=\"$1-latest-pages-articles[0-9].*\.bz2\">" \
+	| gsed "s|href=\"|$path|;s|\">||" \
+)
+filtered=$(node filter.js $files)
+if (( ${#filtered} < 2 ))
+then
+	file="$path/$1-latest-pages-articles.xml.bz2"
+	curl --output-dir "$2" -O "$file"
+	exit 1
+else
+	curl --output-dir "$2" --remote-name-all $filtered
+fi

package/filter.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const files = process.argv.slice(2)
+    .map(file => [
+    file,
+    .../\.xml-p(\d+)p(\d+)\.bz2$/u.exec(file).slice(1).map(Number),
+])
+    .sort(([, a1, a2], [, b1, b2]) => a1 - b1 || a2 - b2)
+    .filter(([, a], i, arr) => a !== arr[i + 1]?.[1])
+    .map(([file]) => file)
+    .join(' ');
+console.log(files);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "lint-wiki-dumps",
-	"version": "0.0.0",
+	"version": "0.2.0",
 	"description": "Lint Wikipedia dumps",
 	"keywords": [
 		"lint",
@@ -13,7 +13,8 @@
 	"license": "GPL-3.0",
 	"author": "Bhsd",
 	"files": [
-		"scan.sh",
+		"*.sh",
+		"!bump.sh",
 		"*.js",
 		"reports/*.html",
 		"reports/dist/",
@@ -27,15 +28,16 @@
 		"url": "git+https://github.com/bhsd-harry/lint-wiki-dumps.git"
 	},
 	"scripts": {
+		"start": "node server.js",
 		"prepublishOnly": "npm run build",
-		"build": "tsc && mv dist/* . && tsc --project reports/tsconfig.json",
+		"build": "tsc && mv dist/* . && esbuild reports/src/*.ts --charset=utf8 --bundle --minify --target=es2019 --sourcemap --outdir=reports/dist && rm reports/dist/common.js*",
 		"lint": "tsc --noEmit && tsc --project reports/tsconfig.json --noEmit && eslint --cache ."
 	},
 	"dependencies": {
-		"@bhsd/common": "^0.9.0",
+		"@bhsd/common": "^0.9.3",
 		"chalk": "^4.1.2",
 		"unbzip2-stream": "^1.4.3",
-		"wikilint": "^2.18.3",
+		"wikilint": "^2.20.0",
 		"xml-stream": "^0.4.5"
 	},
 	"optionalDependencies": {
@@ -47,9 +49,9 @@
 		"@types/mocha": "^10.0.10",
 		"@types/node": "^22.13.1",
 		"@types/unbzip2-stream": "^1.4.3",
-		"@typescript-eslint/eslint-plugin": "^8.23.0",
-		"@typescript-eslint/parser": "^8.23.0",
-		"esbuild": "^0.25.0",
+		"@typescript-eslint/eslint-plugin": "^8.29.0",
+		"@typescript-eslint/parser": "^8.29.0",
+		"esbuild": "^0.25.2",
 		"eslint": "^8.57.1",
 		"eslint-plugin-es-x": "^8.4.1",
 		"eslint-plugin-eslint-comments": "^3.2.0",
@@ -63,7 +65,7 @@
 		"http-server": "^14.1.1",
 		"mocha": "^11.1.0",
 		"stylelint": "^16.14.1",
-		"typescript": "^5.7.3"
+		"typescript": "^5.8.2"
 	},
 	"engines": {
 		"node": ">=18.17.0"

package/parser-parallel.js

@@ -0,0 +1,125 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const cluster_1 = __importDefault(require("cluster"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const chalk_1 = __importDefault(require("chalk"));
+const common_1 = require("@bhsd/common");
+const util_1 = require("./util");
+const processor_1 = require("./processor");
+const [, , site, dir, refresh] = process.argv, target = (0, util_1.normalize)(site);
+if (cluster_1.default.isPrimary) {
+    (0, util_1.init)();
+    const tempFiles = [];
+    for (const file of fs_1.default.readdirSync(util_1.resultDir)) {
+        if (file.startsWith(`${target}-p`) && file.endsWith('.json')) {
+            const oldName = path_1.default.join(util_1.resultDir, file), newName = path_1.default.join(util_1.resultDir, `temp${file.slice(target.length)}`);
+            (0, util_1.reading)(oldName);
+            tempFiles.push(newName);
+            fs_1.default.renameSync(oldName, newName);
+        }
+    }
+    const dumpDir = (0, util_1.replaceTilde)(dir), prefix = `${target}wiki`, files = fs_1.default.readdirSync(dumpDir).filter(file => file.endsWith('.bz2') && file.startsWith(prefix))
+        .map(file => {
+        const filePath = path_1.default.join(dumpDir, file);
+        return [filePath, fs_1.default.statSync(filePath).size];
+    })
+        .sort(([, a], [, b]) => b - a), 
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
+    workers = new Array(Math.min(os_1.default.availableParallelism(), files.length)).fill(undefined)
+        .map(() => cluster_1.default.fork());
+    let i = 0, n = 0, m = 0;
+    console.time('parse');
+    for (; i < workers.length; i++) {
+        const worker = workers[i];
+        // eslint-disable-next-line @typescript-eslint/no-loop-func
+        worker.on('message', ([count, total]) => {
+            n += count;
+            m += total;
+            if (i < files.length) {
+                worker.send(files[i][0]);
+                i++;
+            }
+            else {
+                worker.disconnect();
+            }
+        }).send(files[i][0]);
+    }
+    process.on('exit', () => {
+        console.timeEnd('parse');
+        console.log(chalk_1.default.green(`Parsed ${n} / ${m} pages in total`));
+        for (const file of tempFiles) {
+            fs_1.default.unlinkSync(file);
+        }
+    });
+}
+else {
+    const getStartEnd = (f) => {
+        const p2 = f.lastIndexOf('p');
+        return [Number(f.slice(6, p2)), Number(f.slice(p2 + 1, -5))];
+    };
+    const tempFiles = fs_1.default.readdirSync(util_1.resultDir)
+        .filter(file => file.startsWith('temp-p') && file.endsWith('.json')), ranges = tempFiles.map(getStartEnd), max = Math.max(...ranges.map(([, end]) => end));
+    let start, end, last, data;
+    process.on('message', (file) => {
+        const filename = `${target}${file.slice(file.lastIndexOf('-'), -4)}.json`, tempPath = (0, util_1.getTempPath)(filename), results = (0, util_1.getWriteStream)(tempPath, () => {
+            fs_1.default.renameSync(tempPath, path_1.default.join(util_1.resultDir, filename));
+            process.send([processor.parsed, i]);
+        }), processor = new processor_1.Processor(site, results, refresh);
+        let i = 0;
+        const stop = () => {
+            processor.stop(`parse ${file}`, `${i} pages from ${file}`);
+        };
+        const lint = ($text, ns, title, date, retry = 0) => {
+            try {
+                processor.lint($text, ns, title, date, last, data);
+                return true;
+            }
+            catch (e) {
+                if (e instanceof RangeError && e.message === 'Maximum heap size exceeded') {
+                    if (retry === 0) {
+                        stream.pause();
+                    }
+                    else if (retry > 5) {
+                        processor.error(e, title);
+                        return true;
+                    }
+                    setTimeout(() => {
+                        if (lint($text, ns, title, date, retry + 1)) {
+                            stream.resume();
+                        }
+                    }, 1e4);
+                    return false;
+                }
+                throw e;
+            }
+        };
+        console.time(`parse ${file}`);
+        const stream = (0, util_1.getXmlStream)(file);
+        stream.on('endElement: page', ({ title, ns, id, revision: { model, timestamp, text: { $text } } }) => {
+            if ((0, util_1.isArticle)($text, ns, model)) {
+                (0, common_1.refreshStdout)(`${i++} ${title}`);
+                const pageid = Number(id);
+                if (start === undefined || end === undefined || pageid < start || pageid > end) {
+                    const cur = pageid <= max && ranges.findIndex(([a, b]) => a <= pageid && b >= pageid);
+                    if (cur === false || cur === -1) {
+                        start = undefined;
+                        end = undefined;
+                        last = undefined;
+                    }
+                    else {
+                        [start, end] = ranges[cur];
+                        data = fs_1.default.readFileSync(path_1.default.join(util_1.resultDir, tempFiles[cur]), 'utf8');
+                        last = (0, util_1.getTimestamp)(data);
+                    }
+                }
+                lint($text, ns, title, new Date(timestamp));
+            }
+        });
+        stream.on('end', stop);
+    });
+}

package/parser.js

@@ -5,96 +5,27 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const os_1 = __importDefault(require("os"));
-const perf_hooks_1 = require("perf_hooks");
-const chalk_1 = __importDefault(require("chalk"));
-const unbzip2_stream_1 = __importDefault(require("unbzip2-stream"));
-const xml_stream_1 = __importDefault(require("xml-stream"));
-const wikilint_1 = __importDefault(require("wikilint"));
 const common_1 = require("@bhsd/common");
-const n = Number(process.argv[4]) || Infinity, [, , site, file, , restart] = process.argv;
-wikilint_1.default.config = `${site}wiki`;
-if (!fs_1.default.existsSync('results')) {
-    fs_1.default.mkdirSync('results');
+const util_1 = require("./util");
+const processor_1 = require("./processor");
+const [, , site, file, refresh] = process.argv, filename = `${(0, util_1.normalize)(site)}.json`, filePath = path_1.default.join(util_1.resultDir, filename), tempPath = (0, util_1.getTempPath)(filename), data = fs_1.default.existsSync(filePath) && fs_1.default.readFileSync(filePath, 'utf8');
+if (data) {
+    (0, util_1.reading)(filePath);
 }
-const stream = new xml_stream_1.default(fs_1.default.createReadStream(file.replace(/^~/u, os_1.default.homedir())).pipe((0, unbzip2_stream_1.default)())), output = path_1.default.join('results', `${site}.json`);
-let old;
-try {
-    old = require(`./${output}`); // eslint-disable-line @typescript-eslint/no-require-imports
-}
-catch { }
-const time = old?.['#timestamp'], last = time && new Date(time), results = fs_1.default.createWriteStream(output, { flags: restart ? 'a' : 'w' }), ignore = new Set(['no-arg', 'url-encoding', 'h1', 'var-anchor']);
-let i = 0, latest = last, failed = 0, comma = restart ? ',' : '', stopping = false, restarted = !restart, worst;
-stream.preserve('text', true);
-if (!restart) {
-    results.write('{');
-}
-results.on('close', () => {
+(0, util_1.init)();
+const last = (0, util_1.getTimestamp)(data), results = (0, util_1.getWriteStream)(tempPath, () => {
+    fs_1.default.renameSync(tempPath, filePath);
     process.exit(); // eslint-disable-line n/no-process-exit
-});
-const stop = () => {
-    stopping = true;
-    console.timeEnd('parse');
-    console.log(chalk_1.default.green(`Parsed ${i} pages`));
-    if (failed) {
-        console.error(chalk_1.default.red(`${failed} pages failed to parse`));
-    }
-    if (worst) {
-        console.info(chalk_1.default.yellow(`Worst page: ${worst.title} (${worst.duration.toFixed(3)} ms)`));
-    }
-    results.write(`${comma}\n"#timestamp": ${JSON.stringify(latest)}\n}`);
-    results.close();
-};
-const newEntry = (title, errors) => {
-    results.write(`${comma}\n${JSON.stringify(title)}: ${JSON.stringify(errors, null, '\t')}`);
-    comma ||= ',';
-};
+}), processor = new processor_1.Processor(site, results, refresh, last);
+let i = 0;
 console.time('parse');
+const stream = (0, util_1.getXmlStream)((0, util_1.replaceTilde)(file));
 stream.on('endElement: page', ({ title, ns, revision: { model, timestamp, text: { $text } } }) => {
-    if (i === n) {
-        if (!stopping) {
-            stop();
-        }
-    }
-    else if (restarted && model === 'wikitext' && $text && ns === '0') {
+    if ((0, util_1.isArticle)($text, ns, model)) {
         (0, common_1.refreshStdout)(`${i++} ${title}`);
-        const date = new Date(timestamp);
-        if (last && date <= last) {
-            const previous = old[title];
-            if (previous) {
-                newEntry(title, previous);
-            }
-        }
-        else {
-            latest = !latest || date > latest ? date : latest;
-            try {
-                const start = perf_hooks_1.performance.now(), errors = wikilint_1.default.parse($text).lint()
-                    .filter(({ severity, rule }) => severity === 'error' && !ignore.has(rule)), duration = perf_hooks_1.performance.now() - start;
-                if (errors.length > 0) {
-                    newEntry(title, errors.map(({ severity, suggestions, fix, ...e }) => ({
-                        ...e,
-                        ...suggestions && {
-                            suggestions: suggestions.map(action => ({
-                                ...action,
-                                original: $text.slice(...action.range),
-                            })),
-                        },
-                        ...fix && { fix: { ...fix, original: $text.slice(...fix.range) } },
-                        excerpt: $text.slice(e.startIndex, e.endIndex),
-                    })));
-                }
-                if (!worst || duration > worst.duration) {
-                    worst = { title, duration };
-                }
-            }
-            catch (e) {
-                console.error(chalk_1.default.red(`Error parsing ${title}`), e);
-                failed++;
-            }
-        }
-    }
-    else if (title === restart) {
-        restarted = true;
+        processor.lint($text, ns, title, new Date(timestamp), last, data);
     }
 });
-stream.on('end', stop);
+stream.on('end', () => {
+    processor.stop('parse', `${i} pages`);
+});

package/processor.js

@@ -0,0 +1,118 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Processor = void 0;
+const cluster_1 = __importDefault(require("cluster"));
+const chalk_1 = __importDefault(require("chalk"));
+const wikilint_1 = __importDefault(require("wikilint"));
+const util_1 = require("./util");
+const ignore = new Set(['no-arg', 'url-encoding', 'h1', 'var-anchor']);
+class Processor {
+    parsed = 0;
+    #failed = 0;
+    #comma = '';
+    #results;
+    #refresh;
+    #latest;
+    /** @param site site nickname */
+    constructor(site, results, refresh, latest) {
+        wikilint_1.default.config = `${site}wiki`;
+        this.#results = results;
+        this.#refresh = Boolean(refresh);
+        this.#latest = latest;
+    }
+    /**
+     * Stop the processing and log the results.
+     * @param timer timer name
+     * @param msg additional message to log
+     */
+    stop(timer, msg = '') {
+        console.log();
+        console.timeEnd(timer);
+        console.log(chalk_1.default.green(`Parsed ${this.parsed} / ${msg}`));
+        if (this.#failed) {
+            console.error(chalk_1.default.red(`${this.#failed} pages failed to parse`));
+        }
+        this.#results.write(`${this.#comma}\n"#timestamp": ${JSON.stringify(this.#latest)}\n}`);
+        this.#results.end();
+    }
+    /**
+     * Write a new entry to the results file.
+     * @param title page title
+     * @param errors lint errors
+     */
+    newEntry(title, errors) {
+        this.#results.write(`${this.#comma}\n${JSON.stringify(title)}: ${typeof errors === 'string' ? errors : JSON.stringify(errors, null, '\t')}`);
+        this.#comma ||= ',';
+    }
+    /**
+     * Parse a page and lint it.
+     * @param $text page text
+     * @param ns page namespace
+     * @param title page title
+     * @param date page revision date
+     * @param last last revision date
+     * @param data previous results
+     * @throws `RangeError` maximum heap size exceeded
+     */
+    lint($text, ns, title, date, last, data) {
+        if (!this.#latest || date > this.#latest) {
+            this.#latest = date;
+        }
+        if (last && date <= last) {
+            const previous = (0, util_1.getErrors)(data, title);
+            if (!previous) {
+                return;
+            }
+            else if (!this.#refresh) {
+                this.newEntry(title, previous);
+                return;
+            }
+        }
+        try {
+            const errors = wikilint_1.default.parse($text, ns === '828').lint()
+                .filter(({ severity, rule }) => severity === 'error' && !ignore.has(rule));
+            this.parsed++;
+            if (errors.length > 0) {
+                this.newEntry(title, errors.map(({ severity, suggestions, fix, 
+                /* DISABLED */
+                code, startIndex, endLine, endCol, endIndex, 
+                /* DISABLED END */
+                ...e }) => ({
+                    ...e,
+                    // eslint-disable-next-line @stylistic/multiline-comment-style
+                    /* DISABLED
+
+                    ...suggestions && {
+                        suggestions: suggestions.map(action => ({
+                            ...action,
+                            original: $text.slice(...action.range),
+                        })),
+                    },
+                    ...fix && {fix: {...fix, original: $text.slice(...fix.range)}},
+
+                    */
+                    excerpt: $text.slice(startIndex, endIndex).slice(0, util_1.MAX),
+                })));
+            }
+        }
+        catch (e) {
+            if (cluster_1.default.isWorker && e instanceof RangeError && e.message === 'Maximum heap size exceeded') {
+                throw e;
+            }
+            this.error(e, title);
+        }
+    }
+    /**
+     * Log an error message.
+     * @param e error object
+     * @param title page title
+     */
+    error(e, title) {
+        console.error(chalk_1.default.red(`Error parsing ${title}`), e);
+        this.#failed++;
+    }
+}
+exports.Processor = Processor;

package/report.js

@@ -7,65 +7,106 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const crypto_1 = require("crypto");
 const chalk_1 = __importDefault(require("chalk"));
-const [, , lang] = process.argv;
-const mkdir = (dir) => {
-    if (!fs_1.default.existsSync(dir)) {
-        fs_1.default.mkdirSync(dir);
+const util_1 = require("./util");
+const { argv } = process, [, , lang] = argv, defaultOurDir = path_1.default.join(__dirname, 'reports');
+let [, , , outDir] = argv;
+const mkdir = (dir, empty) => {
+    if (fs_1.default.existsSync(dir)) {
+        if (!empty) {
+            return;
+        }
+        fs_1.default.rmSync(dir, { recursive: true });
     }
+    fs_1.default.mkdirSync(dir);
 };
-const dataDir = path_1.default.join('reports', 'data');
+if (outDir) {
+    mkdir(outDir);
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
+    fs_1.default.cpSync(defaultOurDir, outDir, { recursive: true, force: true });
+}
+else {
+    outDir = defaultOurDir;
+}
+const dataDir = path_1.default.join(outDir, 'data');
 mkdir(dataDir);
 const writeJS = (data, file) => {
     fs_1.default.writeFileSync(path_1.default.join(dataDir, `${file}.js`), `window.data=${JSON.stringify(data)}`);
 };
-const dir = fs_1.default.readdirSync('results'), summary = [], MAX = 100;
+const initJS = (file) => {
+    const stream = fs_1.default.createWriteStream(`${file}.js`);
+    stream.write('window.data={"articles":[');
+    return stream;
+};
+const compare = (a, b) => a.localeCompare(b);
+const dir = fs_1.default.readdirSync(util_1.resultDir), summary = new Set(), ruleRecords = new Map(), wiki = {}, siteDir = path_1.default.join(dataDir, lang), articlesDir = path_1.default.join(siteDir, 'pages');
+let latest;
+mkdir(siteDir, true);
+mkdir(articlesDir);
 for (const file of dir) {
-    try {
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-        const data = require(`./results/${file}`);
-        const site = file.slice(0, -5);
-        summary.push(site);
-        delete data['#timestamp'];
-        if (!lang || lang === site) {
-            const siteDir = path_1.default.join(dataDir, site);
-            mkdir(siteDir);
+    if (!file.endsWith('.json')) {
+        continue;
+    }
+    const fileDir = path_1.default.join(util_1.resultDir, file);
+    if (!fs_1.default.existsSync(fileDir)) {
+        console.error(chalk_1.default.red(`Failed to read ${file}`));
+        continue;
+    }
+    const k = file.search(/-(?:p\d+){2}\.json$/u), site = (k === -1 ? file.slice(0, -5) : file.slice(0, k))
+        .replaceAll('_', '-');
+    summary.add(site);
+    if (lang !== site) {
+        continue;
+    }
+    const data = fs_1.default.readFileSync(fileDir, 'utf8'), date = new Date(data.substr(data.indexOf('\n"#timestamp": "') + 16, 10));
+    latest = !latest || date > latest ? date : latest;
+    for (const mt of data.matchAll(/^(".+"): \[$/gmu)) {
+        const page = JSON.parse(mt[1]), hash = (0, crypto_1.createHash)('sha256').update(page).digest('hex')
+            .slice(0, 8), errors = JSON.parse(data.slice(mt.index + mt[0].length - 1, data.indexOf('\n]', mt.index) + 2)), rules = new Set(), info = [];
+        for (const { rule, startLine, startCol, message, excerpt } of errors) {
+            // article
+            const line = startLine + 1, col = startCol + 1;
+            info.push([rule, line, col, message, excerpt]);
             // wiki
-            const values = Object.values(data), rules = [...new Set(values.flat().map(({ rule }) => rule))].sort((a, b) => a.localeCompare(b)), wiki = [];
-            for (const rule of rules) {
-                wiki.push([rule, values.filter(errors => errors.some(({ rule: r }) => r === rule)).length]);
-                // rule
-                const articles = Object.entries(data).filter(([, errors]) => errors.some(({ rule: r }) => r === rule))
-                    .sort(([a], [b]) => a.localeCompare(b))
-                    .map(([page, errors]) => {
-                    const { startLine, startCol, message, excerpt } = errors.find(({ rule: r }) => r === rule);
-                    return [page, startLine + 1, startCol + 1, message, excerpt.slice(0, MAX * 0.8)];
-                }), batches = Math.ceil(articles.length / 200);
-                for (let i = 0; i < batches; i++) {
-                    writeJS({
-                        articles: articles.slice(i * 200, (i + 1) * 200),
-                        batches,
-                    }, path_1.default.join(site, `${rule}-${i}`));
-                }
+            if (!(rule in wiki)) {
+                wiki[rule] = 0;
             }
-            writeJS(wiki, path_1.default.join(site, 'index'));
-            // article
-            const articlesDir = path_1.default.join(siteDir, 'pages');
-            mkdir(articlesDir);
-            for (const [page, errors] of Object.entries(data)) {
-                const hash = (0, crypto_1.createHash)('sha256').update(page).digest('hex')
-                    .slice(0, 8), info = errors.map(({ startLine, startCol, rule, message, excerpt, }) => [
-                    rule,
-                    startLine + 1,
-                    startCol + 1,
-                    message,
-                    excerpt.slice(0, MAX),
-                ]);
-                writeJS(info, path_1.default.join(site, 'pages', hash));
+            // rule
+            if (!rules.has(rule)) {
+                rules.add(rule);
+                wiki[rule]++;
+                let ruleRecord;
+                if (ruleRecords.has(rule)) {
+                    ruleRecord = ruleRecords.get(rule);
+                }
+                else {
+                    ruleRecord = ['', []];
+                    ruleRecords.set(rule, ruleRecord);
+                }
+                ruleRecord[1].push(page);
+                ruleRecord[0] += `${JSON.stringify([page, line, col, message, excerpt.slice(0, util_1.MAX * 0.8)], null, '\t')},`;
             }
         }
+        writeJS(info, path_1.default.join(site, 'pages', hash));
     }
-    catch {
-        console.error(chalk_1.default.red(`Failed to read ${file}`));
+}
+const timestamp = latest.toISOString().slice(0, 10);
+// rule
+for (const [rule, [str, pages]] of ruleRecords) {
+    const batches = Math.ceil(pages.length / 200);
+    pages.sort(compare);
+    for (let i = 0; i < batches; i++) {
+        const stream = initJS(path_1.default.join(siteDir, `${rule}-${i}`));
+        for (let j = i * 200; j < (i + 1) * 200; j++) {
+            const page = pages[j];
+            if (!page) {
+                break;
+            }
+            const index = str.indexOf(`[\n\t${JSON.stringify(page)}`);
+            stream.write(str.slice(index, str.indexOf('\n]', index) + 3));
+        }
+        stream.write(`],batches:${batches},timestamp:"${timestamp}"}`);
+        stream.end();
     }
 }
-writeJS(summary, 'index');
+writeJS([...summary].sort(compare), 'index');
+writeJS([...Object.entries(wiki).sort(([a], [b]) => a.localeCompare(b)), timestamp], path_1.default.join(lang, 'index'));

package/reports/article.html

@@ -4,6 +4,7 @@
 		<title>Diagnostic Report for Wikipedia</title>
 		<meta charset="utf-8">
 		<meta name="viewport" content="initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0, width=device-width">
+		<link rel="icon" href="data:image/png;base64,iVBORw0KGgo=">
 		<link rel="stylesheet" href="reports.css">
 		<style>td:nth-child(-n+3){text-align:center}</style>
 		<script defer="" src="dist/article.js"></script>
@@ -23,7 +24,7 @@
 					Diagnostic Report for 
 					<a id="article">Article</a>
 				</h2>
-				<table style="table-layout:fixed;width:100%">
+				<table style="table-layout:fixed;width:100%;min-width:900px">
 					<colgroup>
 						<col span="1">
 						<col span="1" style="width:calc(1.6em + 7ch)">

package/reports/dist/article.js

@@ -1,30 +1,2 @@
-"use strict";
-(async () => {
-    const search = new URLSearchParams(location.search), page = search.get('page'), lang = search.get('lang'), buffer = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(page)), hash = [...new Uint8Array(buffer)].slice(0, 4)
-        .map(b => b.toString(16).padStart(2, '0'))
-        .join(''), title = document.querySelector('title'), h2 = document.getElementById('article'), wiki = document.getElementById('wiki'), tbody = document.querySelector('tbody'), script = document.createElement('script');
-    title.textContent = title.textContent.replace('Wikipedia', page);
-    h2.textContent = page;
-    h2.href = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(page)}?redirect=no`;
-    wiki.textContent = `${lang}wiki`;
-    wiki.href += `?lang=${lang}`;
-    script.src = `./data/${lang}/pages/${hash}.js`;
-    script.addEventListener('load', () => {
-        for (const entry of data) {
-            const [rule, startLine, startCol, message, excerpt] = entry, tr = document.createElement('tr'), description = document.createElement('td'), line = document.createElement('td'), column = document.createElement('td'), detail = document.createElement('td'), notice = document.createElement('td'), descriptionLink = document.createElement('a');
-            descriptionLink.textContent = rule;
-            descriptionLink.href = `./rule.html?lang=${lang}&rule=${rule}`;
-            description.className = 'excerpt';
-            description.append(descriptionLink);
-            line.textContent = String(startLine);
-            column.textContent = String(startCol);
-            detail.textContent = message;
-            detail.className = 'excerpt';
-            notice.textContent = excerpt;
-            notice.className = 'excerpt mono';
-            tr.append(description, line, column, detail, notice);
-            tbody.append(tr);
-        }
-    });
-    document.head.append(script);
-})();
+"use strict";(()=>{var c=(n,e)=>{n.href=e,e.startsWith("https://")&&(n.target="_blank",n.rel="noopener")},l=(n,e)=>{let t=document.createElement("script");t.src=n,t.addEventListener("load",e),document.head.append(t)},d=n=>n==="mediawiki"?"www.mediawiki.org":`${n}.wikipedia.org`,g=(n,e)=>{let t=document.querySelector(n);return t.textContent=t.textContent.replace("Wikipedia",e),t},a=(n,e,t)=>{let r=document.getElementById(n);typeof e=="function"?r.href=e(r.href):e?c(r,e):r.removeAttribute("href"),t!==void 0&&(r.textContent=t)},m=(n,e,t,r)=>{let i=document.createElement(n),o=document.createElement("a");return o.textContent=e,c(o,t),i.append(o),r&&(i.className=r),i},s=(n,e)=>{let t=document.createElement("td");return t.textContent=n,e&&(t.className=e),t},p=(...n)=>{let e=document.createElement("tr");e.append(...n),document.querySelector("tbody").append(e)},u=(n,e,t,r)=>[s(String(n)),s(String(e)),s(t,"excerpt"),s(r,"excerpt mono")];(async()=>{let n=new URLSearchParams(location.search),e=n.get("page"),t=n.get("lang"),r=await crypto.subtle.digest("SHA-256",new TextEncoder().encode(e)),i=[...new Uint8Array(r)].slice(0,4).map(o=>o.toString(16).padStart(2,"0")).join("");g("title",e),a("article",`https://${d(t)}/wiki/${encodeURIComponent(e)}?redirect=no`,e),a("wiki",o=>`${o}?lang=${t}`,`${t}wiki`),l(`./data/${t}/pages/${i}.js`,()=>{for(let[o,E,f,x,L]of data)p(m("td",o,`./rule.html?lang=${t}&rule=${o}`,"excerpt"),...u(E,f,x,L))})})();})();
+//# sourceMappingURL=article.js.map

package/reports/dist/article.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/common.ts", "../src/article.ts"],
+  "sourcesContent": ["const armorLink = (a: HTMLAnchorElement, href: string): void => {\n\ta.href = href;\n\tif (href.startsWith('https://')) {\n\t\ta.target = '_blank';\n\t\ta.rel = 'noopener';\n\t}\n};\n\nexport const load = (src: string, callback: () => void): void => {\n\tconst script = document.createElement('script');\n\tscript.src = src;\n\tscript.addEventListener('load', callback);\n\tdocument.head.append(script);\n};\n\nexport const getHost = (lang: string): string => lang === 'mediawiki' ? 'www.mediawiki.org' : `${lang}.wikipedia.org`;\n\nexport const update = (tag: string, replace: string): Element => {\n\tconst ele = document.querySelector(tag)!;\n\tele.textContent = ele.textContent!.replace('Wikipedia', replace);\n\treturn ele;\n};\n\nexport const updateLink = (id: string, href: string | ((s: string) => string) | false, text?: string): void => {\n\tconst a = document.getElementById(id) as HTMLAnchorElement;\n\tif (typeof href === 'function') {\n\t\ta.href = href(a.href);\n\t} else if (href) {\n\t\tarmorLink(a, href);\n\t} else {\n\t\ta.removeAttribute('href');\n\t}\n\tif (text !== undefined) {\n\t\ta.textContent = text;\n\t}\n};\n\nexport const addLink = (tag: string, text: string, href: string, className?: string): HTMLElement => {\n\tconst container = document.createElement(tag),\n\t\ta = document.createElement('a');\n\ta.textContent = text;\n\tarmorLink(a, href);\n\tcontainer.append(a);\n\tif (className) {\n\t\tcontainer.className = className;\n\t}\n\treturn container;\n};\n\nexport const createTd = (text: string, className?: string): HTMLTableCellElement => {\n\tconst td = document.createElement('td');\n\ttd.textContent = text;\n\tif (className) {\n\t\ttd.className = className;\n\t}\n\treturn td;\n};\n\nexport const insertRow = (...tds: HTMLElement[]): void => {\n\tconst tr = document.createElement('tr');\n\ttr.append(...tds);\n\tdocument.querySelector('tbody')!.append(tr);\n};\n\nexport const getErrorInfo = (\n\tstartLine: number,\n\tstartCol: number,\n\tmessage: string,\n\texcerpt: string,\n): [HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement] => [\n\tcreateTd(String(startLine)),\n\tcreateTd(String(startCol)),\n\tcreateTd(message, 'excerpt'),\n\tcreateTd(excerpt, 'excerpt mono'),\n];\n", "import {load, getHost, update, updateLink, addLink, insertRow, getErrorInfo} from './common';\n\ndeclare const data: [string, number, number, string, string][];\n\n(async () => {\n\tconst search = new URLSearchParams(location.search),\n\t\tpage = search.get('page')!,\n\t\tlang = search.get('lang')!,\n\t\tbuffer = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(page)),\n\t\thash = [...new Uint8Array(buffer)].slice(0, 4)\n\t\t\t.map(b => b.toString(16).padStart(2, '0'))\n\t\t\t.join('');\n\tupdate('title', page);\n\tupdateLink('article', `https://${getHost(lang)}/wiki/${encodeURIComponent(page)}?redirect=no`, page);\n\tupdateLink('wiki', s => `${s}?lang=${lang}`, `${lang}wiki`);\n\tload(`./data/${lang}/pages/${hash}.js`, () => {\n\t\tfor (const [rule, startLine, startCol, message, excerpt] of data) {\n\t\t\tinsertRow(\n\t\t\t\taddLink('td', rule, `./rule.html?lang=${lang}&rule=${rule}`, 'excerpt'),\n\t\t\t\t...getErrorInfo(startLine, startCol, message, excerpt),\n\t\t\t);\n\t\t}\n\t});\n})();\n"],
+  "mappings": "mBAAA,IAAMA,EAAY,CAACC,EAAsBC,IAAuB,CAC/DD,EAAE,KAAOC,EACLA,EAAK,WAAW,UAAU,IAC7BD,EAAE,OAAS,SACXA,EAAE,IAAM,WAEV,EAEaE,EAAO,CAACC,EAAaC,IAA+B,CAChE,IAAMC,EAAS,SAAS,cAAc,QAAQ,EAC9CA,EAAO,IAAMF,EACbE,EAAO,iBAAiB,OAAQD,CAAQ,EACxC,SAAS,KAAK,OAAOC,CAAM,CAC5B,EAEaC,EAAWC,GAAyBA,IAAS,YAAc,oBAAsB,GAAGA,CAAI,iBAExFC,EAAS,CAACC,EAAaC,IAA6B,CAChE,IAAMC,EAAM,SAAS,cAAcF,CAAG,EACtC,OAAAE,EAAI,YAAcA,EAAI,YAAa,QAAQ,YAAaD,CAAO,EACxDC,CACR,EAEaC,EAAa,CAACC,EAAYZ,EAAgDa,IAAwB,CAC9G,IAAMd,EAAI,SAAS,eAAea,CAAE,EAChC,OAAOZ,GAAS,WACnBD,EAAE,KAAOC,EAAKD,EAAE,IAAI,EACVC,EACVF,EAAUC,EAAGC,CAAI,EAEjBD,EAAE,gBAAgB,MAAM,EAErBc,IAAS,SACZd,EAAE,YAAcc,EAElB,EAEaC,EAAU,CAACN,EAAaK,EAAcb,EAAce,IAAoC,CACpG,IAAMC,EAAY,SAAS,cAAcR,CAAG,EAC3CT,EAAI,SAAS,cAAc,GAAG,EAC/B,OAAAA,EAAE,YAAcc,EAChBf,EAAUC,EAAGC,CAAI,EACjBgB,EAAU,OAAOjB,CAAC,EACdgB,IACHC,EAAU,UAAYD,GAEhBC,CACR,EAEaC,EAAW,CAACJ,EAAcE,IAA6C,CACnF,IAAMG,EAAK,SAAS,cAAc,IAAI,EACtC,OAAAA,EAAG,YAAcL,EACbE,IACHG,EAAG,UAAYH,GAETG,CACR,EAEaC,EAAY,IAAIC,IAA6B,CACzD,IAAMC,EAAK,SAAS,cAAc,IAAI,EACtCA,EAAG,OAAO,GAAGD,CAAG,EAChB,SAAS,cAAc,OAAO,EAAG,OAAOC,CAAE,CAC3C,EAEaC,EAAe,CAC3BC,EACAC,EACAC,EACAC,IAC8F,CAC9FT,EAAS,OAAOM,CAAS,CAAC,EAC1BN,EAAS,OAAOO,CAAQ,CAAC,EACzBP,EAASQ,EAAS,SAAS,EAC3BR,EAASS,EAAS,cAAc,CACjC,GCtEC,SAAY,CACZ,IAAMC,EAAS,IAAI,gBAAgB,SAAS,MAAM,EACjDC,EAAOD,EAAO,IAAI,MAAM,EACxBE,EAAOF,EAAO,IAAI,MAAM,EACxBG,EAAS,MAAM,OAAO,OAAO,OAAO,UAAW,IAAI,YAAY,EAAE,OAAOF,CAAI,CAAC,EAC7EG,EAAO,CAAC,GAAG,IAAI,WAAWD,CAAM,CAAC,EAAE,MAAM,EAAG,CAAC,EAC3C,IAAIE,GAAKA,EAAE,SAAS,EAAE,EAAE,SAAS,EAAG,GAAG,CAAC,EACxC,KAAK,EAAE,EACVC,EAAO,QAASL,CAAI,EACpBM,EAAW,UAAW,WAAWC,EAAQN,CAAI,CAAC,SAAS,mBAAmBD,CAAI,CAAC,eAAgBA,CAAI,EACnGM,EAAW,OAAQE,GAAK,GAAGA,CAAC,SAASP,CAAI,GAAI,GAAGA,CAAI,MAAM,EAC1DQ,EAAK,UAAUR,CAAI,UAAUE,CAAI,MAAO,IAAM,CAC7C,OAAW,CAACO,EAAMC,EAAWC,EAAUC,EAASC,CAAO,IAAK,KAC3DC,EACCC,EAAQ,KAAMN,EAAM,oBAAoBT,CAAI,SAASS,CAAI,GAAI,SAAS,EACtE,GAAGO,EAAaN,EAAWC,EAAUC,EAASC,CAAO,CACtD,CAEF,CAAC,CACF,GAAG",
+  "names": ["armorLink", "a", "href", "load", "src", "callback", "script", "getHost", "lang", "update", "tag", "replace", "ele", "updateLink", "id", "text", "addLink", "className", "container", "createTd", "td", "insertRow", "tds", "tr", "getErrorInfo", "startLine", "startCol", "message", "excerpt", "search", "page", "lang", "buffer", "hash", "b", "update", "updateLink", "getHost", "s", "load", "rule", "startLine", "startCol", "message", "excerpt", "insertRow", "addLink", "getErrorInfo"]
+}

package/reports/dist/index.js

@@ -1,15 +1,2 @@
-"use strict";
-(() => {
-    const container = document.getElementById('container'), script = document.createElement('script');
-    script.src = './data/index.js';
-    script.addEventListener('load', () => {
-        container.append(...window.data.map(lang => {
-            const div = document.createElement('div'), a = document.createElement('a');
-            a.href = `./wiki.html?lang=${lang}`;
-            a.innerText = `${lang}.wikipedia.org`;
-            div.append(a);
-            return div;
-        }));
-    });
-    document.head.append(script);
-})();
+"use strict";(()=>{var a=(e,t)=>{e.href=t,t.startsWith("https://")&&(e.target="_blank",e.rel="noopener")},s=(e,t)=>{let n=document.createElement("script");n.src=e,n.addEventListener("load",t),document.head.append(n)};var d=(e,t,n,i)=>{let r=document.createElement(e),o=document.createElement("a");return o.textContent=t,a(o,n),r.append(o),i&&(r.className=i),r};s("./data/index.js",()=>{document.getElementById("container").append(...data.map(e=>d("div",`${e}wiki`,`./wiki.html?lang=${e}`)))});})();
+//# sourceMappingURL=index.js.map

package/reports/dist/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/common.ts", "../src/index.ts"],
+  "sourcesContent": ["const armorLink = (a: HTMLAnchorElement, href: string): void => {\n\ta.href = href;\n\tif (href.startsWith('https://')) {\n\t\ta.target = '_blank';\n\t\ta.rel = 'noopener';\n\t}\n};\n\nexport const load = (src: string, callback: () => void): void => {\n\tconst script = document.createElement('script');\n\tscript.src = src;\n\tscript.addEventListener('load', callback);\n\tdocument.head.append(script);\n};\n\nexport const getHost = (lang: string): string => lang === 'mediawiki' ? 'www.mediawiki.org' : `${lang}.wikipedia.org`;\n\nexport const update = (tag: string, replace: string): Element => {\n\tconst ele = document.querySelector(tag)!;\n\tele.textContent = ele.textContent!.replace('Wikipedia', replace);\n\treturn ele;\n};\n\nexport const updateLink = (id: string, href: string | ((s: string) => string) | false, text?: string): void => {\n\tconst a = document.getElementById(id) as HTMLAnchorElement;\n\tif (typeof href === 'function') {\n\t\ta.href = href(a.href);\n\t} else if (href) {\n\t\tarmorLink(a, href);\n\t} else {\n\t\ta.removeAttribute('href');\n\t}\n\tif (text !== undefined) {\n\t\ta.textContent = text;\n\t}\n};\n\nexport const addLink = (tag: string, text: string, href: string, className?: string): HTMLElement => {\n\tconst container = document.createElement(tag),\n\t\ta = document.createElement('a');\n\ta.textContent = text;\n\tarmorLink(a, href);\n\tcontainer.append(a);\n\tif (className) {\n\t\tcontainer.className = className;\n\t}\n\treturn container;\n};\n\nexport const createTd = (text: string, className?: string): HTMLTableCellElement => {\n\tconst td = document.createElement('td');\n\ttd.textContent = text;\n\tif (className) {\n\t\ttd.className = className;\n\t}\n\treturn td;\n};\n\nexport const insertRow = (...tds: HTMLElement[]): void => {\n\tconst tr = document.createElement('tr');\n\ttr.append(...tds);\n\tdocument.querySelector('tbody')!.append(tr);\n};\n\nexport const getErrorInfo = (\n\tstartLine: number,\n\tstartCol: number,\n\tmessage: string,\n\texcerpt: string,\n): [HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement] => [\n\tcreateTd(String(startLine)),\n\tcreateTd(String(startCol)),\n\tcreateTd(message, 'excerpt'),\n\tcreateTd(excerpt, 'excerpt mono'),\n];\n", "import {load, addLink} from './common';\n\ndeclare const data: string[];\n\nload('./data/index.js', () => {\n\tdocument.getElementById('container')!.append(...data.map(\n\t\tlang => addLink('div', `${lang}wiki`, `./wiki.html?lang=${lang}`),\n\t));\n});\n"],
+  "mappings": "mBAAA,IAAMA,EAAY,CAACC,EAAsBC,IAAuB,CAC/DD,EAAE,KAAOC,EACLA,EAAK,WAAW,UAAU,IAC7BD,EAAE,OAAS,SACXA,EAAE,IAAM,WAEV,EAEaE,EAAO,CAACC,EAAaC,IAA+B,CAChE,IAAMC,EAAS,SAAS,cAAc,QAAQ,EAC9CA,EAAO,IAAMF,EACbE,EAAO,iBAAiB,OAAQD,CAAQ,EACxC,SAAS,KAAK,OAAOC,CAAM,CAC5B,EAwBO,IAAMC,EAAU,CAACC,EAAaC,EAAcC,EAAcC,IAAoC,CACpG,IAAMC,EAAY,SAAS,cAAcJ,CAAG,EAC3CK,EAAI,SAAS,cAAc,GAAG,EAC/B,OAAAA,EAAE,YAAcJ,EAChBK,EAAUD,EAAGH,CAAI,EACjBE,EAAU,OAAOC,CAAC,EACdF,IACHC,EAAU,UAAYD,GAEhBC,CACR,EC3CAG,EAAK,kBAAmB,IAAM,CAC7B,SAAS,eAAe,WAAW,EAAG,OAAO,GAAG,KAAK,IACpDC,GAAQC,EAAQ,MAAO,GAAGD,CAAI,OAAQ,oBAAoBA,CAAI,EAAE,CACjE,CAAC,CACF,CAAC",
+  "names": ["armorLink", "a", "href", "load", "src", "callback", "script", "addLink", "tag", "text", "href", "className", "container", "a", "armorLink", "load", "lang", "addLink"]
+}

package/reports/dist/rule.js

@@ -1,48 +1,2 @@
-"use strict";
-(() => {
-    const search = new URLSearchParams(location.search), lang = search.get('lang'), rule = search.get('rule'), batch = Math.floor(Number(search.get('start') || 0) / 200), endStr = String((batch + 1) * 200), prev = document.getElementById('prev'), next = document.getElementById('next'), start = document.getElementById('start'), end = document.getElementById('end'), title = document.querySelector('title'), h2 = document.querySelector('h2'), wiki = document.getElementById('wiki'), tbody = document.querySelector('tbody'), script = document.createElement('script');
-    h2.textContent = h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org: ${rule}`);
-    title.textContent = title.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
-    wiki.textContent = `${lang}wiki`;
-    wiki.href += `?lang=${lang}`;
-    if (batch === 0) {
-        prev.removeAttribute('href');
-    }
-    else {
-        start.textContent = String(batch * 200 + 1);
-        end.textContent = endStr;
-        search.set('start', String((batch - 1) * 200));
-        prev.href = `${location.pathname}?${search}`;
-    }
-    search.set('start', endStr);
-    next.href = `${location.pathname}?${search}`;
-    script.src = `./data/${lang}/${rule}-${batch}.js`;
-    script.addEventListener('load', () => {
-        if (data.batches === batch + 1) {
-            next.removeAttribute('href');
-            end.textContent = String(batch * 200 + data.articles.length);
-        }
-        for (const [page, startLine, startCol, message, excerpt] of data.articles) {
-            const tr = document.createElement('tr'), article = document.createElement('td'), edit = document.createElement('td'), line = document.createElement('td'), column = document.createElement('td'), detail = document.createElement('td'), notice = document.createElement('td'), more = document.createElement('td'), articleLink = document.createElement('a'), editLink = document.createElement('a'), moreLink = document.createElement('a');
-            articleLink.textContent = page;
-            articleLink.href = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(page)}?redirect=no`;
-            article.className = 'excerpt';
-            article.append(articleLink);
-            editLink.textContent = 'edit';
-            editLink.href = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(page)}?action=edit`;
-            edit.append(editLink);
-            line.textContent = String(startLine);
-            column.textContent = String(startCol);
-            detail.textContent = message;
-            detail.className = 'excerpt';
-            notice.textContent = excerpt;
-            notice.className = 'excerpt mono';
-            moreLink.textContent = 'more';
-            moreLink.href = `./article.html?lang=${lang}&page=${encodeURIComponent(page)}`;
-            more.append(moreLink);
-            tr.append(article, edit, line, column, detail, notice, more);
-            tbody.append(tr);
-        }
-    });
-    document.head.append(script);
-})();
+"use strict";(()=>{var u=(t,e)=>{t.href=e,e.startsWith("https://")&&(t.target="_blank",t.rel="noopener")},E=(t,e)=>{let n=document.createElement("script");n.src=t,n.addEventListener("load",e),document.head.append(n)},h=t=>t==="mediawiki"?"www.mediawiki.org":`${t}.wikipedia.org`,p=(t,e)=>{let n=document.querySelector(t);return n.textContent=n.textContent.replace("Wikipedia",e),n},d=(t,e,n)=>{let r=document.getElementById(t);typeof e=="function"?r.href=e(r.href):e?u(r,e):r.removeAttribute("href"),n!==void 0&&(r.textContent=n)},m=(t,e,n,r)=>{let s=document.createElement(t),c=document.createElement("a");return c.textContent=e,u(c,n),s.append(c),r&&(s.className=r),s},l=(t,e)=>{let n=document.createElement("td");return n.textContent=t,e&&(n.className=e),n},$=(...t)=>{let e=document.createElement("tr");e.append(...t),document.querySelector("tbody").append(e)},x=(t,e,n,r)=>[l(String(t)),l(String(e)),l(n,"excerpt"),l(r,"excerpt mono")];var o=new URLSearchParams(location.search),i=o.get("lang"),f=o.get("rule"),a=Math.floor(Number(o.get("start")||0)/200),b=h(i),L=p("h2",`${i}wiki: ${f}`);p("title",`${i}wiki`);d("wiki",t=>`${t}?lang=${i}`,`${i}wiki`);o.set("start",String((a-1)*200));d("prev",a!==0&&`${location.pathname}?${o}`);document.getElementById("start").textContent=String(a*200+1);E(`./data/${i}/${f}-${a}.js`,()=>{let t=String(a*200+data.articles.length);L.textContent+=` (${data.timestamp})`,document.getElementById("end").textContent=t,o.set("start",t),d("next",data.batches!==a+1&&`${location.pathname}?${o}`),document.querySelector("table").after(document.getElementById("nav").cloneNode(!0));for(let[e,n,r,s,c]of data.articles){let g=encodeURIComponent(e);$(m("td",e,`https://${b}/wiki/${g}?redirect=no`,"excerpt"),m("td","edit",`https://${b}/wiki/${g}?action=edit`),...x(n,r,s,c),m("td","more",`./article.html?lang=${i}&page=${g}`))}});})();
+//# sourceMappingURL=rule.js.map

package/reports/dist/rule.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/common.ts", "../src/rule.ts"],
+  "sourcesContent": ["const armorLink = (a: HTMLAnchorElement, href: string): void => {\n\ta.href = href;\n\tif (href.startsWith('https://')) {\n\t\ta.target = '_blank';\n\t\ta.rel = 'noopener';\n\t}\n};\n\nexport const load = (src: string, callback: () => void): void => {\n\tconst script = document.createElement('script');\n\tscript.src = src;\n\tscript.addEventListener('load', callback);\n\tdocument.head.append(script);\n};\n\nexport const getHost = (lang: string): string => lang === 'mediawiki' ? 'www.mediawiki.org' : `${lang}.wikipedia.org`;\n\nexport const update = (tag: string, replace: string): Element => {\n\tconst ele = document.querySelector(tag)!;\n\tele.textContent = ele.textContent!.replace('Wikipedia', replace);\n\treturn ele;\n};\n\nexport const updateLink = (id: string, href: string | ((s: string) => string) | false, text?: string): void => {\n\tconst a = document.getElementById(id) as HTMLAnchorElement;\n\tif (typeof href === 'function') {\n\t\ta.href = href(a.href);\n\t} else if (href) {\n\t\tarmorLink(a, href);\n\t} else {\n\t\ta.removeAttribute('href');\n\t}\n\tif (text !== undefined) {\n\t\ta.textContent = text;\n\t}\n};\n\nexport const addLink = (tag: string, text: string, href: string, className?: string): HTMLElement => {\n\tconst container = document.createElement(tag),\n\t\ta = document.createElement('a');\n\ta.textContent = text;\n\tarmorLink(a, href);\n\tcontainer.append(a);\n\tif (className) {\n\t\tcontainer.className = className;\n\t}\n\treturn container;\n};\n\nexport const createTd = (text: string, className?: string): HTMLTableCellElement => {\n\tconst td = document.createElement('td');\n\ttd.textContent = text;\n\tif (className) {\n\t\ttd.className = className;\n\t}\n\treturn td;\n};\n\nexport const insertRow = (...tds: HTMLElement[]): void => {\n\tconst tr = document.createElement('tr');\n\ttr.append(...tds);\n\tdocument.querySelector('tbody')!.append(tr);\n};\n\nexport const getErrorInfo = (\n\tstartLine: number,\n\tstartCol: number,\n\tmessage: string,\n\texcerpt: string,\n): [HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement] => [\n\tcreateTd(String(startLine)),\n\tcreateTd(String(startCol)),\n\tcreateTd(message, 'excerpt'),\n\tcreateTd(excerpt, 'excerpt mono'),\n];\n", "import {load, getHost, update, addLink, insertRow, getErrorInfo, updateLink} from './common';\n\ndeclare const data: {\n\tarticles: [string, number, number, string, string][];\n\tbatches: number;\n\ttimestamp: string;\n};\n\nconst search = new URLSearchParams(location.search),\n\tlang = search.get('lang'),\n\trule = search.get('rule'),\n\tbatch = Math.floor(Number(search.get('start') || 0) / 200),\n\thost = getHost(lang!),\n\th2 = update('h2', `${lang}wiki: ${rule}`);\nupdate('title', `${lang}wiki`);\nupdateLink('wiki', s => `${s}?lang=${lang}`, `${lang}wiki`);\nsearch.set('start', String((batch - 1) * 200));\nupdateLink('prev', batch !== 0 && `${location.pathname}?${search}`);\ndocument.getElementById('start')!.textContent = String(batch * 200 + 1);\nload(`./data/${lang}/${rule}-${batch}.js`, () => {\n\tconst endStr = String(batch * 200 + data.articles.length);\n\th2.textContent += ` (${data.timestamp})`;\n\tdocument.getElementById('end')!.textContent = endStr;\n\tsearch.set('start', endStr);\n\tupdateLink('next', data.batches !== batch + 1 && `${location.pathname}?${search}`);\n\tdocument.querySelector('table')!\n\t\t.after(document.getElementById('nav')!.cloneNode(true));\n\tfor (const [page, startLine, startCol, message, excerpt] of data.articles) {\n\t\tconst title = encodeURIComponent(page);\n\t\tinsertRow(\n\t\t\taddLink('td', page, `https://${host}/wiki/${title}?redirect=no`, 'excerpt'),\n\t\t\taddLink('td', 'edit', `https://${host}/wiki/${title}?action=edit`),\n\t\t\t...getErrorInfo(startLine, startCol, message, excerpt),\n\t\t\taddLink('td', 'more', `./article.html?lang=${lang}&page=${title}`),\n\t\t);\n\t}\n});\n"],
+  "mappings": "mBAAA,IAAMA,EAAY,CAACC,EAAsBC,IAAuB,CAC/DD,EAAE,KAAOC,EACLA,EAAK,WAAW,UAAU,IAC7BD,EAAE,OAAS,SACXA,EAAE,IAAM,WAEV,EAEaE,EAAO,CAACC,EAAaC,IAA+B,CAChE,IAAMC,EAAS,SAAS,cAAc,QAAQ,EAC9CA,EAAO,IAAMF,EACbE,EAAO,iBAAiB,OAAQD,CAAQ,EACxC,SAAS,KAAK,OAAOC,CAAM,CAC5B,EAEaC,EAAWC,GAAyBA,IAAS,YAAc,oBAAsB,GAAGA,CAAI,iBAExFC,EAAS,CAACC,EAAaC,IAA6B,CAChE,IAAMC,EAAM,SAAS,cAAcF,CAAG,EACtC,OAAAE,EAAI,YAAcA,EAAI,YAAa,QAAQ,YAAaD,CAAO,EACxDC,CACR,EAEaC,EAAa,CAACC,EAAYZ,EAAgDa,IAAwB,CAC9G,IAAMd,EAAI,SAAS,eAAea,CAAE,EAChC,OAAOZ,GAAS,WACnBD,EAAE,KAAOC,EAAKD,EAAE,IAAI,EACVC,EACVF,EAAUC,EAAGC,CAAI,EAEjBD,EAAE,gBAAgB,MAAM,EAErBc,IAAS,SACZd,EAAE,YAAcc,EAElB,EAEaC,EAAU,CAACN,EAAaK,EAAcb,EAAce,IAAoC,CACpG,IAAMC,EAAY,SAAS,cAAcR,CAAG,EAC3CT,EAAI,SAAS,cAAc,GAAG,EAC/B,OAAAA,EAAE,YAAcc,EAChBf,EAAUC,EAAGC,CAAI,EACjBgB,EAAU,OAAOjB,CAAC,EACdgB,IACHC,EAAU,UAAYD,GAEhBC,CACR,EAEaC,EAAW,CAACJ,EAAcE,IAA6C,CACnF,IAAMG,EAAK,SAAS,cAAc,IAAI,EACtC,OAAAA,EAAG,YAAcL,EACbE,IACHG,EAAG,UAAYH,GAETG,CACR,EAEaC,EAAY,IAAIC,IAA6B,CACzD,IAAMC,EAAK,SAAS,cAAc,IAAI,EACtCA,EAAG,OAAO,GAAGD,CAAG,EAChB,SAAS,cAAc,OAAO,EAAG,OAAOC,CAAE,CAC3C,EAEaC,EAAe,CAC3BC,EACAC,EACAC,EACAC,IAC8F,CAC9FT,EAAS,OAAOM,CAAS,CAAC,EAC1BN,EAAS,OAAOO,CAAQ,CAAC,EACzBP,EAASQ,EAAS,SAAS,EAC3BR,EAASS,EAAS,cAAc,CACjC,EClEA,IAAMC,EAAS,IAAI,gBAAgB,SAAS,MAAM,EACjDC,EAAOD,EAAO,IAAI,MAAM,EACxBE,EAAOF,EAAO,IAAI,MAAM,EACxBG,EAAQ,KAAK,MAAM,OAAOH,EAAO,IAAI,OAAO,GAAK,CAAC,EAAI,GAAG,EACzDI,EAAOC,EAAQJ,CAAK,EACpBK,EAAKC,EAAO,KAAM,GAAGN,CAAI,SAASC,CAAI,EAAE,EACzCK,EAAO,QAAS,GAAGN,CAAI,MAAM,EAC7BO,EAAW,OAAQC,GAAK,GAAGA,CAAC,SAASR,CAAI,GAAI,GAAGA,CAAI,MAAM,EAC1DD,EAAO,IAAI,QAAS,QAAQG,EAAQ,GAAK,GAAG,CAAC,EAC7CK,EAAW,OAAQL,IAAU,GAAK,GAAG,SAAS,QAAQ,IAAIH,CAAM,EAAE,EAClE,SAAS,eAAe,OAAO,EAAG,YAAc,OAAOG,EAAQ,IAAM,CAAC,EACtEO,EAAK,UAAUT,CAAI,IAAIC,CAAI,IAAIC,CAAK,MAAO,IAAM,CAChD,IAAMQ,EAAS,OAAOR,EAAQ,IAAM,KAAK,SAAS,MAAM,EACxDG,EAAG,aAAe,KAAK,KAAK,SAAS,IACrC,SAAS,eAAe,KAAK,EAAG,YAAcK,EAC9CX,EAAO,IAAI,QAASW,CAAM,EAC1BH,EAAW,OAAQ,KAAK,UAAYL,EAAQ,GAAK,GAAG,SAAS,QAAQ,IAAIH,CAAM,EAAE,EACjF,SAAS,cAAc,OAAO,EAC5B,MAAM,SAAS,eAAe,KAAK,EAAG,UAAU,EAAI,CAAC,EACvD,OAAW,CAACY,EAAMC,EAAWC,EAAUC,EAASC,CAAO,IAAK,KAAK,SAAU,CAC1E,IAAMC,EAAQ,mBAAmBL,CAAI,EACrCM,EACCC,EAAQ,KAAMP,EAAM,WAAWR,CAAI,SAASa,CAAK,eAAgB,SAAS,EAC1EE,EAAQ,KAAM,OAAQ,WAAWf,CAAI,SAASa,CAAK,cAAc,EACjE,GAAGG,EAAaP,EAAWC,EAAUC,EAASC,CAAO,EACrDG,EAAQ,KAAM,OAAQ,uBAAuBlB,CAAI,SAASgB,CAAK,EAAE,CAClE,CACD,CACD,CAAC",
+  "names": ["armorLink", "a", "href", "load", "src", "callback", "script", "getHost", "lang", "update", "tag", "replace", "ele", "updateLink", "id", "text", "addLink", "className", "container", "createTd", "td", "insertRow", "tds", "tr", "getErrorInfo", "startLine", "startCol", "message", "excerpt", "search", "lang", "rule", "batch", "host", "getHost", "h2", "update", "updateLink", "s", "load", "endStr", "page", "startLine", "startCol", "message", "excerpt", "title", "insertRow", "addLink", "getErrorInfo"]
+}

package/reports/dist/wiki.js

@@ -1,19 +1,2 @@
-"use strict";
-(() => {
-    const lang = new URLSearchParams(location.search).get('lang'), script = document.createElement('script'), title = document.querySelector('title'), h2 = document.querySelector('h2'), tbody = document.querySelector('tbody');
-    h2.textContent = h2.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
-    title.textContent = title.textContent.replace('Wikipedia', `${lang}.wikipedia.org`);
-    script.src = `./data/${lang}/index.js`;
-    script.addEventListener('load', () => {
-        for (const [rule, count] of data) {
-            const tr = document.createElement('tr'), description = document.createElement('td'), pages = document.createElement('td'), a = document.createElement('a');
-            a.textContent = rule;
-            a.href = `./rule.html?lang=${lang}&rule=${rule}`;
-            description.append(a);
-            pages.textContent = String(count);
-            tr.append(description, pages);
-            tbody.append(tr);
-        }
-    });
-    document.head.append(script);
-})();
+"use strict";(()=>{var g=(t,e)=>{t.href=e,e.startsWith("https://")&&(t.target="_blank",t.rel="noopener")},l=(t,e)=>{let n=document.createElement("script");n.src=t,n.addEventListener("load",e),document.head.append(n)};var s=(t,e)=>{let n=document.querySelector(t);return n.textContent=n.textContent.replace("Wikipedia",e),n};var c=(t,e,n,a)=>{let o=document.createElement(t),i=document.createElement("a");return i.textContent=e,g(i,n),o.append(i),a&&(o.className=a),o},d=(t,e)=>{let n=document.createElement("td");return n.textContent=t,e&&(n.className=e),n},m=(...t)=>{let e=document.createElement("tr");e.append(...t),document.querySelector("tbody").append(e)};var r=new URLSearchParams(location.search).get("lang"),p=s("h2",`${r}wiki`);s("title",`${r}wiki`);l(`./data/${r}/index.js`,()=>{p.textContent+=` (${data[data.length-1]})`;for(let[t,e]of data.slice(0,-1))m(c("td",t,`./rule.html?lang=${r}&rule=${t}`),d(String(e)))});})();
+//# sourceMappingURL=wiki.js.map

package/reports/dist/wiki.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/common.ts", "../src/wiki.ts"],
+  "sourcesContent": ["const armorLink = (a: HTMLAnchorElement, href: string): void => {\n\ta.href = href;\n\tif (href.startsWith('https://')) {\n\t\ta.target = '_blank';\n\t\ta.rel = 'noopener';\n\t}\n};\n\nexport const load = (src: string, callback: () => void): void => {\n\tconst script = document.createElement('script');\n\tscript.src = src;\n\tscript.addEventListener('load', callback);\n\tdocument.head.append(script);\n};\n\nexport const getHost = (lang: string): string => lang === 'mediawiki' ? 'www.mediawiki.org' : `${lang}.wikipedia.org`;\n\nexport const update = (tag: string, replace: string): Element => {\n\tconst ele = document.querySelector(tag)!;\n\tele.textContent = ele.textContent!.replace('Wikipedia', replace);\n\treturn ele;\n};\n\nexport const updateLink = (id: string, href: string | ((s: string) => string) | false, text?: string): void => {\n\tconst a = document.getElementById(id) as HTMLAnchorElement;\n\tif (typeof href === 'function') {\n\t\ta.href = href(a.href);\n\t} else if (href) {\n\t\tarmorLink(a, href);\n\t} else {\n\t\ta.removeAttribute('href');\n\t}\n\tif (text !== undefined) {\n\t\ta.textContent = text;\n\t}\n};\n\nexport const addLink = (tag: string, text: string, href: string, className?: string): HTMLElement => {\n\tconst container = document.createElement(tag),\n\t\ta = document.createElement('a');\n\ta.textContent = text;\n\tarmorLink(a, href);\n\tcontainer.append(a);\n\tif (className) {\n\t\tcontainer.className = className;\n\t}\n\treturn container;\n};\n\nexport const createTd = (text: string, className?: string): HTMLTableCellElement => {\n\tconst td = document.createElement('td');\n\ttd.textContent = text;\n\tif (className) {\n\t\ttd.className = className;\n\t}\n\treturn td;\n};\n\nexport const insertRow = (...tds: HTMLElement[]): void => {\n\tconst tr = document.createElement('tr');\n\ttr.append(...tds);\n\tdocument.querySelector('tbody')!.append(tr);\n};\n\nexport const getErrorInfo = (\n\tstartLine: number,\n\tstartCol: number,\n\tmessage: string,\n\texcerpt: string,\n): [HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement, HTMLTableCellElement] => [\n\tcreateTd(String(startLine)),\n\tcreateTd(String(startCol)),\n\tcreateTd(message, 'excerpt'),\n\tcreateTd(excerpt, 'excerpt mono'),\n];\n", "import {load, update, addLink, createTd, insertRow} from './common';\n\ndeclare const data: [...[string, number][], string];\n\nconst lang = new URLSearchParams(location.search).get('lang'),\n\th2 = update('h2', `${lang}wiki`);\nupdate('title', `${lang}wiki`);\nload(`./data/${lang}/index.js`, () => {\n\th2.textContent += ` (${data[data.length - 1] as string})`;\n\tfor (const [rule, count] of data.slice(0, -1) as [string, number][]) {\n\t\tinsertRow(\n\t\t\taddLink('td', rule, `./rule.html?lang=${lang}&rule=${rule}`),\n\t\t\tcreateTd(String(count)),\n\t\t);\n\t}\n});\n"],
+  "mappings": "mBAAA,IAAMA,EAAY,CAACC,EAAsBC,IAAuB,CAC/DD,EAAE,KAAOC,EACLA,EAAK,WAAW,UAAU,IAC7BD,EAAE,OAAS,SACXA,EAAE,IAAM,WAEV,EAEaE,EAAO,CAACC,EAAaC,IAA+B,CAChE,IAAMC,EAAS,SAAS,cAAc,QAAQ,EAC9CA,EAAO,IAAMF,EACbE,EAAO,iBAAiB,OAAQD,CAAQ,EACxC,SAAS,KAAK,OAAOC,CAAM,CAC5B,EAIO,IAAMC,EAAS,CAACC,EAAaC,IAA6B,CAChE,IAAMC,EAAM,SAAS,cAAcF,CAAG,EACtC,OAAAE,EAAI,YAAcA,EAAI,YAAa,QAAQ,YAAaD,CAAO,EACxDC,CACR,EAgBO,IAAMC,EAAU,CAACC,EAAaC,EAAcC,EAAcC,IAAoC,CACpG,IAAMC,EAAY,SAAS,cAAcJ,CAAG,EAC3CK,EAAI,SAAS,cAAc,GAAG,EAC/B,OAAAA,EAAE,YAAcJ,EAChBK,EAAUD,EAAGH,CAAI,EACjBE,EAAU,OAAOC,CAAC,EACdF,IACHC,EAAU,UAAYD,GAEhBC,CACR,EAEaG,EAAW,CAACN,EAAcE,IAA6C,CACnF,IAAMK,EAAK,SAAS,cAAc,IAAI,EACtC,OAAAA,EAAG,YAAcP,EACbE,IACHK,EAAG,UAAYL,GAETK,CACR,EAEaC,EAAY,IAAIC,IAA6B,CACzD,IAAMC,EAAK,SAAS,cAAc,IAAI,EACtCA,EAAG,OAAO,GAAGD,CAAG,EAChB,SAAS,cAAc,OAAO,EAAG,OAAOC,CAAE,CAC3C,EC1DA,IAAMC,EAAO,IAAI,gBAAgB,SAAS,MAAM,EAAE,IAAI,MAAM,EAC3DC,EAAKC,EAAO,KAAM,GAAGF,CAAI,MAAM,EAChCE,EAAO,QAAS,GAAGF,CAAI,MAAM,EAC7BG,EAAK,UAAUH,CAAI,YAAa,IAAM,CACrCC,EAAG,aAAe,KAAK,KAAK,KAAK,OAAS,CAAC,CAAW,IACtD,OAAW,CAACG,EAAMC,CAAK,IAAK,KAAK,MAAM,EAAG,EAAE,EAC3CC,EACCC,EAAQ,KAAMH,EAAM,oBAAoBJ,CAAI,SAASI,CAAI,EAAE,EAC3DI,EAAS,OAAOH,CAAK,CAAC,CACvB,CAEF,CAAC",
+  "names": ["armorLink", "a", "href", "load", "src", "callback", "script", "update", "tag", "replace", "ele", "addLink", "tag", "text", "href", "className", "container", "a", "armorLink", "createTd", "td", "insertRow", "tds", "tr", "lang", "h2", "update", "load", "rule", "count", "insertRow", "addLink", "createTd"]
+}

package/reports/index.html

@@ -4,6 +4,7 @@
 		<title>Diagnostic Report for Wikipedia Dumps</title>
 		<meta charset="utf-8">
 		<meta name="viewport" content="initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0, width=device-width">
+		<link rel="icon" href="data:image/png;base64,iVBORw0KGgo=">
 		<link rel="stylesheet" href="reports.css">
 		<script defer="" src="dist/index.js"></script>
 	</head>

package/reports/reports.css

@@ -29,6 +29,7 @@ article {
 	display: block;
 	width: 100%;
 	padding: 1rem 2rem 2rem;
+	overflow-x: auto;
 }
 h2 {
 	font-size: 1.3em;
@@ -65,6 +66,7 @@ th {
 }
 .mono {
 	font-family: monospace;
+	white-space: pre;
 }
 
 @media screen and (max-width: 720px) {

package/reports/rule.html

@@ -4,6 +4,7 @@
 		<title>Diagnostic Report for Wikipedia</title>
 		<meta charset="utf-8">
 		<meta name="viewport" content="initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0, width=device-width">
+		<link rel="icon" href="data:image/png;base64,iVBORw0KGgo=">
 		<link rel="stylesheet" href="reports.css">
 		<style>td:nth-child(-n+4):not(:first-child),td:last-child{text-align:center}</style>
 		<script defer="" src="dist/rule.js"></script>
@@ -27,7 +28,7 @@
 					<span id="end">200</span>
 					<a href="#" id="next" title="Next">→</a>
 				</div>
-				<table style="table-layout:fixed;width:100%">
+				<table style="table-layout:fixed;width:100%;min-width:900px">
 					<colgroup>
 						<col span="1">
 						<col span="1" style="width:calc(1.6em + 5ch)">

package/reports/wiki.html

@@ -4,6 +4,7 @@
 		<title>Diagnostic Report for Wikipedia</title>
 		<meta charset="utf-8">
 		<meta name="viewport" content="initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0, width=device-width">
+		<link rel="icon" href="data:image/png;base64,iVBORw0KGgo=">
 		<link rel="stylesheet" href="reports.css">
 		<script defer="" src="dist/wiki.js"></script>
 	</head>

package/scan.sh

@@ -1,16 +1,22 @@
 #!/usr/local/bin/bash
 if (( $# < 2 ))
 then
-	echo 'Usage: npx lint-wiki-dumps <language> <path to download>'
+	echo 'Usage: npx lint-wiki-dumps <language> <path to download> [path to HTML output]'
 	echo 'Example: npx lint-wiki-dumps zh-yue ~/Downloads/dumps'
 	exit 1
 fi
-site="${1}wiki" # example: zh-yuewiki
 target="${1//-/_}wiki" # example: zh_yuewiki
-file="${target}-latest-pages-articles.xml.bz2"
-if (( $# < 3 ))
+npx getParserConfig "${1}wiki" "https://$1.wikipedia.org/w/"
+bash download.sh "$target" "$2"
+if (( $? == 1 ))
 then
-	curl --output-dir "$2" -O "https://dumps.wikimedia.org/$target/latest/$file"
-	npx getParserConfig "$site" "https://$1.wikipedia.org/w/"
+	echo 'Switching to single-threaded mode'
+	node parser.js "$1" "$2/$target-latest-pages-articles.xml.bz2"
+else
+	node parser-parallel.js "$1" "$2" "$4"
+fi
+if (( $? == 0))
+then
+	echo 'Starting report generation'
+	node report.js "$1" "$3"
 fi
-node parser.js "$1" "$2/$file" "$3" "$4" && node report.js "$1"

package/server.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const http_1 = require("http");
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const port = parseInt(process.env['PORT'] || '8000');
+(0, http_1.createServer)(({ url }, res) => {
+    if (!url || url === '/') {
+        url = 'index.html'; // eslint-disable-line no-param-reassign
+    }
+    const file = new URL(`http://localhost/${path_1.default.join('reports', url)}`).pathname.slice(1), ext = path_1.default.extname(file);
+    let contentType;
+    switch (ext) {
+        case '.js':
+            contentType = 'text/javascript';
+            break;
+        case '.css':
+            contentType = 'text/css';
+            break;
+        default:
+            contentType = 'text/html';
+    }
+    if (fs_1.default.existsSync(file)) {
+        res.writeHead(200, {
+            'content-type': contentType,
+            'x-content-type-options': 'nosniff',
+            'cache-control': `max-age=${60 * 60 * 24}, public`,
+        });
+        res.end(fs_1.default.readFileSync(file), 'utf8');
+    }
+    else {
+        res.writeHead(301, { Location: '/' });
+        res.end();
+    }
+}).listen(port);

package/util.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalize = exports.reading = exports.replaceTilde = exports.isArticle = exports.getErrors = exports.getTimestamp = exports.getXmlStream = exports.getWriteStream = exports.init = exports.getTempPath = exports.resultDir = exports.MAX = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const os_1 = __importDefault(require("os"));
+const chalk_1 = __importDefault(require("chalk"));
+const unbzip2_stream_1 = __importDefault(require("unbzip2-stream"));
+const xml_stream_1 = __importDefault(require("xml-stream"));
+exports.MAX = 100, exports.resultDir = path_1.default.join(__dirname, 'results');
+const tempDir = path_1.default.join(__dirname, 'temp');
+const getTempPath = (file) => path_1.default.join(tempDir, file);
+exports.getTempPath = getTempPath;
+const init = () => {
+    if (!fs_1.default.existsSync(exports.resultDir)) {
+        fs_1.default.mkdirSync(exports.resultDir);
+    }
+    if (!fs_1.default.existsSync(tempDir)) {
+        fs_1.default.mkdirSync(tempDir);
+    }
+};
+exports.init = init;
+const getWriteStream = (file, callback) => {
+    const stream = fs_1.default.createWriteStream(file);
+    stream.write('{');
+    stream.on('close', callback);
+    return stream;
+};
+exports.getWriteStream = getWriteStream;
+const getXmlStream = (file) => {
+    const readable = fs_1.default.createReadStream(file).pipe((0, unbzip2_stream_1.default)()), stream = new xml_stream_1.default(readable);
+    readable.on('error', e => {
+        console.error(chalk_1.default.red(`Error unzipping ${file}`));
+        throw e;
+    });
+    stream.preserve('text', true);
+    return stream;
+};
+exports.getXmlStream = getXmlStream;
+const getTimestamp = (data) => {
+    if (!data) {
+        return undefined;
+    }
+    const i = data.indexOf('"#timestamp": "') + 15;
+    return new Date(data.slice(i, data.indexOf('"', i)));
+};
+exports.getTimestamp = getTimestamp;
+const getErrors = (data, page) => {
+    const str = JSON.stringify(page), i = data.indexOf(`${str}: [`);
+    if (i === -1) {
+        return undefined;
+    }
+    const j = i + str.length + 2;
+    return data.slice(j, data.indexOf('\n]', j) + 2);
+};
+exports.getErrors = getErrors;
+const isArticle = ($text, ns, model) => ns === '0' && model === 'wikitext' && Boolean($text);
+exports.isArticle = isArticle;
+const replaceTilde = (str) => str.replace(/^~/u, os_1.default.homedir());
+exports.replaceTilde = replaceTilde;
+const reading = (file) => {
+    console.log(chalk_1.default.green(`Reading ${file}`));
+};
+exports.reading = reading;
+const normalize = (str) => str.replaceAll('-', '_');
+exports.normalize = normalize;