@nitpicker/core 0.23.1 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -35
- package/lib/assert-u32-id.d.ts +15 -0
- package/lib/assert-u32-id.js +22 -0
- package/lib/check-native-executor-config.d.ts +21 -0
- package/lib/check-native-executor-config.js +24 -0
- package/lib/compute-alias-keys.d.ts +28 -0
- package/lib/compute-alias-keys.js +30 -0
- package/lib/compute-body-hash.d.ts +22 -0
- package/lib/compute-body-hash.js +25 -0
- package/lib/compute-url-sort-ranks.d.ts +28 -0
- package/lib/compute-url-sort-ranks.js +49 -0
- package/lib/create-isolated-cluster-run.d.ts +35 -0
- package/lib/create-isolated-cluster-run.js +52 -0
- package/lib/create-template-classification-run.d.ts +49 -0
- package/lib/create-template-classification-run.js +386 -0
- package/lib/group-alias-candidates.d.ts +40 -0
- package/lib/group-alias-candidates.js +104 -0
- package/lib/load-native-binding.d.ts +26 -0
- package/lib/load-native-binding.js +152 -0
- package/lib/match-js-license-comment-signal.d.ts +19 -0
- package/lib/match-js-license-comment-signal.js +21 -0
- package/lib/match-selector-html.d.ts +41 -0
- package/lib/match-selector-html.js +42 -0
- package/lib/match-selector-snapshots.d.ts +40 -0
- package/lib/match-selector-snapshots.js +57 -0
- package/lib/match-structural-technology-signals.d.ts +23 -0
- package/lib/match-structural-technology-signals.js +25 -0
- package/lib/max-decoded-snapshot-bytes.d.ts +22 -0
- package/lib/max-decoded-snapshot-bytes.js +35 -0
- package/lib/native-binding.d.ts +9 -0
- package/lib/native-binding.js +9 -0
- package/lib/pack-stored-snapshots.d.ts +23 -0
- package/lib/pack-stored-snapshots.js +53 -0
- package/lib/run-with-signal.d.ts +27 -0
- package/lib/run-with-signal.js +54 -0
- package/lib/search-html-document.d.ts +32 -0
- package/lib/search-html-document.js +31 -0
- package/lib/search-html-snapshots.d.ts +46 -0
- package/lib/search-html-snapshots.js +54 -0
- package/lib/template-block-sample-size.d.ts +13 -0
- package/lib/template-block-sample-size.js +14 -0
- package/lib/template-exact-page-limit.d.ts +18 -0
- package/lib/template-exact-page-limit.js +19 -0
- package/lib/template-memory-budget.d.ts +16 -0
- package/lib/template-memory-budget.js +17 -0
- package/lib/to-native-selector-plan.d.ts +16 -0
- package/lib/to-native-selector-plan.js +40 -0
- package/lib/to-u32-ids.d.ts +14 -0
- package/lib/to-u32-ids.js +23 -0
- package/lib/types.d.ts +736 -248
- package/lib/validate-search-pattern.d.ts +25 -0
- package/lib/validate-search-pattern.js +27 -0
- package/package.json +83 -16
- package/lib/core.d.ts +0 -4
- package/lib/core.js +0 -4
- package/lib/discover-analyze-plugins.d.ts +0 -14
- package/lib/discover-analyze-plugins.js +0 -32
- package/lib/get-table-cache-root.d.ts +0 -18
- package/lib/get-table-cache-root.js +0 -22
- package/lib/hooks/define-plugin.d.ts +0 -68
- package/lib/hooks/define-plugin.js +0 -69
- package/lib/import-modules.d.ts +0 -24
- package/lib/import-modules.js +0 -38
- package/lib/load-plugin-settings.d.ts +0 -44
- package/lib/load-plugin-settings.js +0 -125
- package/lib/nitpicker.d.ts +0 -150
- package/lib/nitpicker.js +0 -437
- package/lib/page-analysis-worker.d.ts +0 -58
- package/lib/page-analysis-worker.js +0 -68
- package/lib/read-plugin-labels.d.ts +0 -15
- package/lib/read-plugin-labels.js +0 -30
- package/lib/table.d.ts +0 -75
- package/lib/table.js +0 -132
- package/lib/url-event-bus.d.ts +0 -33
- package/lib/url-event-bus.js +0 -21
- package/lib/worker/runner.d.ts +0 -25
- package/lib/worker/runner.js +0 -31
- package/lib/worker/types.d.ts +0 -23
- package/lib/worker/types.js +0 -1
- package/lib/worker/worker-pool.d.ts +0 -82
- package/lib/worker/worker-pool.js +0 -278
- package/lib/worker/worker.d.ts +0 -34
- package/lib/worker/worker.js +0 -108
package/lib/nitpicker.d.ts
DELETED
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
import type { AnalyzeOptions, Config, NitpickerEvent, PluginOverrides } from './types.js';
|
|
2
|
-
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
3
|
-
import { Archive } from '@nitpicker/crawler';
|
|
4
|
-
export { definePlugin } from './hooks/define-plugin.js';
|
|
5
|
-
/**
|
|
6
|
-
* Core orchestrator for running analyze plugins against a `.nitpicker` archive.
|
|
7
|
-
*
|
|
8
|
-
* Nitpicker opens an existing archive (produced by the crawler), loads the
|
|
9
|
-
* user's plugin configuration via cosmiconfig, then runs each plugin against
|
|
10
|
-
* every page in the archive. Results are stored back into the archive as
|
|
11
|
-
* `analysis/report`, `analysis/table`, and SQL-backed analysis violation
|
|
12
|
-
* tables via the archive facade.
|
|
13
|
-
*
|
|
14
|
-
* ## Architecture decisions
|
|
15
|
-
*
|
|
16
|
-
* - **Worker pool per plugin**: Each plugin gets its own long-lived
|
|
17
|
-
* {@link ./worker/worker-pool.ts!WorkerPool}, sized by either the plugin's
|
|
18
|
-
* declared `concurrency` or {@link DEFAULT_CONCURRENCY} (CPU core count).
|
|
19
|
-
* Workers are spawned once and reused across every page, so the V8 isolate
|
|
20
|
-
* / JSDOM module / plugin module boot cost is paid only `concurrency`
|
|
21
|
-
* times per plugin instead of once per page.
|
|
22
|
-
*
|
|
23
|
-
* - **Plugin-outer, page-inner loop**: Plugins are processed sequentially.
|
|
24
|
-
* For each plugin, pages are submitted to that plugin's pool, which queues
|
|
25
|
-
* them and dispatches to idle workers. This enables per-plugin progress
|
|
26
|
-
* tracking via {@link https://www.npmjs.com/package/@d-zero/dealer | Lanes}.
|
|
27
|
-
*
|
|
28
|
-
* - **Cache layer**: Results are cached per `pluginName:url` using
|
|
29
|
-
* `@d-zero/shared/cache` so that re-running analysis after a partial failure
|
|
30
|
-
* skips already-processed pages. The cache is cleared at the start of each run.
|
|
31
|
-
* @example
|
|
32
|
-
* ```ts
|
|
33
|
-
* import { Nitpicker } from '@nitpicker/core';
|
|
34
|
-
*
|
|
35
|
-
* // Open an existing archive — `Symbol.asyncDispose` closes it on scope exit
|
|
36
|
-
* await using nitpicker = await Nitpicker.open('./example.nitpicker');
|
|
37
|
-
*
|
|
38
|
-
* // Run all configured analyze plugins
|
|
39
|
-
* await nitpicker.analyze();
|
|
40
|
-
*
|
|
41
|
-
* // Or run only specific plugins by name
|
|
42
|
-
* await nitpicker.analyze(['@nitpicker/analyze-axe']);
|
|
43
|
-
*
|
|
44
|
-
* // Write updated archive back to disk
|
|
45
|
-
* await nitpicker.write();
|
|
46
|
-
* ```
|
|
47
|
-
* @see {@link ./types.ts!NitpickerEvent} for emitted events
|
|
48
|
-
* @see {@link ./types.ts!Config} for the resolved configuration model
|
|
49
|
-
*/
|
|
50
|
-
export declare class Nitpicker extends EventEmitter<NitpickerEvent> {
|
|
51
|
-
#private;
|
|
52
|
-
/** The underlying archive instance. */
|
|
53
|
-
get archive(): Archive;
|
|
54
|
-
/**
|
|
55
|
-
* @param archive - An opened {@link Archive} instance to analyze.
|
|
56
|
-
* Use {@link Nitpicker.open} for a convenient static factory.
|
|
57
|
-
*/
|
|
58
|
-
constructor(archive: Archive);
|
|
59
|
-
/**
|
|
60
|
-
* Enables `await using nitpicker = ...`. Delegates to the wrapped
|
|
61
|
-
* archive's `Symbol.asyncDispose` (i.e. {@link Archive.close}).
|
|
62
|
-
*/
|
|
63
|
-
[Symbol.asyncDispose](): Promise<void>;
|
|
64
|
-
/**
|
|
65
|
-
* Runs all configured analyze plugins (or a filtered subset) against
|
|
66
|
-
* every page in the archive.
|
|
67
|
-
*
|
|
68
|
-
* Plugins are processed **sequentially** (one at a time). For each plugin,
|
|
69
|
-
* pages are processed in **parallel** through a dedicated long-lived
|
|
70
|
-
* {@link ./worker/worker-pool.ts!WorkerPool} sized either by the plugin's
|
|
71
|
-
* declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
|
|
72
|
-
* enables per-plugin progress tracking via Lanes.
|
|
73
|
-
*
|
|
74
|
-
* The analysis proceeds in up to two phases:
|
|
75
|
-
*
|
|
76
|
-
* 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
|
|
77
|
-
* page analysis tasks to the plugin's worker pool. Progress is
|
|
78
|
-
* displayed via Lanes if provided in options.
|
|
79
|
-
*
|
|
80
|
-
* 2. **`eachUrl` phase** - For all plugins with `eachUrl`, runs
|
|
81
|
-
* sequentially in the main thread. These are lightweight checks
|
|
82
|
-
* that don't need DOM access.
|
|
83
|
-
*
|
|
84
|
-
* On completion, data entries are stored in the archive:
|
|
85
|
-
* - `analysis/report` - Full {@link Report} with headers and data
|
|
86
|
-
* - `analysis/table` - The raw {@link Table} instance (serialized)
|
|
87
|
-
* - SQL-backed analysis violation tables - Flat {@link Violation} records
|
|
88
|
-
* @param filter - Optional list of plugin module names to run.
|
|
89
|
-
* If omitted, all configured plugins are executed.
|
|
90
|
-
* @param options - Optional settings for progress display.
|
|
91
|
-
* @example
|
|
92
|
-
* ```ts
|
|
93
|
-
* // Run all plugins
|
|
94
|
-
* await nitpicker.analyze();
|
|
95
|
-
*
|
|
96
|
-
* // Run only axe and markuplint with Lanes progress
|
|
97
|
-
* await nitpicker.analyze(
|
|
98
|
-
* ['@nitpicker/analyze-axe', '@nitpicker/analyze-markuplint'],
|
|
99
|
-
* { lanes },
|
|
100
|
-
* );
|
|
101
|
-
* ```
|
|
102
|
-
*/
|
|
103
|
-
analyze(filter?: string[], options?: AnalyzeOptions): Promise<void>;
|
|
104
|
-
/**
|
|
105
|
-
* Loads and caches the plugin configuration from the user's config file.
|
|
106
|
-
*
|
|
107
|
-
* Uses cosmiconfig to search for `.nitpickerrc`, `.nitpickerrc.json`,
|
|
108
|
-
* `nitpicker.config.js`, or a `nitpicker` key in `package.json`.
|
|
109
|
-
* The result is cached after the first call.
|
|
110
|
-
* @returns Resolved {@link Config} with the `analyze` plugin list.
|
|
111
|
-
*/
|
|
112
|
-
getConfig(): Promise<Config>;
|
|
113
|
-
/**
|
|
114
|
-
* Sets CLI-specified plugin overrides that take precedence over
|
|
115
|
-
* config-file settings.
|
|
116
|
-
*
|
|
117
|
-
* Must be called before {@link getConfig} to take effect. If config
|
|
118
|
-
* has already been loaded, calling this method clears the cache so
|
|
119
|
-
* that the next `getConfig()` call re-loads with the new overrides.
|
|
120
|
-
* @param overrides - Plugin setting overrides from CLI flags.
|
|
121
|
-
*/
|
|
122
|
-
setPluginOverrides(overrides: PluginOverrides): void;
|
|
123
|
-
/**
|
|
124
|
-
* Writes the archive (including any new analysis results) to disk
|
|
125
|
-
* as a `.nitpicker` tar file, then emits a `writeFile` event.
|
|
126
|
-
* @fires NitpickerEvent#writeFile
|
|
127
|
-
*/
|
|
128
|
-
write(): Promise<void>;
|
|
129
|
-
/**
|
|
130
|
-
* Opens an existing `.nitpicker` archive file and returns a ready-to-use
|
|
131
|
-
* Nitpicker instance.
|
|
132
|
-
*
|
|
133
|
-
* This is the recommended way to create a Nitpicker instance. It extracts
|
|
134
|
-
* the archive to a temporary directory, opens the SQLite database, and
|
|
135
|
-
* enables plugin data access.
|
|
136
|
-
* @param filePath - Path to the `.nitpicker` archive file.
|
|
137
|
-
* @param onExtractProgress - Forwarded to `Archive.open` — called during
|
|
138
|
-
* the untar step with bytes read so far and the archive's total size
|
|
139
|
-
* (issue #294: a large archive's extraction can take tens of seconds
|
|
140
|
-
* with no other signal it isn't hung).
|
|
141
|
-
* @returns A new Nitpicker instance backed by the opened archive.
|
|
142
|
-
* @example
|
|
143
|
-
* ```ts
|
|
144
|
-
* const nitpicker = await Nitpicker.open('./site.nitpicker');
|
|
145
|
-
* await nitpicker.analyze();
|
|
146
|
-
* await nitpicker.write();
|
|
147
|
-
* ```
|
|
148
|
-
*/
|
|
149
|
-
static open(filePath: string, onExtractProgress?: (readBytes: number, totalBytes: number) => void): Promise<Nitpicker>;
|
|
150
|
-
}
|
package/lib/nitpicker.js
DELETED
|
@@ -1,437 +0,0 @@
|
|
|
1
|
-
import os from 'node:os';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import { Cache } from '@d-zero/shared/cache';
|
|
4
|
-
import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
|
|
5
|
-
import { Archive } from '@nitpicker/crawler';
|
|
6
|
-
import c from 'ansi-colors';
|
|
7
|
-
import { getTableCacheRoot } from './get-table-cache-root.js';
|
|
8
|
-
import { importModules } from './import-modules.js';
|
|
9
|
-
import { loadPluginSettings } from './load-plugin-settings.js';
|
|
10
|
-
import { Table } from './table.js';
|
|
11
|
-
import { UrlEventBus } from './url-event-bus.js';
|
|
12
|
-
import { WorkerPool } from './worker/worker-pool.js';
|
|
13
|
-
export { definePlugin } from './hooks/define-plugin.js';
|
|
14
|
-
const __filename = new URL(import.meta.url).pathname;
|
|
15
|
-
const __dirname = path.dirname(__filename);
|
|
16
|
-
/**
|
|
17
|
-
* Resolved path to the compiled per-page analysis module.
|
|
18
|
-
* Each task posted to the pool carries this path so the worker can
|
|
19
|
-
* dynamically import it (cached after first import).
|
|
20
|
-
* @see {@link ./page-analysis-worker.ts} for the source
|
|
21
|
-
*/
|
|
22
|
-
const pageAnalysisWorkerPath = path.resolve(__dirname, 'page-analysis-worker.js');
|
|
23
|
-
/**
|
|
24
|
-
* Resolved path to the compiled long-lived worker thread script.
|
|
25
|
-
* Used as the `workerPath` when constructing {@link WorkerPool} instances.
|
|
26
|
-
*/
|
|
27
|
-
const workerPath = path.resolve(__dirname, 'worker/worker.js');
|
|
28
|
-
/**
|
|
29
|
-
* Default concurrency used when a plugin does not declare its own value.
|
|
30
|
-
* Tracks the number of CPU cores so that worker pools scale with the host.
|
|
31
|
-
*/
|
|
32
|
-
const DEFAULT_CONCURRENCY = Math.max(1, os.cpus().length);
|
|
33
|
-
/**
|
|
34
|
-
* Core orchestrator for running analyze plugins against a `.nitpicker` archive.
|
|
35
|
-
*
|
|
36
|
-
* Nitpicker opens an existing archive (produced by the crawler), loads the
|
|
37
|
-
* user's plugin configuration via cosmiconfig, then runs each plugin against
|
|
38
|
-
* every page in the archive. Results are stored back into the archive as
|
|
39
|
-
* `analysis/report`, `analysis/table`, and SQL-backed analysis violation
|
|
40
|
-
* tables via the archive facade.
|
|
41
|
-
*
|
|
42
|
-
* ## Architecture decisions
|
|
43
|
-
*
|
|
44
|
-
* - **Worker pool per plugin**: Each plugin gets its own long-lived
|
|
45
|
-
* {@link ./worker/worker-pool.ts!WorkerPool}, sized by either the plugin's
|
|
46
|
-
* declared `concurrency` or {@link DEFAULT_CONCURRENCY} (CPU core count).
|
|
47
|
-
* Workers are spawned once and reused across every page, so the V8 isolate
|
|
48
|
-
* / JSDOM module / plugin module boot cost is paid only `concurrency`
|
|
49
|
-
* times per plugin instead of once per page.
|
|
50
|
-
*
|
|
51
|
-
* - **Plugin-outer, page-inner loop**: Plugins are processed sequentially.
|
|
52
|
-
* For each plugin, pages are submitted to that plugin's pool, which queues
|
|
53
|
-
* them and dispatches to idle workers. This enables per-plugin progress
|
|
54
|
-
* tracking via {@link https://www.npmjs.com/package/@d-zero/dealer | Lanes}.
|
|
55
|
-
*
|
|
56
|
-
* - **Cache layer**: Results are cached per `pluginName:url` using
|
|
57
|
-
* `@d-zero/shared/cache` so that re-running analysis after a partial failure
|
|
58
|
-
* skips already-processed pages. The cache is cleared at the start of each run.
|
|
59
|
-
* @example
|
|
60
|
-
* ```ts
|
|
61
|
-
* import { Nitpicker } from '@nitpicker/core';
|
|
62
|
-
*
|
|
63
|
-
* // Open an existing archive — `Symbol.asyncDispose` closes it on scope exit
|
|
64
|
-
* await using nitpicker = await Nitpicker.open('./example.nitpicker');
|
|
65
|
-
*
|
|
66
|
-
* // Run all configured analyze plugins
|
|
67
|
-
* await nitpicker.analyze();
|
|
68
|
-
*
|
|
69
|
-
* // Or run only specific plugins by name
|
|
70
|
-
* await nitpicker.analyze(['@nitpicker/analyze-axe']);
|
|
71
|
-
*
|
|
72
|
-
* // Write updated archive back to disk
|
|
73
|
-
* await nitpicker.write();
|
|
74
|
-
* ```
|
|
75
|
-
* @see {@link ./types.ts!NitpickerEvent} for emitted events
|
|
76
|
-
* @see {@link ./types.ts!Config} for the resolved configuration model
|
|
77
|
-
*/
|
|
78
|
-
export class Nitpicker extends EventEmitter {
|
|
79
|
-
/**
|
|
80
|
-
* The underlying archive instance providing access to the SQLite database
|
|
81
|
-
* and file storage. Injected via constructor or created by `Nitpicker.open()`.
|
|
82
|
-
*/
|
|
83
|
-
#archive;
|
|
84
|
-
/**
|
|
85
|
-
* Lazily loaded and cached plugin configuration.
|
|
86
|
-
* `null` until `getConfig()` is first called.
|
|
87
|
-
*/
|
|
88
|
-
#config = null;
|
|
89
|
-
/**
|
|
90
|
-
* CLI-specified plugin setting overrides.
|
|
91
|
-
* Passed to `loadPluginSettings()` to merge with config-file settings.
|
|
92
|
-
*/
|
|
93
|
-
#pluginOverrides = {};
|
|
94
|
-
/** The underlying archive instance. */
|
|
95
|
-
get archive() {
|
|
96
|
-
return this.#archive;
|
|
97
|
-
}
|
|
98
|
-
/**
|
|
99
|
-
* @param archive - An opened {@link Archive} instance to analyze.
|
|
100
|
-
* Use {@link Nitpicker.open} for a convenient static factory.
|
|
101
|
-
*/
|
|
102
|
-
constructor(archive) {
|
|
103
|
-
super();
|
|
104
|
-
this.#archive = archive;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Enables `await using nitpicker = ...`. Delegates to the wrapped
|
|
108
|
-
* archive's `Symbol.asyncDispose` (i.e. {@link Archive.close}).
|
|
109
|
-
*/
|
|
110
|
-
async [Symbol.asyncDispose]() {
|
|
111
|
-
await this.#archive.close();
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Runs all configured analyze plugins (or a filtered subset) against
|
|
115
|
-
* every page in the archive.
|
|
116
|
-
*
|
|
117
|
-
* Plugins are processed **sequentially** (one at a time). For each plugin,
|
|
118
|
-
* pages are processed in **parallel** through a dedicated long-lived
|
|
119
|
-
* {@link ./worker/worker-pool.ts!WorkerPool} sized either by the plugin's
|
|
120
|
-
* declared concurrency or by {@link DEFAULT_CONCURRENCY}. This architecture
|
|
121
|
-
* enables per-plugin progress tracking via Lanes.
|
|
122
|
-
*
|
|
123
|
-
* The analysis proceeds in up to two phases:
|
|
124
|
-
*
|
|
125
|
-
* 1. **`eachPage` phase** - For each plugin with `eachPage`, dispatches
|
|
126
|
-
* page analysis tasks to the plugin's worker pool. Progress is
|
|
127
|
-
* displayed via Lanes if provided in options.
|
|
128
|
-
*
|
|
129
|
-
* 2. **`eachUrl` phase** - For all plugins with `eachUrl`, runs
|
|
130
|
-
* sequentially in the main thread. These are lightweight checks
|
|
131
|
-
* that don't need DOM access.
|
|
132
|
-
*
|
|
133
|
-
* On completion, data entries are stored in the archive:
|
|
134
|
-
* - `analysis/report` - Full {@link Report} with headers and data
|
|
135
|
-
* - `analysis/table` - The raw {@link Table} instance (serialized)
|
|
136
|
-
* - SQL-backed analysis violation tables - Flat {@link Violation} records
|
|
137
|
-
* @param filter - Optional list of plugin module names to run.
|
|
138
|
-
* If omitted, all configured plugins are executed.
|
|
139
|
-
* @param options - Optional settings for progress display.
|
|
140
|
-
* @example
|
|
141
|
-
* ```ts
|
|
142
|
-
* // Run all plugins
|
|
143
|
-
* await nitpicker.analyze();
|
|
144
|
-
*
|
|
145
|
-
* // Run only axe and markuplint with Lanes progress
|
|
146
|
-
* await nitpicker.analyze(
|
|
147
|
-
* ['@nitpicker/analyze-axe', '@nitpicker/analyze-markuplint'],
|
|
148
|
-
* { lanes },
|
|
149
|
-
* );
|
|
150
|
-
* ```
|
|
151
|
-
*/
|
|
152
|
-
async analyze(filter, options) {
|
|
153
|
-
const config = await this.getConfig();
|
|
154
|
-
const plugins = filter
|
|
155
|
-
? config.analyze.filter((plugin) => filter?.includes(plugin.name))
|
|
156
|
-
: config.analyze;
|
|
157
|
-
const analyzeMods = await importModules(plugins);
|
|
158
|
-
const lanes = options?.lanes;
|
|
159
|
-
const table = new Table();
|
|
160
|
-
for (const mod of analyzeMods) {
|
|
161
|
-
if (!mod.headers) {
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
164
|
-
if (!mod.eachPage) {
|
|
165
|
-
continue;
|
|
166
|
-
}
|
|
167
|
-
table.addHeaders(mod.headers);
|
|
168
|
-
}
|
|
169
|
-
const allViolations = [];
|
|
170
|
-
const cache = new Cache('nitpicker-axe', getTableCacheRoot());
|
|
171
|
-
await cache.clear();
|
|
172
|
-
// Build plugin metadata: lane IDs and display labels
|
|
173
|
-
const eachPagePlugins = [];
|
|
174
|
-
for (const [i, plugin] of plugins.entries()) {
|
|
175
|
-
if (analyzeMods[i]?.eachPage) {
|
|
176
|
-
eachPagePlugins.push({ plugin, modIndex: i });
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
const pluginLaneIds = new Map();
|
|
180
|
-
const pluginLabels = new Map();
|
|
181
|
-
const pluginCompletionDetails = new Map();
|
|
182
|
-
for (const [laneId, { plugin, modIndex }] of eachPagePlugins.entries()) {
|
|
183
|
-
pluginLaneIds.set(plugin.name, laneId);
|
|
184
|
-
pluginLabels.set(plugin.name, analyzeMods[modIndex]?.label ?? plugin.name);
|
|
185
|
-
}
|
|
186
|
-
// Initialize all lanes as Waiting
|
|
187
|
-
for (const [name, id] of pluginLaneIds) {
|
|
188
|
-
const label = pluginLabels.get(name) ?? name;
|
|
189
|
-
lanes?.update(id, c.dim(`${label}: Waiting...`));
|
|
190
|
-
}
|
|
191
|
-
await this.archive.getPagesWithRefs(100_000, async (pages) => {
|
|
192
|
-
const urlEmitter = new UrlEventBus();
|
|
193
|
-
// Phase 1: eachPage plugins (sequentially, pages in parallel)
|
|
194
|
-
for (const [pluginSeqIndex, { plugin, modIndex }] of eachPagePlugins.entries()) {
|
|
195
|
-
const laneId = pluginLaneIds.get(plugin.name);
|
|
196
|
-
const label = pluginLabels.get(plugin.name) ?? plugin.name;
|
|
197
|
-
let done = 0;
|
|
198
|
-
let pluginViolationCount = 0;
|
|
199
|
-
const updateProgress = () => {
|
|
200
|
-
const pluginPercent = Math.round((done / pages.length) * 100);
|
|
201
|
-
const overallPercent = Math.round(((pluginSeqIndex + done / pages.length) / eachPagePlugins.length) * 100);
|
|
202
|
-
lanes?.header(`[${pluginSeqIndex + 1}/${eachPagePlugins.length}] Analyzing (${overallPercent}%)`);
|
|
203
|
-
lanes?.update(laneId, `${label}: ${done}/${pages.length} (${pluginPercent}%)%braille%`);
|
|
204
|
-
};
|
|
205
|
-
updateProgress();
|
|
206
|
-
const pluginConcurrency = Math.max(1, analyzeMods[modIndex]?.concurrency ?? DEFAULT_CONCURRENCY);
|
|
207
|
-
const pool = new WorkerPool({
|
|
208
|
-
size: pluginConcurrency,
|
|
209
|
-
workerPath,
|
|
210
|
-
});
|
|
211
|
-
let pluginPageDataCount = 0;
|
|
212
|
-
let pluginErrorCount = 0;
|
|
213
|
-
try {
|
|
214
|
-
// Bound the number of in-flight IIFEs so HTML strings loaded
|
|
215
|
-
// via page.getHtml() do not pile up on the main thread while
|
|
216
|
-
// they wait for a worker. A small buffer above pool size keeps
|
|
217
|
-
// the pool continuously fed without staging too much HTML.
|
|
218
|
-
const inFlight = new Set();
|
|
219
|
-
const highWater = pluginConcurrency * 2;
|
|
220
|
-
for (const [pageIndex, page] of pages.entries()) {
|
|
221
|
-
const task = (async () => {
|
|
222
|
-
try {
|
|
223
|
-
const cacheKey = `${plugin.name}:${page.url.href}`;
|
|
224
|
-
const cached = await cache.load(cacheKey);
|
|
225
|
-
if (cached) {
|
|
226
|
-
const { pages: cachedPages, violations } = cached;
|
|
227
|
-
if (cachedPages) {
|
|
228
|
-
table.addData(cachedPages);
|
|
229
|
-
pluginPageDataCount += Object.keys(cachedPages).length;
|
|
230
|
-
}
|
|
231
|
-
if (violations) {
|
|
232
|
-
for (const v of violations) {
|
|
233
|
-
allViolations.push(v);
|
|
234
|
-
}
|
|
235
|
-
pluginViolationCount += violations.length;
|
|
236
|
-
}
|
|
237
|
-
done++;
|
|
238
|
-
updateProgress();
|
|
239
|
-
return;
|
|
240
|
-
}
|
|
241
|
-
const html = await page.getHtml();
|
|
242
|
-
if (!html) {
|
|
243
|
-
done++;
|
|
244
|
-
updateProgress();
|
|
245
|
-
return;
|
|
246
|
-
}
|
|
247
|
-
const report = await pool.run({
|
|
248
|
-
filePath: pageAnalysisWorkerPath,
|
|
249
|
-
num: pageIndex,
|
|
250
|
-
total: pages.length,
|
|
251
|
-
emitter: urlEmitter,
|
|
252
|
-
initialData: {
|
|
253
|
-
plugin,
|
|
254
|
-
pages: {
|
|
255
|
-
html,
|
|
256
|
-
url: page.url,
|
|
257
|
-
},
|
|
258
|
-
},
|
|
259
|
-
});
|
|
260
|
-
const tablePages = {};
|
|
261
|
-
if (report?.page) {
|
|
262
|
-
tablePages[page.url.href] = report.page;
|
|
263
|
-
table.addDataToUrl(page.url, report.page);
|
|
264
|
-
pluginPageDataCount++;
|
|
265
|
-
}
|
|
266
|
-
await cache.store(cacheKey, {
|
|
267
|
-
pages: Object.keys(tablePages).length > 0 ? tablePages : undefined,
|
|
268
|
-
violations: report?.violations,
|
|
269
|
-
});
|
|
270
|
-
if (report?.violations) {
|
|
271
|
-
for (const v of report.violations) {
|
|
272
|
-
allViolations.push(v);
|
|
273
|
-
}
|
|
274
|
-
pluginViolationCount += report.violations.length;
|
|
275
|
-
}
|
|
276
|
-
done++;
|
|
277
|
-
updateProgress();
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
pluginErrorCount++;
|
|
281
|
-
done++;
|
|
282
|
-
updateProgress();
|
|
283
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
284
|
-
await this.emit('error', {
|
|
285
|
-
message: `[${plugin.name}] Failed to analyze ${page.url.href}: ${message}`,
|
|
286
|
-
error: error instanceof Error ? error : null,
|
|
287
|
-
});
|
|
288
|
-
}
|
|
289
|
-
})();
|
|
290
|
-
inFlight.add(task);
|
|
291
|
-
task.then(() => inFlight.delete(task), () => inFlight.delete(task));
|
|
292
|
-
if (inFlight.size >= highWater) {
|
|
293
|
-
await Promise.race(inFlight);
|
|
294
|
-
}
|
|
295
|
-
}
|
|
296
|
-
await Promise.all(inFlight);
|
|
297
|
-
}
|
|
298
|
-
finally {
|
|
299
|
-
await pool.terminate();
|
|
300
|
-
}
|
|
301
|
-
// Warn when plugin produced no page data at all
|
|
302
|
-
if (pluginPageDataCount === 0 && pages.length > 0) {
|
|
303
|
-
const errorDetail = pluginErrorCount > 0 ? ` (${pluginErrorCount} errors occurred)` : '';
|
|
304
|
-
await this.emit('error', {
|
|
305
|
-
message: `[${plugin.name}] Produced no data for ${pages.length} pages${errorDetail}. Check plugin configuration and HTML snapshots.`,
|
|
306
|
-
error: null,
|
|
307
|
-
});
|
|
308
|
-
}
|
|
309
|
-
// Mark this plugin as Done
|
|
310
|
-
const detail = pluginViolationCount > 0
|
|
311
|
-
? `${pluginViolationCount} violations`
|
|
312
|
-
: `${done} pages`;
|
|
313
|
-
pluginCompletionDetails.set(plugin.name, detail);
|
|
314
|
-
lanes?.update(laneId, c.green(`${label}: Done (${detail})`));
|
|
315
|
-
// Dim inactive lanes
|
|
316
|
-
for (const [name, id] of pluginLaneIds) {
|
|
317
|
-
if (name === plugin.name) {
|
|
318
|
-
continue;
|
|
319
|
-
}
|
|
320
|
-
const otherLabel = pluginLabels.get(name) ?? name;
|
|
321
|
-
const completionDetail = pluginCompletionDetails.get(name);
|
|
322
|
-
if (completionDetail) {
|
|
323
|
-
lanes?.update(id, c.dim(`${otherLabel}: Done (${completionDetail})`));
|
|
324
|
-
}
|
|
325
|
-
else {
|
|
326
|
-
lanes?.update(id, c.dim(`${otherLabel}: Waiting...`));
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
// Phase 2: eachUrl plugins (main thread, sequential)
|
|
331
|
-
for (const page of pages) {
|
|
332
|
-
const url = page.url;
|
|
333
|
-
const isExternal = page.isExternal;
|
|
334
|
-
for (const mod of analyzeMods) {
|
|
335
|
-
if (!mod.eachUrl) {
|
|
336
|
-
continue;
|
|
337
|
-
}
|
|
338
|
-
try {
|
|
339
|
-
const report = await mod.eachUrl({ url, isExternal });
|
|
340
|
-
if (!report) {
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
const { page: reportPage, violations } = report;
|
|
344
|
-
if (reportPage) {
|
|
345
|
-
table.addDataToUrl(url, reportPage);
|
|
346
|
-
}
|
|
347
|
-
if (violations) {
|
|
348
|
-
for (const v of violations) {
|
|
349
|
-
allViolations.push(v);
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
catch (error) {
|
|
354
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
355
|
-
await this.emit('error', {
|
|
356
|
-
message: `[eachUrl] Failed to analyze ${url.href}: ${message}`,
|
|
357
|
-
error: error instanceof Error ? error : null,
|
|
358
|
-
});
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
}, {
|
|
363
|
-
withRefs: false,
|
|
364
|
-
});
|
|
365
|
-
const report = {
|
|
366
|
-
name: 'general',
|
|
367
|
-
pageData: table.toJSON(),
|
|
368
|
-
};
|
|
369
|
-
await this.archive.setData('analysis/report', report);
|
|
370
|
-
await this.archive.setData('analysis/table', table);
|
|
371
|
-
await this.archive.replaceAnalysisViolations(allViolations);
|
|
372
|
-
}
|
|
373
|
-
/**
|
|
374
|
-
* Loads and caches the plugin configuration from the user's config file.
|
|
375
|
-
*
|
|
376
|
-
* Uses cosmiconfig to search for `.nitpickerrc`, `.nitpickerrc.json`,
|
|
377
|
-
* `nitpicker.config.js`, or a `nitpicker` key in `package.json`.
|
|
378
|
-
* The result is cached after the first call.
|
|
379
|
-
* @returns Resolved {@link Config} with the `analyze` plugin list.
|
|
380
|
-
*/
|
|
381
|
-
async getConfig() {
|
|
382
|
-
if (!this.#config) {
|
|
383
|
-
this.#config = await loadPluginSettings({}, this.#pluginOverrides);
|
|
384
|
-
}
|
|
385
|
-
return this.#config;
|
|
386
|
-
}
|
|
387
|
-
/**
|
|
388
|
-
* Sets CLI-specified plugin overrides that take precedence over
|
|
389
|
-
* config-file settings.
|
|
390
|
-
*
|
|
391
|
-
* Must be called before {@link getConfig} to take effect. If config
|
|
392
|
-
* has already been loaded, calling this method clears the cache so
|
|
393
|
-
* that the next `getConfig()` call re-loads with the new overrides.
|
|
394
|
-
* @param overrides - Plugin setting overrides from CLI flags.
|
|
395
|
-
*/
|
|
396
|
-
setPluginOverrides(overrides) {
|
|
397
|
-
this.#pluginOverrides = overrides;
|
|
398
|
-
this.#config = null;
|
|
399
|
-
}
|
|
400
|
-
/**
|
|
401
|
-
* Writes the archive (including any new analysis results) to disk
|
|
402
|
-
* as a `.nitpicker` tar file, then emits a `writeFile` event.
|
|
403
|
-
* @fires NitpickerEvent#writeFile
|
|
404
|
-
*/
|
|
405
|
-
async write() {
|
|
406
|
-
await this.#archive.write();
|
|
407
|
-
await this.emit('writeFile', { filePath: this.#archive.filePath });
|
|
408
|
-
}
|
|
409
|
-
/**
|
|
410
|
-
* Opens an existing `.nitpicker` archive file and returns a ready-to-use
|
|
411
|
-
* Nitpicker instance.
|
|
412
|
-
*
|
|
413
|
-
* This is the recommended way to create a Nitpicker instance. It extracts
|
|
414
|
-
* the archive to a temporary directory, opens the SQLite database, and
|
|
415
|
-
* enables plugin data access.
|
|
416
|
-
* @param filePath - Path to the `.nitpicker` archive file.
|
|
417
|
-
* @param onExtractProgress - Forwarded to `Archive.open` — called during
|
|
418
|
-
* the untar step with bytes read so far and the archive's total size
|
|
419
|
-
* (issue #294: a large archive's extraction can take tens of seconds
|
|
420
|
-
* with no other signal it isn't hung).
|
|
421
|
-
* @returns A new Nitpicker instance backed by the opened archive.
|
|
422
|
-
* @example
|
|
423
|
-
* ```ts
|
|
424
|
-
* const nitpicker = await Nitpicker.open('./site.nitpicker');
|
|
425
|
-
* await nitpicker.analyze();
|
|
426
|
-
* await nitpicker.write();
|
|
427
|
-
* ```
|
|
428
|
-
*/
|
|
429
|
-
static async open(filePath, onExtractProgress) {
|
|
430
|
-
const archive = await Archive.open({
|
|
431
|
-
filePath,
|
|
432
|
-
openPluginData: true,
|
|
433
|
-
onExtractProgress,
|
|
434
|
-
});
|
|
435
|
-
return new Nitpicker(archive);
|
|
436
|
-
}
|
|
437
|
-
}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Worker thread module for per-page single-plugin execution.
|
|
3
|
-
*
|
|
4
|
-
* This is the default export loaded by {@link ./worker/runner.ts!runner}
|
|
5
|
-
* for each task dispatched by the {@link ./worker/worker-pool.ts!WorkerPool}. It:
|
|
6
|
-
*
|
|
7
|
-
* 1. Dynamically imports the configured plugin via {@link importModules}
|
|
8
|
-
* (cached after the first task on the same worker)
|
|
9
|
-
* 2. If the plugin implements `eachPage`:
|
|
10
|
-
* - Creates a JSDOM instance from the raw HTML, with `runScripts:
|
|
11
|
-
* 'outside-only'` so that plugins can use `window.eval` to run
|
|
12
|
-
* DOM-scoped scripts (e.g. injecting a browser-oriented library that
|
|
13
|
-
* needs to bind to that specific window — see
|
|
14
|
-
* `@nitpicker/analyze-axe`'s `injectAxe`)
|
|
15
|
-
* - Calls the plugin's `eachPage` hook with the DOM window
|
|
16
|
-
* - Closes the JSDOM window to free memory
|
|
17
|
-
* 3. Returns the plugin result as {@link ReportPage} or `null`
|
|
18
|
-
*
|
|
19
|
-
* One invocation processes exactly one plugin × one page, even though
|
|
20
|
-
* the worker hosting this module handles many tasks over its lifetime.
|
|
21
|
-
* `eachPage` plugins must not rely on `dom.window` being reachable from
|
|
22
|
-
* `globalThis` — nothing exposes it there — and must not cache state that
|
|
23
|
-
* outlives a single call, since the window backing it is closed once the
|
|
24
|
-
* call returns.
|
|
25
|
-
* @module
|
|
26
|
-
*/
|
|
27
|
-
import type { Plugin, ReportPage } from './types.js';
|
|
28
|
-
import type { UrlEventBus } from './url-event-bus.js';
|
|
29
|
-
import type { ExURL as URL } from '@d-zero/shared/parse-url';
|
|
30
|
-
/**
|
|
31
|
-
* Initial data payload for the page analysis worker.
|
|
32
|
-
* Wrapped inside the `task` message dispatched by the {@link ./worker/worker-pool.ts!WorkerPool}
|
|
33
|
-
* and consumed by this module's default export.
|
|
34
|
-
*
|
|
35
|
-
* Uses `type` instead of `interface` because this type must satisfy the
|
|
36
|
-
* `Record<string, unknown>` constraint required by the Worker message serialization.
|
|
37
|
-
*/
|
|
38
|
-
export type PageAnalysisWorkerData = {
|
|
39
|
-
/** Single analyze plugin to execute against the page. */
|
|
40
|
-
plugin: Plugin;
|
|
41
|
-
/** Page data containing raw HTML and parsed URL. */
|
|
42
|
-
pages: {
|
|
43
|
-
html: string;
|
|
44
|
-
url: URL;
|
|
45
|
-
};
|
|
46
|
-
};
|
|
47
|
-
/**
|
|
48
|
-
* Executes a single plugin's `eachPage` hook against a single page.
|
|
49
|
-
* @template T - Column key union from the plugin's headers.
|
|
50
|
-
* @param data - Contains the single plugin and page data (HTML + URL).
|
|
51
|
-
* @param urlEventBus - Event bus for URL discovery events (forwarded to the main thread).
|
|
52
|
-
* @param num - Zero-based index of the current page in the batch.
|
|
53
|
-
* @param total - Total number of pages being processed.
|
|
54
|
-
* @returns Report data from the plugin, or `null` if skipped.
|
|
55
|
-
* @see {@link ./worker/runner.ts!runner} for how this function is called
|
|
56
|
-
* @see {@link ./nitpicker.ts!Nitpicker.analyze} for the orchestration context
|
|
57
|
-
*/
|
|
58
|
-
export default function <T extends string>(data: PageAnalysisWorkerData, urlEventBus: UrlEventBus, num: number, total: number): Promise<ReportPage<T> | null>;
|