@nitpicker/cli 0.4.2

package/lib/commands/crawl.js

@@ -0,0 +1,249 @@
+import path from 'node:path';
+import { readList } from '@d-zero/readtext/list';
+import { CrawlerOrchestrator } from '@nitpicker/crawler';
+import { log, verbosely } from '../crawl/debug.js';
+import { diff } from '../crawl/diff.js';
+import { eventAssignments } from '../crawl/event-assignments.js';
+/**
+ * Command definition for the `crawl` sub-command.
+ * Defines all CLI flags with their types, defaults, and descriptions.
+ * @see {@link crawl} for the main entry point that dispatches to startCrawl/resumeCrawl/diff
+ */
+export const commandDef = {
+    desc: 'Crawl a website',
+    flags: {
+        resume: {
+            type: 'string',
+            shortFlag: 'R',
+            desc: 'Resume crawling from a stub file',
+        },
+        interval: {
+            type: 'number',
+            shortFlag: 'I',
+            desc: 'An interval time on request when crawles',
+        },
+        image: {
+            type: 'boolean',
+            default: true,
+            desc: 'Getting images (use --no-image to disable)',
+        },
+        fetchExternal: {
+            type: 'boolean',
+            default: true,
+            desc: 'Fetch external links (use --no-fetch-external to disable)',
+        },
+        parallels: {
+            type: 'number',
+            shortFlag: 'P',
+            desc: 'Number of parallel scraping',
+        },
+        recursive: {
+            type: 'boolean',
+            default: true,
+            desc: 'Recursive crawling (use --no-recursive to disable)',
+        },
+        scope: {
+            type: 'string',
+            desc: 'Set hosts and URLs as scope',
+        },
+        exclude: {
+            type: 'string',
+            isMultiple: true,
+            desc: 'Excluding page URL path (glob pattern)',
+        },
+        excludeKeyword: {
+            type: 'string',
+            isMultiple: true,
+            desc: 'Exclude keyword in document of page',
+        },
+        excludeUrl: {
+            type: 'string',
+            isMultiple: true,
+            desc: 'Exclude external URL prefix',
+        },
+        disableQueries: {
+            type: 'boolean',
+            shortFlag: 'Q',
+            desc: 'Disable queries that the URL has',
+        },
+        imageFileSizeThreshold: {
+            type: 'number',
+            desc: 'Image file size threshold',
+        },
+        single: {
+            type: 'boolean',
+            desc: 'Single page mode',
+        },
+        maxExcludedDepth: {
+            type: 'number',
+            desc: 'Avoid crawling depths above a set number',
+        },
+        retry: {
+            type: 'number',
+            default: 3,
+            desc: 'Number of retry attempts per URL on scrape failure',
+        },
+        list: {
+            type: 'string',
+            isMultiple: true,
+            desc: 'Running only each page from the list',
+        },
+        listFile: {
+            type: 'string',
+            desc: 'Running only each page from the list file',
+        },
+        userAgent: {
+            type: 'string',
+            desc: 'Custom User-Agent string for HTTP requests',
+        },
+        ignoreRobots: {
+            type: 'boolean',
+            desc: 'Ignore robots.txt restrictions (use responsibly)',
+        },
+        verbose: {
+            type: 'boolean',
+            desc: 'Output verbose log to standard out',
+        },
+        silent: {
+            type: 'boolean',
+            desc: 'No output log to standard out',
+        },
+        diff: {
+            type: 'boolean',
+            desc: 'Diff mode',
+        },
+    },
+};
+/**
+ * Sets up signal handlers for graceful shutdown and starts event logging.
+ *
+ * Registers SIGINT/SIGBREAK/SIGHUP/SIGABRT handlers that kill zombie
+ * Chromium processes before exiting, then delegates to {@link eventAssignments}
+ * for progress output.
+ * @param trigger - Display label for the crawl (URL or stub file path)
+ * @param orchestrator - The initialized CrawlerOrchestrator instance
+ * @param config - The resolved archive configuration
+ * @param logType - Output verbosity level
+ */
+function run(trigger, orchestrator, config, logType) {
+    const killed = () => {
+        orchestrator.garbageCollect();
+        process.exit();
+    };
+    process.on('SIGINT', killed);
+    process.on('SIGBREAK', killed);
+    process.on('SIGHUP', killed);
+    process.on('SIGABRT', killed);
+    const head = [
+        `🐳 ${trigger} (New scraping)`,
+        ...Object.entries(config).map(([key, value]) => `  ${key}: ${value}`),
+    ];
+    return eventAssignments(orchestrator, head, logType);
+}
+/**
+ * Starts a fresh crawl session for the given URLs.
+ *
+ * Creates a CrawlerOrchestrator, runs the crawl, writes the archive,
+ * and cleans up browser processes. Exits with code 1 if errors occurred.
+ * @param siteUrl - One or more root URLs to crawl
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+async function startCrawl(siteUrl, flags) {
+    const errStack = [];
+    const isList = !!flags.list?.length;
+    const orchestrator = await CrawlerOrchestrator.crawling(siteUrl, {
+        ...flags,
+        scope: flags.scope?.split(',').map((s) => s.trim()),
+        list: isList,
+        recursive: isList ? false : flags.recursive,
+    }, (orchestrator, config) => {
+        run(config.baseUrl, orchestrator, config, flags.verbose ? 'verbose' : flags.silent ? 'silent' : 'normal').catch((error) => errStack.push(error));
+    });
+    await orchestrator.write();
+    orchestrator.garbageCollect();
+    if (errStack.length > 0) {
+        formatCrawlErrors(errStack);
+        process.exit(1);
+    }
+}
+/**
+ * Resumes a previously interrupted crawl from a stub file (temporary directory).
+ *
+ * Restores the crawl state from the archive, applies any flag overrides,
+ * and continues crawling from where the previous session left off.
+ * @param stubFilePath - Path to the stub file or temporary directory to resume from
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+async function resumeCrawl(stubFilePath, flags) {
+    const errStack = [];
+    const absFilePath = path.isAbsolute(stubFilePath)
+        ? stubFilePath
+        : path.resolve(process.cwd(), stubFilePath);
+    const orchestrator = await CrawlerOrchestrator.resume(absFilePath, {
+        ...flags,
+        scope: flags.scope?.split(',').map((s) => s.trim()),
+        list: false,
+    }, (orchestrator, config) => {
+        run(stubFilePath, orchestrator, config, flags.verbose ? 'verbose' : flags.silent ? 'silent' : 'normal').catch((error) => errStack.push(error));
+    });
+    await orchestrator.write();
+    orchestrator.garbageCollect();
+    if (errStack.length > 0) {
+        formatCrawlErrors(errStack);
+        process.exit(1);
+    }
+}
+/**
+ * Main entry point for the `crawl` CLI command.
+ *
+ * Dispatches to one of four modes based on the flags:
+ * 1. `--diff` mode: Compares two archive files and outputs URL lists
+ * 2. `--resume` mode: Resumes a previously interrupted crawl
+ * 3. `--list-file` / `--list` mode: Crawls a pre-defined URL list (non-recursive)
+ * 4. Default mode: Crawls from a single root URL
+ * @param args - Positional arguments (typically one or two URLs/file paths)
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+export async function crawl(args, flags) {
+    if (flags.verbose && !flags.silent) {
+        verbosely();
+    }
+    log('Options: %O', flags);
+    if (flags.diff) {
+        const a = args[0];
+        const b = args[1];
+        if (!a || !b) {
+            throw new Error('Please provide two file paths to compare');
+        }
+        await diff(a, b);
+        return;
+    }
+    if (flags.resume) {
+        await resumeCrawl(flags.resume, flags);
+        return;
+    }
+    if (flags.listFile) {
+        const list = await readList(path.resolve(process.cwd(), flags.listFile));
+        flags.list = list;
+        await startCrawl(list, flags);
+        return;
+    }
+    if (flags.list && flags.list.length > 0) {
+        const pageList = [...flags.list, ...args];
+        await startCrawl(pageList, flags);
+        return;
+    }
+    const siteUrl = args[0];
+    if (siteUrl) {
+        await startCrawl([siteUrl], flags);
+        return;
+    }
+}
+/**
+ * Prints a summary of errors that occurred during crawling to stderr.
+ * @param errStack - Array of errors collected during the crawl session
+ */
+function formatCrawlErrors(errStack) {
+    // eslint-disable-next-line no-console
+    console.error(`\nCompleted with ${errStack.length} error(s).`);
+}

package/lib/commands/report.d.ts

@@ -0,0 +1,45 @@
+import type { InferFlags } from '@d-zero/roar';
+/**
+ * Command definition for the `report` sub-command.
+ * @see {@link report} for the main entry point
+ */
+export declare const commandDef: {
+    readonly desc: "Generate a Google Sheets report";
+    readonly flags: {
+        readonly sheet: {
+            readonly shortFlag: "S";
+            readonly type: "string";
+            readonly isRequired: true;
+            readonly desc: "Google Sheets URL";
+        };
+        readonly credentials: {
+            readonly shortFlag: "C";
+            readonly type: "string";
+            readonly default: "./credentials.json";
+            readonly desc: "Path to credentials file (keep this file secure and out of version control)";
+        };
+        readonly config: {
+            readonly shortFlag: "c";
+            readonly type: "string";
+            readonly desc: "Path to config file";
+        };
+        readonly limit: {
+            readonly shortFlag: "l";
+            readonly type: "number";
+            readonly default: 100000;
+            readonly desc: "Limit number of rows";
+        };
+    };
+};
+type ReportFlags = InferFlags<typeof commandDef.flags>;
+/**
+ * Main entry point for the `report` CLI command.
+ *
+ * Reads a `.nitpicker` archive and generates a Google Sheets report
+ * by delegating to `@nitpicker/report-google-sheets`. Requires a Google
+ * Sheets URL and a service account credentials file.
+ * @param args - Positional arguments; first argument is the `.nitpicker` file path
+ * @param flags - Parsed CLI flags from the `report` command
+ */
+export declare function report(args: string[], flags: ReportFlags): Promise<void>;
+export {};

package/lib/commands/report.js

@@ -0,0 +1,62 @@
+import { report as runReport } from '@nitpicker/report-google-sheets';
+/**
+ * Command definition for the `report` sub-command.
+ * @see {@link report} for the main entry point
+ */
+export const commandDef = {
+    desc: 'Generate a Google Sheets report',
+    flags: {
+        sheet: {
+            shortFlag: 'S',
+            type: 'string',
+            isRequired: true,
+            desc: 'Google Sheets URL',
+        },
+        credentials: {
+            shortFlag: 'C',
+            type: 'string',
+            default: './credentials.json',
+            desc: 'Path to credentials file (keep this file secure and out of version control)',
+        },
+        config: {
+            shortFlag: 'c',
+            type: 'string',
+            desc: 'Path to config file',
+        },
+        limit: {
+            shortFlag: 'l',
+            type: 'number',
+            default: 100_000,
+            desc: 'Limit number of rows',
+        },
+    },
+};
+/**
+ * Main entry point for the `report` CLI command.
+ *
+ * Reads a `.nitpicker` archive and generates a Google Sheets report
+ * by delegating to `@nitpicker/report-google-sheets`. Requires a Google
+ * Sheets URL and a service account credentials file.
+ * @param args - Positional arguments; first argument is the `.nitpicker` file path
+ * @param flags - Parsed CLI flags from the `report` command
+ */
+export async function report(args, flags) {
+    const filePath = args[0];
+    if (!filePath) {
+        return;
+    }
+    const sheetUrl = flags.sheet;
+    if (!sheetUrl) {
+        return;
+    }
+    const credentialFilePath = flags.credentials;
+    const configFilePath = flags.config || null;
+    const limit = flags.limit;
+    await runReport({
+        filePath,
+        sheetUrl,
+        credentialFilePath,
+        configPath: configFilePath,
+        limit,
+    });
+}

package/lib/crawl/debug.d.ts

@@ -0,0 +1,6 @@
+import debug from 'debug';
+export declare const log: debug.Debugger;
+/**
+ *
+ */
+export declare function verbosely(): void;

package/lib/crawl/debug.js

@@ -0,0 +1,10 @@
+import debug from 'debug';
+export const log = debug('Nitpicker').extend('CLI');
+/**
+ *
+ */
+export function verbosely() {
+    if (!debug.enabled('Nitpicker')) {
+        debug.enable('Nitpicker*,-Nitpicker:Crawler:Deal,-Nitpicker:Scraper:DOM:Details:*,-Nitpicker:Scraper:Resource:*');
+    }
+}

package/lib/crawl/diff.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Compares two `.nitpicker` archives and writes their URL lists to `a.txt` and `b.txt`.
+ *
+ * Extracts active internal HTML pages (2xx/3xx status) from both archives,
+ * sorts them in natural URL order, and writes to the current working directory.
+ * The output files can then be compared using standard diff tools.
+ * @param a - File path to the first `.nitpicker` archive
+ * @param b - File path to the second `.nitpicker` archive
+ */
+export declare function diff(a: string, b: string): Promise<void>;

package/lib/crawl/diff.js

@@ -0,0 +1,38 @@
+import fs from 'node:fs/promises';
+import { sortUrl } from '@d-zero/shared/sort-url';
+import { Archive } from '@nitpicker/crawler';
+/**
+ * Compares two `.nitpicker` archives and writes their URL lists to `a.txt` and `b.txt`.
+ *
+ * Extracts active internal HTML pages (2xx/3xx status) from both archives,
+ * sorts them in natural URL order, and writes to the current working directory.
+ * The output files can then be compared using standard diff tools.
+ * @param a - File path to the first `.nitpicker` archive
+ * @param b - File path to the second `.nitpicker` archive
+ */
+export async function diff(a, b) {
+    const archiveA = await Archive.open({ filePath: a });
+    const archiveB = await Archive.open({ filePath: b });
+    const pagesA = await archiveA.getPages();
+    const pagesB = await archiveB.getPages();
+    const listA = pagesA.filter(isActive).map((page) => page.url.withoutHashAndAuth);
+    const listB = pagesB.filter(isActive).map((page) => page.url.withoutHashAndAuth);
+    const sortedA = sortUrl(listA).map((url) => url.withoutHashAndAuth);
+    const sortedB = sortUrl(listB).map((url) => url.withoutHashAndAuth);
+    await fs.writeFile('a.txt', sortedA.join('\n'), 'utf8');
+    await fs.writeFile('b.txt', sortedB.join('\n'), 'utf8');
+    await archiveA.close();
+    await archiveB.close();
+}
+/**
+ * Filters for active internal HTML pages (status 200-399, non-external).
+ * @param page - The page to check
+ * @returns `true` if the page is an active internal HTML page
+ */
+function isActive(page) {
+    return (page.isPage() &&
+        !page.isExternal &&
+        page.status &&
+        page.status >= 200 &&
+        page.status < 400);
+}

package/lib/crawl/event-assignments.d.ts

@@ -0,0 +1,17 @@
+import type { CrawlerOrchestrator } from '@nitpicker/crawler';
+type LogType = 'verbose' | 'normal' | 'silent';
+/**
+ * Registers event listeners on the CrawlerOrchestrator for CLI progress display.
+ *
+ * Outputs the initial configuration summary to stderr, then listens for
+ * `error`, `writeFileStart`, and `writeFileEnd` events. Returns a Promise
+ * that resolves when the archive file has been written or rejects on error.
+ *
+ * WHY stderr: The crawl progress output is informational and should not
+ * interfere with stdout, which may be piped to other tools.
+ * @param orchestrator - The CrawlerOrchestrator to listen on
+ * @param initialLog - Lines to display at the start (URL + config summary)
+ * @param logType - Verbosity level; `'silent'` suppresses all output
+ */
+export declare function eventAssignments(orchestrator: CrawlerOrchestrator, initialLog: string[], logType: LogType): Promise<void>;
+export {};

package/lib/crawl/event-assignments.js

@@ -0,0 +1,33 @@
+import c from 'ansi-colors';
+/**
+ * Registers event listeners on the CrawlerOrchestrator for CLI progress display.
+ *
+ * Outputs the initial configuration summary to stderr, then listens for
+ * `error`, `writeFileStart`, and `writeFileEnd` events. Returns a Promise
+ * that resolves when the archive file has been written or rejects on error.
+ *
+ * WHY stderr: The crawl progress output is informational and should not
+ * interfere with stdout, which may be piped to other tools.
+ * @param orchestrator - The CrawlerOrchestrator to listen on
+ * @param initialLog - Lines to display at the start (URL + config summary)
+ * @param logType - Verbosity level; `'silent'` suppresses all output
+ */
+export async function eventAssignments(orchestrator, initialLog, logType) {
+    if (logType === 'silent') {
+        return;
+    }
+    return new Promise((resolve, reject) => {
+        const [firstLine, ...restLines] = initialLog;
+        process.stderr.write([c.bold(firstLine ?? ''), ...restLines.map((l) => c.dim(l))].join('\n') + '\n');
+        orchestrator.on('error', (error) => {
+            reject(error);
+        });
+        orchestrator.on('writeFileStart', ({ filePath }) => {
+            process.stderr.write(`📥 Writing to: ${c.cyan(filePath)}\n`);
+        });
+        orchestrator.on('writeFileEnd', ({ filePath }) => {
+            process.stderr.write(`📥 Done: ${c.cyan(filePath)}\n`);
+            resolve();
+        });
+    });
+}

package/lib/index.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/index.js

@@ -0,0 +1,28 @@
+import { parseCli } from '@d-zero/roar';
+import { analyze, commandDef as analyzeDef } from './commands/analyze.js';
+import { crawl, commandDef as crawlDef } from './commands/crawl.js';
+import { report, commandDef as reportDef } from './commands/report.js';
+process.title = 'Nitpicker CLI';
+const cli = parseCli({
+    name: 'nitpicker',
+    commands: {
+        crawl: crawlDef,
+        analyze: analyzeDef,
+        report: reportDef,
+    },
+    onError: () => true,
+});
+switch (cli.command) {
+    case 'crawl': {
+        await crawl(cli.args, cli.flags);
+        break;
+    }
+    case 'analyze': {
+        await analyze(cli.args, cli.flags);
+        break;
+    }
+    case 'report': {
+        await report(cli.args, cli.flags);
+        break;
+    }
+}

package/lib/log.d.ts

@@ -0,0 +1,8 @@
+import type { Nitpicker } from '@nitpicker/core';
+/**
+ *
+ * @param nitpicker
+ * @param startLog
+ * @param verbose
+ */
+export declare function log(nitpicker: Nitpicker, startLog: string[], verbose: boolean): Promise<never>;

package/lib/log.js

@@ -0,0 +1,45 @@
+import Logger from './logger.js';
+/**
+ *
+ * @param nitpicker
+ * @param startLog
+ * @param verbose
+ */
+export async function log(nitpicker, startLog, verbose) {
+    return new Promise((_, reject) => {
+        const plugins = new Map();
+        const logger = new Logger({
+            dots: ['.  ', '.. ', '...'],
+            propeller: ['\\', '|', '/', '-'],
+        });
+        if (verbose) {
+            logger.verbosely();
+        }
+        if (startLog) {
+            logger.write(startLog);
+            logger.close();
+        }
+        nitpicker.on('error', (err) => {
+            const error = err.error || new Error(err.message);
+            reject(error);
+        });
+        nitpicker.on('audit', (plugin) => {
+            if (verbose) {
+                logger.write(`${plugin.module}: ${plugin.log}`);
+                return;
+            }
+            plugins.set(plugin.module, plugin.log);
+            const logs = [...plugins].map(([mod, log]) => {
+                const content = log === 'Finished' ? '✨ Finished' : `%propeller% ${log}`;
+                return `    🔬 ${mod}: ${content}`;
+            });
+            const heading = '   Audit through%dots%';
+            logger.write([heading, ...logs]);
+        });
+        nitpicker.on('writeFile', ({ filePath }) => {
+            logger.close();
+            logger.write(`  📥 Write file: ${filePath}`);
+            logger.close();
+        });
+    });
+}

package/lib/logger.d.ts

@@ -0,0 +1,10 @@
+export default class Logger {
+    verbose: boolean;
+    constructor(_options?: {
+        dots?: string[];
+        propeller?: string[];
+    });
+    verbosely(): void;
+    write(lines: string | string[]): void;
+    close(): void;
+}

package/lib/logger.js

@@ -0,0 +1,16 @@
+export default class Logger {
+    verbose = false;
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    constructor(_options) { }
+    verbosely() {
+        this.verbose = true;
+    }
+    write(lines) {
+        const output = Array.isArray(lines) ? lines : [lines];
+        for (const line of output) {
+            const cleaned = line.replaceAll(/%dots%|%propeller%/g, '');
+            process.stderr.write(cleaned + '\n');
+        }
+    }
+    close() { }
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+	"name": "@nitpicker/cli",
+	"version": "0.4.2",
+	"description": "Web site crawler, analyzer, and reporter CLI",
+	"author": "D-ZERO",
+	"license": "Apache-2.0",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/d-zero-dev/nitpicker.git",
+		"directory": "packages/@nitpicker/cli"
+	},
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": "./lib/index.js",
+			"types": "./lib/index.d.ts"
+		}
+	},
+	"bin": {
+		"nitpicker": "./bin/nitpicker.js"
+	},
+	"scripts": {
+		"build": "tsc",
+		"clean": "tsc --build --clean"
+	},
+	"dependencies": {
+		"@d-zero/dealer": "1.6.3",
+		"@d-zero/readtext": "1.1.19",
+		"@d-zero/roar": "2.0.0",
+		"@d-zero/shared": "0.20.0",
+		"@nitpicker/core": "0.4.2",
+		"@nitpicker/crawler": "0.4.2",
+		"@nitpicker/report-google-sheets": "0.4.2",
+		"ansi-colors": "4.1.3",
+		"debug": "4.4.3",
+		"enquirer": "2.4.1"
+	},
+	"devDependencies": {
+		"@types/debug": "4.1.12"
+	},
+	"gitHead": "14066d0e7b9e652ddd7a5abb5f20bba682f09c0c"
+}