@nitpicker/cli 0.4.2

package/src/analyze/log.spec.ts

@@ -0,0 +1,88 @@
+import { afterEach, describe, it, expect, vi, beforeEach } from 'vitest';
+
+import { log } from './log.js';
+
+describe('log', () => {
+	let consoleSpy: ReturnType<typeof vi.spyOn>;
+	let consoleErrorSpy: ReturnType<typeof vi.spyOn>;
+	let mockNitpicker: {
+		handlers: Map<string, ((...args: never[]) => void)[]>;
+		on: ReturnType<typeof vi.fn>;
+		emit: (event: string, data: unknown) => void;
+	};
+
+	beforeEach(() => {
+		consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+		consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+		mockNitpicker = {
+			handlers: new Map(),
+			on: vi.fn((event: string, handler: (...args: never[]) => void) => {
+				const handlers = mockNitpicker.handlers.get(event) ?? [];
+				handlers.push(handler);
+				mockNitpicker.handlers.set(event, handlers);
+			}),
+			emit(event: string, data: unknown) {
+				const handlers = this.handlers.get(event) ?? [];
+				for (const handler of handlers) {
+					handler(data as never);
+				}
+			},
+		};
+	});
+
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it('prints all start log lines', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, ['line1', 'line2', 'line3']);
+
+		expect(consoleSpy).toHaveBeenCalledTimes(3);
+		expect(consoleSpy).toHaveBeenNthCalledWith(1, 'line1');
+		expect(consoleSpy).toHaveBeenNthCalledWith(2, 'line2');
+		expect(consoleSpy).toHaveBeenNthCalledWith(3, 'line3');
+	});
+
+	it('registers writeFile event handler', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, []);
+
+		expect(mockNitpicker.on).toHaveBeenCalledWith('writeFile', expect.any(Function));
+	});
+
+	it('registers error event handler', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, []);
+
+		expect(mockNitpicker.on).toHaveBeenCalledWith('error', expect.any(Function));
+	});
+
+	it('logs write file path on writeFile event', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, []);
+		consoleSpy.mockClear();
+
+		mockNitpicker.emit('writeFile', { filePath: '/path/to/output.nitpicker' });
+
+		expect(consoleSpy).toHaveBeenCalledWith('  📥 Write file: /path/to/output.nitpicker');
+	});
+
+	it('logs error message on error event', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, []);
+
+		mockNitpicker.emit('error', { message: 'something went wrong' });
+
+		expect(consoleErrorSpy).toHaveBeenCalledWith('something went wrong');
+	});
+
+	it('handles empty start log array', () => {
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		log(mockNitpicker as any, []);
+
+		// No console.log calls for start lines (only event handlers registered)
+		expect(consoleSpy).not.toHaveBeenCalled();
+	});
+});

package/src/analyze/log.ts

@@ -0,0 +1,28 @@
+import type { Nitpicker } from '@nitpicker/core';
+
+/**
+ * Registers event listeners on a Nitpicker instance for analyze output,
+ * and prints the initial summary lines.
+ *
+ * Outputs the start log lines (site URL, file path), then listens for
+ * `writeFile` and `error` events to provide CLI feedback.
+ * Progress display is handled by Lanes in the calling code.
+ * @param nitpicker - The Nitpicker instance to listen on
+ * @param startLog - Lines to display at the start (URL + file path summary)
+ */
+export function log(nitpicker: Nitpicker, startLog: string[]): void {
+	for (const line of startLog) {
+		// eslint-disable-next-line no-console
+		console.log(line);
+	}
+
+	nitpicker.on('writeFile', ({ filePath }) => {
+		// eslint-disable-next-line no-console
+		console.log(`  📥 Write file: ${filePath}`);
+	});
+
+	nitpicker.on('error', (err) => {
+		// eslint-disable-next-line no-console
+		console.error(err.message);
+	});
+}

package/src/commands/analyze.ts

@@ -0,0 +1,107 @@
+import type { CommandDef, InferFlags } from '@d-zero/roar';
+
+import path from 'node:path';
+
+import { Lanes } from '@d-zero/dealer';
+import { Nitpicker, readPluginLabels } from '@nitpicker/core';
+import enquirer from 'enquirer';
+
+import { log } from '../analyze/log.js';
+
+const { prompt } = enquirer;
+
+/**
+ * Command definition for the `analyze` sub-command.
+ * @see {@link analyze} for the main entry point
+ */
+export const commandDef = {
+	desc: 'Analyze a .nitpicker archive',
+	flags: {
+		all: {
+			type: 'boolean',
+			desc: 'Run all analysis plugins',
+		},
+		verbose: {
+			type: 'boolean',
+			desc: 'Output logs verbosely',
+		},
+	},
+} as const satisfies CommandDef;
+
+type AnalyzeFlags = InferFlags<typeof commandDef.flags>;
+
+/**
+ * Main entry point for the `analyze` CLI command.
+ *
+ * Opens a `.nitpicker` archive, loads the configured analyze plugins,
+ * presents an interactive multi-select prompt (unless `--all` is specified),
+ * runs the selected plugins with per-plugin Lanes progress display, and
+ * writes results back to the archive.
+ *
+ * WHY enquirer prompt: Allows users to selectively run expensive plugins
+ * (e.g. Lighthouse) without re-running everything. The `--all` flag
+ * bypasses the prompt for CI/automation use cases.
+ * @param args - Positional arguments; first argument is the `.nitpicker` file path
+ * @param flags - Parsed CLI flags from the `analyze` command
+ */
+export async function analyze(args: string[], flags: AnalyzeFlags) {
+	const filePath = args[0];
+
+	if (!filePath) {
+		return;
+	}
+
+	const isTTY = process.stdout.isTTY;
+
+	const absFilePath = path.isAbsolute(filePath)
+		? filePath
+		: path.resolve(process.cwd(), filePath);
+	// eslint-disable-next-line no-console
+	console.log(`  📦 Extracting archive: ${absFilePath}`);
+	const nitpicker = await Nitpicker.open(absFilePath);
+
+	const config = await nitpicker.getConfig();
+	const plugins = config.analyze || [];
+	const pluginNameList = plugins.map((plugin) => plugin.name);
+
+	if (pluginNameList.length === 0) {
+		// eslint-disable-next-line no-console
+		console.error(
+			'No analyze plugins found. Install @nitpicker/analyze-* packages or configure them in .nitpickerrc.',
+		);
+		return;
+	}
+
+	let filter: string[] | undefined;
+
+	if (!flags.all) {
+		const labels = await readPluginLabels(plugins);
+		const choices = plugins.map((plugin) => ({
+			name: plugin.name,
+			message: labels.get(plugin.name) || plugin.name,
+		}));
+		const res = await prompt<{ filter: string[] }>([
+			{
+				message: 'What do you analyze?',
+				name: 'filter',
+				type: 'multiselect',
+				choices,
+			},
+		]);
+		filter = res.filter;
+	}
+
+	const siteUrl = (await nitpicker.archive.getUrl()) || '<Unknown URL>';
+
+	log(nitpicker, [`🥢 ${siteUrl} (${filePath})`, `  📤 Read file: ${absFilePath}`]);
+
+	const lanes = new Lanes({ verbose: !isTTY, indent: '  ' });
+	try {
+		await nitpicker.analyze(filter, { lanes, verbose: !isTTY });
+	} finally {
+		lanes.close();
+	}
+
+	await nitpicker.write();
+	await nitpicker.archive.close();
+}

package/src/commands/crawl.ts

@@ -0,0 +1,303 @@
+import type { CommandDef, InferFlags } from '@d-zero/roar';
+import type { Config, CrawlerError } from '@nitpicker/crawler';
+
+import path from 'node:path';
+
+import { readList } from '@d-zero/readtext/list';
+import { CrawlerOrchestrator } from '@nitpicker/crawler';
+
+import { log, verbosely } from '../crawl/debug.js';
+import { diff } from '../crawl/diff.js';
+import { eventAssignments } from '../crawl/event-assignments.js';
+
+/**
+ * Command definition for the `crawl` sub-command.
+ * Defines all CLI flags with their types, defaults, and descriptions.
+ * @see {@link crawl} for the main entry point that dispatches to startCrawl/resumeCrawl/diff
+ */
+export const commandDef = {
+	desc: 'Crawl a website',
+	flags: {
+		resume: {
+			type: 'string',
+			shortFlag: 'R',
+			desc: 'Resume crawling from a stub file',
+		},
+		interval: {
+			type: 'number',
+			shortFlag: 'I',
+			desc: 'An interval time on request when crawles',
+		},
+		image: {
+			type: 'boolean',
+			default: true,
+			desc: 'Getting images (use --no-image to disable)',
+		},
+		fetchExternal: {
+			type: 'boolean',
+			default: true,
+			desc: 'Fetch external links (use --no-fetch-external to disable)',
+		},
+		parallels: {
+			type: 'number',
+			shortFlag: 'P',
+			desc: 'Number of parallel scraping',
+		},
+		recursive: {
+			type: 'boolean',
+			default: true,
+			desc: 'Recursive crawling (use --no-recursive to disable)',
+		},
+		scope: {
+			type: 'string',
+			desc: 'Set hosts and URLs as scope',
+		},
+		exclude: {
+			type: 'string',
+			isMultiple: true,
+			desc: 'Excluding page URL path (glob pattern)',
+		},
+		excludeKeyword: {
+			type: 'string',
+			isMultiple: true,
+			desc: 'Exclude keyword in document of page',
+		},
+		excludeUrl: {
+			type: 'string',
+			isMultiple: true,
+			desc: 'Exclude external URL prefix',
+		},
+		disableQueries: {
+			type: 'boolean',
+			shortFlag: 'Q',
+			desc: 'Disable queries that the URL has',
+		},
+		imageFileSizeThreshold: {
+			type: 'number',
+			desc: 'Image file size threshold',
+		},
+		single: {
+			type: 'boolean',
+			desc: 'Single page mode',
+		},
+		maxExcludedDepth: {
+			type: 'number',
+			desc: 'Avoid crawling depths above a set number',
+		},
+		retry: {
+			type: 'number',
+			default: 3,
+			desc: 'Number of retry attempts per URL on scrape failure',
+		},
+		list: {
+			type: 'string',
+			isMultiple: true,
+			desc: 'Running only each page from the list',
+		},
+		listFile: {
+			type: 'string',
+			desc: 'Running only each page from the list file',
+		},
+		userAgent: {
+			type: 'string',
+			desc: 'Custom User-Agent string for HTTP requests',
+		},
+		ignoreRobots: {
+			type: 'boolean',
+			desc: 'Ignore robots.txt restrictions (use responsibly)',
+		},
+		verbose: {
+			type: 'boolean',
+			desc: 'Output verbose log to standard out',
+		},
+		silent: {
+			type: 'boolean',
+			desc: 'No output log to standard out',
+		},
+		diff: {
+			type: 'boolean',
+			desc: 'Diff mode',
+		},
+	},
+} as const satisfies CommandDef;
+
+type CrawlFlags = InferFlags<typeof commandDef.flags>;
+
+type LogType = 'verbose' | 'normal' | 'silent';
+
+/**
+ * Sets up signal handlers for graceful shutdown and starts event logging.
+ *
+ * Registers SIGINT/SIGBREAK/SIGHUP/SIGABRT handlers that kill zombie
+ * Chromium processes before exiting, then delegates to {@link eventAssignments}
+ * for progress output.
+ * @param trigger - Display label for the crawl (URL or stub file path)
+ * @param orchestrator - The initialized CrawlerOrchestrator instance
+ * @param config - The resolved archive configuration
+ * @param logType - Output verbosity level
+ */
+function run(
+	trigger: string,
+	orchestrator: CrawlerOrchestrator,
+	config: Config,
+	logType: LogType,
+) {
+	const killed = () => {
+		orchestrator.garbageCollect();
+		process.exit();
+	};
+	process.on('SIGINT', killed);
+	process.on('SIGBREAK', killed);
+	process.on('SIGHUP', killed);
+	process.on('SIGABRT', killed);
+
+	const head = [
+		`🐳 ${trigger} (New scraping)`,
+		...Object.entries(config).map(([key, value]) => `  ${key}: ${value}`),
+	];
+	return eventAssignments(orchestrator, head, logType);
+}
+
+/**
+ * Starts a fresh crawl session for the given URLs.
+ *
+ * Creates a CrawlerOrchestrator, runs the crawl, writes the archive,
+ * and cleans up browser processes. Exits with code 1 if errors occurred.
+ * @param siteUrl - One or more root URLs to crawl
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+async function startCrawl(siteUrl: string[], flags: CrawlFlags) {
+	const errStack: (CrawlerError | Error)[] = [];
+
+	const isList = !!flags.list?.length;
+	const orchestrator = await CrawlerOrchestrator.crawling(
+		siteUrl,
+		{
+			...flags,
+			scope: flags.scope?.split(',').map((s) => s.trim()),
+			list: isList,
+			recursive: isList ? false : flags.recursive,
+		},
+		(orchestrator, config) => {
+			run(
+				config.baseUrl,
+				orchestrator,
+				config,
+				flags.verbose ? 'verbose' : flags.silent ? 'silent' : 'normal',
+			).catch((error) => errStack.push(error));
+		},
+	);
+
+	await orchestrator.write();
+
+	orchestrator.garbageCollect();
+
+	if (errStack.length > 0) {
+		formatCrawlErrors(errStack);
+		process.exit(1);
+	}
+}
+
+/**
+ * Resumes a previously interrupted crawl from a stub file (temporary directory).
+ *
+ * Restores the crawl state from the archive, applies any flag overrides,
+ * and continues crawling from where the previous session left off.
+ * @param stubFilePath - Path to the stub file or temporary directory to resume from
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+async function resumeCrawl(stubFilePath: string, flags: CrawlFlags) {
+	const errStack: (CrawlerError | Error)[] = [];
+	const absFilePath = path.isAbsolute(stubFilePath)
+		? stubFilePath
+		: path.resolve(process.cwd(), stubFilePath);
+
+	const orchestrator = await CrawlerOrchestrator.resume(
+		absFilePath,
+		{
+			...flags,
+			scope: flags.scope?.split(',').map((s) => s.trim()),
+			list: false,
+		},
+		(orchestrator, config) => {
+			run(
+				stubFilePath,
+				orchestrator,
+				config,
+				flags.verbose ? 'verbose' : flags.silent ? 'silent' : 'normal',
+			).catch((error) => errStack.push(error));
+		},
+	);
+
+	await orchestrator.write();
+
+	orchestrator.garbageCollect();
+
+	if (errStack.length > 0) {
+		formatCrawlErrors(errStack);
+		process.exit(1);
+	}
+}
+
+/**
+ * Main entry point for the `crawl` CLI command.
+ *
+ * Dispatches to one of four modes based on the flags:
+ * 1. `--diff` mode: Compares two archive files and outputs URL lists
+ * 2. `--resume` mode: Resumes a previously interrupted crawl
+ * 3. `--list-file` / `--list` mode: Crawls a pre-defined URL list (non-recursive)
+ * 4. Default mode: Crawls from a single root URL
+ * @param args - Positional arguments (typically one or two URLs/file paths)
+ * @param flags - Parsed CLI flags from the `crawl` command
+ */
+export async function crawl(args: string[], flags: CrawlFlags) {
+	if (flags.verbose && !flags.silent) {
+		verbosely();
+	}
+
+	log('Options: %O', flags);
+
+	if (flags.diff) {
+		const a = args[0];
+		const b = args[1];
+		if (!a || !b) {
+			throw new Error('Please provide two file paths to compare');
+		}
+		await diff(a, b);
+		return;
+	}
+
+	if (flags.resume) {
+		await resumeCrawl(flags.resume, flags);
+		return;
+	}
+
+	if (flags.listFile) {
+		const list = await readList(path.resolve(process.cwd(), flags.listFile));
+		flags.list = list;
+		await startCrawl(list, flags);
+		return;
+	}
+
+	if (flags.list && flags.list.length > 0) {
+		const pageList = [...flags.list, ...args];
+		await startCrawl(pageList, flags);
+		return;
+	}
+
+	const siteUrl = args[0];
+
+	if (siteUrl) {
+		await startCrawl([siteUrl], flags);
+		return;
+	}
+}
+
+/**
+ * Prints a summary of errors that occurred during crawling to stderr.
+ * @param errStack - Array of errors collected during the crawl session
+ */
+function formatCrawlErrors(errStack: (CrawlerError | Error)[]) {
+	// eslint-disable-next-line no-console
+	console.error(`\nCompleted with ${errStack.length} error(s).`);
+}

package/src/commands/report.ts

@@ -0,0 +1,73 @@
+import type { CommandDef, InferFlags } from '@d-zero/roar';
+
+import { report as runReport } from '@nitpicker/report-google-sheets';
+
+/**
+ * Command definition for the `report` sub-command.
+ * @see {@link report} for the main entry point
+ */
+export const commandDef = {
+	desc: 'Generate a Google Sheets report',
+	flags: {
+		sheet: {
+			shortFlag: 'S',
+			type: 'string',
+			isRequired: true,
+			desc: 'Google Sheets URL',
+		},
+		credentials: {
+			shortFlag: 'C',
+			type: 'string',
+			default: './credentials.json',
+			desc: 'Path to credentials file (keep this file secure and out of version control)',
+		},
+		config: {
+			shortFlag: 'c',
+			type: 'string',
+			desc: 'Path to config file',
+		},
+		limit: {
+			shortFlag: 'l',
+			type: 'number',
+			default: 100_000,
+			desc: 'Limit number of rows',
+		},
+	},
+} as const satisfies CommandDef;
+
+type ReportFlags = InferFlags<typeof commandDef.flags>;
+
+/**
+ * Main entry point for the `report` CLI command.
+ *
+ * Reads a `.nitpicker` archive and generates a Google Sheets report
+ * by delegating to `@nitpicker/report-google-sheets`. Requires a Google
+ * Sheets URL and a service account credentials file.
+ * @param args - Positional arguments; first argument is the `.nitpicker` file path
+ * @param flags - Parsed CLI flags from the `report` command
+ */
+export async function report(args: string[], flags: ReportFlags) {
+	const filePath = args[0];
+
+	if (!filePath) {
+		return;
+	}
+
+	const sheetUrl = flags.sheet;
+
+	if (!sheetUrl) {
+		return;
+	}
+
+	const credentialFilePath = flags.credentials;
+	const configFilePath = flags.config || null;
+	const limit = flags.limit;
+
+	await runReport({
+		filePath,
+		sheetUrl,
+		credentialFilePath,
+		configPath: configFilePath,
+		limit,
+	});
+}

package/src/crawl/debug.ts

@@ -0,0 +1,14 @@
+import debug from 'debug';
+
+export const log = debug('Nitpicker').extend('CLI');
+
+/**
+ *
+ */
+export function verbosely() {
+	if (!debug.enabled('Nitpicker')) {
+		debug.enable(
+			'Nitpicker*,-Nitpicker:Crawler:Deal,-Nitpicker:Scraper:DOM:Details:*,-Nitpicker:Scraper:Resource:*',
+		);
+	}
+}

package/src/crawl/diff.ts

@@ -0,0 +1,48 @@
+import type { Page } from '@nitpicker/crawler';
+
+import fs from 'node:fs/promises';
+
+import { sortUrl } from '@d-zero/shared/sort-url';
+import { Archive } from '@nitpicker/crawler';
+
+/**
+ * Compares two `.nitpicker` archives and writes their URL lists to `a.txt` and `b.txt`.
+ *
+ * Extracts active internal HTML pages (2xx/3xx status) from both archives,
+ * sorts them in natural URL order, and writes to the current working directory.
+ * The output files can then be compared using standard diff tools.
+ * @param a - File path to the first `.nitpicker` archive
+ * @param b - File path to the second `.nitpicker` archive
+ */
+export async function diff(a: string, b: string) {
+	const archiveA = await Archive.open({ filePath: a });
+	const archiveB = await Archive.open({ filePath: b });
+	const pagesA = await archiveA.getPages();
+	const pagesB = await archiveB.getPages();
+	const listA = pagesA.filter(isActive).map((page) => page.url.withoutHashAndAuth);
+	const listB = pagesB.filter(isActive).map((page) => page.url.withoutHashAndAuth);
+
+	const sortedA = sortUrl(listA).map((url) => url.withoutHashAndAuth);
+	const sortedB = sortUrl(listB).map((url) => url.withoutHashAndAuth);
+
+	await fs.writeFile('a.txt', sortedA.join('\n'), 'utf8');
+	await fs.writeFile('b.txt', sortedB.join('\n'), 'utf8');
+
+	await archiveA.close();
+	await archiveB.close();
+}
+
+/**
+ * Filters for active internal HTML pages (status 200-399, non-external).
+ * @param page - The page to check
+ * @returns `true` if the page is an active internal HTML page
+ */
+function isActive(page: Page) {
+	return (
+		page.isPage() &&
+		!page.isExternal &&
+		page.status &&
+		page.status >= 200 &&
+		page.status < 400
+	);
+}