@nitpicker/report-google-sheets 0.4.1

package/lib/data/create-resources.js

@@ -0,0 +1,45 @@
+import { reportLog } from '../debug.js';
+import { createCellData } from '../sheets/create-cell-data.js';
+import { defaultCellFormat } from '../sheets/default-cell-format.js';
+/**
+ * Creates the "Resources" sheet configuration.
+ *
+ * Lists all network resources (CSS, JS, images, fonts, etc.) discovered
+ * during crawling, with one row per unique resource URL. Each row includes
+ * HTTP status, content type, size, and the number of pages that reference
+ * the resource (with URLs listed in the cell note).
+ *
+ * Uses `eachResource` (Phase 3) to iterate over the archive's resource
+ * table rather than page-by-page iteration.
+ */
+export const createResources = () => {
+    return {
+        name: 'Resources',
+        createHeaders() {
+            return [
+                'URL',
+                'Status Code',
+                'Status Text',
+                'Content Type',
+                'Content Length',
+                'Referrers',
+            ];
+        },
+        async eachResource(resource) {
+            reportLog(`Read: Resource referrers (Search: ${resource.url})`);
+            const referrers = await resource.getReferrers();
+            const data = [
+                createCellData({ value: resource.url }, defaultCellFormat),
+                createCellData({ value: resource.status }, defaultCellFormat),
+                createCellData({ value: resource.statusText }, defaultCellFormat),
+                createCellData({ value: resource.contentType }, defaultCellFormat),
+                createCellData({ value: resource.contentLength }, defaultCellFormat),
+                createCellData({
+                    value: `${referrers.length} pages`,
+                    note: referrers.join('\n'),
+                }, defaultCellFormat),
+            ];
+            return [data];
+        },
+    };
+};

package/lib/data/create-violations.d.ts

@@ -0,0 +1,12 @@
+import type { CreateSheet } from '../sheets/types.js';
+/**
+ * Creates the "Violations" sheet configuration.
+ *
+ * Aggregates all violations from analyze plugin reports into a single
+ * flat table. Uses `addRows` (Phase 4) rather than `eachPage` because
+ * violation data comes from the report objects, not from per-page
+ * archive iteration. This means it runs in parallel with page/resource
+ * processing phases.
+ * @param reports
+ */
+export declare const createViolations: CreateSheet;

package/lib/data/create-violations.js

@@ -0,0 +1,42 @@
+import { pLog } from '../debug.js';
+import { createCellData } from '../sheets/create-cell-data.js';
+import { defaultCellFormat } from '../sheets/default-cell-format.js';
+const log = pLog.extend('Violations');
+/**
+ * Creates the "Violations" sheet configuration.
+ *
+ * Aggregates all violations from analyze plugin reports into a single
+ * flat table. Uses `addRows` (Phase 4) rather than `eachPage` because
+ * violation data comes from the report objects, not from per-page
+ * archive iteration. This means it runs in parallel with page/resource
+ * processing phases.
+ * @param reports
+ */
+export const createViolations = (reports) => {
+    return {
+        name: 'Violations',
+        createHeaders() {
+            return ['Validator', 'Severity', 'Rule', 'Code', 'Message', 'URL'];
+        },
+        addRows: () => {
+            const data = [];
+            for (const report of reports) {
+                if (!report.violations) {
+                    continue;
+                }
+                log('From %s', report.name);
+                for (const violation of report.violations) {
+                    data.push([
+                        createCellData({ value: violation.validator }, defaultCellFormat),
+                        createCellData({ value: violation.severity }, defaultCellFormat),
+                        createCellData({ value: violation.rule }, defaultCellFormat),
+                        createCellData({ value: violation.code }, defaultCellFormat),
+                        createCellData({ value: violation.message }, defaultCellFormat),
+                        createCellData({ value: violation.url }, defaultCellFormat),
+                    ]);
+                }
+            }
+            return data;
+        },
+    };
+};

package/lib/debug.d.ts

@@ -0,0 +1,6 @@
+import debug from 'debug';
+export declare const log: debug.Debugger;
+export declare const sheetLog: debug.Debugger;
+export declare const archiveLog: debug.Debugger;
+export declare const reportLog: debug.Debugger;
+export declare const pLog: debug.Debugger;

package/lib/debug.js

@@ -0,0 +1,6 @@
+import debug from 'debug';
+export const log = debug('Nitpicker').extend('GoogleSpreadsheet');
+export const sheetLog = log.extend('Sheet');
+export const archiveLog = log.extend('Archive');
+export const reportLog = log.extend('Report');
+export const pLog = log.extend('Plugin');

package/lib/index.d.ts

@@ -0,0 +1 @@
+export { report } from './report.js';

package/lib/index.js

@@ -0,0 +1 @@
+export { report } from './report.js';

package/lib/load-config.d.ts

@@ -0,0 +1,6 @@
+import type { ConfigJSON } from '@nitpicker/types';
+/**
+ *
+ * @param filePath
+ */
+export declare function loadConfig(filePath: string | null): Promise<ConfigJSON>;

package/lib/load-config.js

@@ -0,0 +1,14 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+/**
+ *
+ * @param filePath
+ */
+export async function loadConfig(filePath) {
+    if (!filePath) {
+        return {};
+    }
+    const absFilePath = path.isAbsolute(filePath) ? filePath : path.resolve(filePath);
+    const data = await fs.readFile(absFilePath, { encoding: 'utf8' });
+    return JSON.parse(data);
+}

package/lib/report.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Parameters for {@link report}.
+ */
+export interface ReportParams {
+    /** Path to the `.nitpicker` archive file. */
+    readonly filePath: string;
+    /** URL of the target Google Spreadsheet. */
+    readonly sheetUrl: string;
+    /** Path to the OAuth2 credentials JSON file. */
+    readonly credentialFilePath: string;
+    /** Path to the nitpicker config file, or `null` for defaults. */
+    readonly configPath: string | null;
+    /** Batch size for `getPagesWithRefs()` pagination (default: 100,000). */
+    readonly limit: number;
+}
+/**
+ * Generates a Google Sheets report from a `.nitpicker` archive file.
+ *
+ * This is the main entry point for the `nitpicker report` command.
+ * It orchestrates the full reporting pipeline:
+ *
+ * 1. Authenticates with Google Sheets API using OAuth2 credentials.
+ * 2. Opens the `.nitpicker` archive and loads its configuration.
+ * 3. Loads analyze plugin reports from the archive.
+ * 4. Presents an interactive multi-select prompt for the user to
+ *    choose which sheets to generate.
+ * 5. Delegates to `createSheets()` for phased data generation and upload.
+ *
+ * Rate limiting from the Google Sheets API (429 / 403) is handled
+ * gracefully via the `Sheets.onLog` callback, which displays a
+ * countdown timer in the terminal using the `Lanes` progress display.
+ * @param params - レポート生成に必要なパラメータ
+ * @example
+ * ```ts
+ * await report({
+ *   filePath: './output.nitpicker',
+ *   sheetUrl: 'https://docs.google.com/spreadsheets/d/xxx/edit',
+ *   credentialFilePath: './credentials.json',
+ *   configPath: './nitpicker.config.json',
+ *   limit: 100_000,
+ * });
+ * ```
+ */
+export declare function report(params: ReportParams): Promise<void>;

package/lib/report.js

@@ -0,0 +1,180 @@
+import { Lanes } from '@d-zero/dealer';
+import { authentication } from '@d-zero/google-auth';
+import { Sheets } from '@d-zero/google-sheets';
+import c from 'ansi-colors';
+import enquirer from 'enquirer';
+import { getArchive } from './archive.js';
+import { addToSummary } from './data/add-to-summary.js';
+import { createDiscrepancies } from './data/create-discrepancies.js';
+import { createImageList } from './data/create-image-list.js';
+import { createLinks } from './data/create-links.js';
+import { createPageList } from './data/create-page-list.js';
+import { createReferrersRelationalTable } from './data/create-referrers-relational-table.js';
+import { createResourcesRelationalTable } from './data/create-resources-relational-table.js';
+import { createResources } from './data/create-resources.js';
+import { createViolations } from './data/create-violations.js';
+import { archiveLog, log } from './debug.js';
+import { loadConfig } from './load-config.js';
+import { getPluginReports } from './reports/get-plugin-reports.js';
+import { createSheets } from './sheets/create-sheets.js';
+/**
+ * Generates a Google Sheets report from a `.nitpicker` archive file.
+ *
+ * This is the main entry point for the `nitpicker report` command.
+ * It orchestrates the full reporting pipeline:
+ *
+ * 1. Authenticates with Google Sheets API using OAuth2 credentials.
+ * 2. Opens the `.nitpicker` archive and loads its configuration.
+ * 3. Loads analyze plugin reports from the archive.
+ * 4. Presents an interactive multi-select prompt for the user to
+ *    choose which sheets to generate.
+ * 5. Delegates to `createSheets()` for phased data generation and upload.
+ *
+ * Rate limiting from the Google Sheets API (429 / 403) is handled
+ * gracefully via the `Sheets.onLog` callback, which displays a
+ * countdown timer in the terminal using the `Lanes` progress display.
+ * @param params - レポート生成に必要なパラメータ
+ * @example
+ * ```ts
+ * await report({
+ *   filePath: './output.nitpicker',
+ *   sheetUrl: 'https://docs.google.com/spreadsheets/d/xxx/edit',
+ *   credentialFilePath: './credentials.json',
+ *   configPath: './nitpicker.config.json',
+ *   limit: 100_000,
+ * });
+ * ```
+ */
+export async function report(params) {
+    const { filePath, sheetUrl, credentialFilePath, configPath, limit } = params;
+    log('Initialization');
+    const SCOPES = ['https://www.googleapis.com/auth/spreadsheets'];
+    log('Authenticating');
+    const auth = await authentication(credentialFilePath, SCOPES, {
+        tokenFilePath: 'token.json',
+    });
+    log('Authentication succeeded');
+    log('Opening archive: %s', filePath);
+    const archive = await getArchive(filePath);
+    log('Archive opened');
+    log('Loading config');
+    const config = await loadConfig(configPath);
+    log('Config loaded');
+    const plugins = config.plugins?.analyze
+        ? Object.keys(config.plugins.analyze)
+        : undefined;
+    log('Loaded plugins: %O', plugins);
+    log('Loading plugin reports');
+    const reports = await getPluginReports(archive /*plugins*/);
+    log('Plugin reports loaded: %d', reports.length);
+    const sheets = new Sheets(sheetUrl, auth);
+    log('Reporting starts');
+    const sheetNames = [
+        'Page List',
+        'Links',
+        'Resources',
+        'Images',
+        'Violations',
+        'Discrepancies',
+        'Summary',
+        'Referrers Relational Table',
+        'Resources Relational Table',
+    ];
+    log('Choice creating data');
+    const chosenSheets = await enquirer
+        .prompt([
+        {
+            message: 'What do you report?',
+            name: 'sheetName',
+            type: 'multiselect',
+            choices: sheetNames,
+        },
+    ])
+        .catch(() => {
+        // enquirer v2.4.1: Ctrl+C 後に readline を二重 close して
+        // ERR_USE_AFTER_CLOSE が unhandled rejection になるため、
+        // 即座に終了して回避する
+        process.exit(0);
+    });
+    if (!chosenSheets) {
+        log('Choice creating data');
+        archiveLog('Closes file');
+        await archive.close();
+        return;
+    }
+    log('Chosen sheets: %O', chosenSheets.sheetName);
+    const createSheetList = [];
+    if (chosenSheets.sheetName.includes('Page List')) {
+        createSheetList.push(createPageList);
+    }
+    if (chosenSheets.sheetName.includes('Links')) {
+        createSheetList.push(createLinks);
+    }
+    if (chosenSheets.sheetName.includes('Discrepancies')) {
+        createSheetList.push(createDiscrepancies);
+    }
+    if (chosenSheets.sheetName.includes('Violations')) {
+        createSheetList.push(createViolations);
+    }
+    if (chosenSheets.sheetName.includes('Referrers Relational Table')) {
+        createSheetList.push(createReferrersRelationalTable);
+    }
+    if (chosenSheets.sheetName.includes('Resources Relational Table')) {
+        createSheetList.push(createResourcesRelationalTable);
+    }
+    if (chosenSheets.sheetName.includes('Resources')) {
+        createSheetList.push(createResources);
+    }
+    if (chosenSheets.sheetName.includes('Images')) {
+        createSheetList.push(createImageList);
+    }
+    // eslint-disable-next-line no-console
+    console.log(`\nGenerating ${createSheetList.length} sheet(s)...\n`);
+    const lanes = new Lanes({ verbose: !process.stdout.isTTY, indent: '  ' });
+    log('Lanes created (verbose: %s)', !process.stdout.isTTY);
+    const RATE_LIMIT_LANE = 10_000;
+    let countdownSeq = 0;
+    let waitingCount = 0;
+    sheets.onLog = (message) => {
+        if (message.waiting && message.waitTime) {
+            waitingCount++;
+            const id = `rateLimit_${countdownSeq++}`;
+            const label = message.message === 'TooManyRequestError'
+                ? 'Too Many Requests (429)'
+                : message.message === 'UserRateLimitExceededError'
+                    ? 'Rate Limit Exceeded (403)'
+                    : 'Connection Reset';
+            lanes.update(RATE_LIMIT_LANE, c.yellow(`${label}: waiting %countdown(${message.waitTime}, ${id}, s)%s`));
+        }
+        else {
+            waitingCount--;
+            if (waitingCount <= 0) {
+                waitingCount = 0;
+                lanes.delete(RATE_LIMIT_LANE);
+            }
+        }
+    };
+    log('Reporting starts (limit: %d)', limit);
+    try {
+        await createSheets({
+            sheets,
+            archive,
+            reports,
+            limit,
+            createSheetList,
+            options: { lanes },
+        });
+    }
+    finally {
+        lanes.close();
+    }
+    log('Reporting done');
+    // eslint-disable-next-line no-console
+    console.log('\nReport complete.');
+    if (chosenSheets.sheetName.includes('Summary')) {
+        await addToSummary( /*sheets, archive, reports*/);
+    }
+    archiveLog('Closes file');
+    await archive.close();
+    log('Done');
+}

package/lib/reports/get-plugin-reports.d.ts

@@ -0,0 +1,7 @@
+import type { Archive } from '@nitpicker/crawler';
+import type { Report } from '@nitpicker/types';
+/**
+ *
+ * @param archive
+ */
+export declare function getPluginReports(archive: Archive): Promise<Report[]>;

package/lib/reports/get-plugin-reports.js

@@ -0,0 +1,19 @@
+import { reportLog } from '../debug.js';
+/**
+ *
+ * @param archive
+ */
+export async function getPluginReports(archive) {
+    const reports = [];
+    reportLog('Load');
+    try {
+        const report = await archive.getData('analysis/report');
+        if (report) {
+            reports.push(report);
+        }
+    }
+    catch {
+        reportLog('Failed: report is not found');
+    }
+    return reports;
+}

package/lib/sheets/create-cell-data.d.ts

@@ -0,0 +1 @@
+export { createCellData } from '@d-zero/google-sheets';

package/lib/sheets/create-cell-data.js

@@ -0,0 +1 @@
+export { createCellData } from '@d-zero/google-sheets';

package/lib/sheets/create-sheets.d.ts

@@ -0,0 +1,74 @@
+import type { CreateSheet } from './types.js';
+import type { Lanes } from '@d-zero/dealer';
+import type { Sheets } from '@d-zero/google-sheets';
+import type { Archive } from '@nitpicker/crawler';
+import type { Report } from '@nitpicker/types';
+/**
+ * Parameters for {@link createSheets}.
+ */
+export interface CreateSheetsParams {
+    /** Google Sheets API ラッパー */
+    readonly sheets: Sheets;
+    /** クロール結果のアーカイブ */
+    readonly archive: Archive;
+    /** 監査プラグインのレポート配列 */
+    readonly reports: Report[];
+    /** getPagesWithRefs のバッチサイズ（デフォルト 100,000） */
+    readonly limit: number;
+    /** シート設定のファクトリ関数配列 */
+    readonly createSheetList: CreateSheet[];
+    /** Lanes インスタンスを含むオプション */
+    readonly options?: {
+        /** Lanes instance for terminal progress display. */
+        readonly lanes: Lanes;
+    };
+}
+/**
+ * Google Sheets にシートを作成し、データを投入してフォーマットする。
+ *
+ * ## 処理フェーズ
+ *
+ * 5つのフェーズで構成され、Phase 2+3 と Phase 4 は並列実行される:
+ *
+ * ```
+ * Phase 1 (Creating sheets)
+ *   → Phase 2 (Processing pages) ─→ Phase 3 (Processing resources)
+ *   → Phase 4 (Plugin data / addRows)  ← Phase 2+3 と並列
+ * → Phase 5 (Formatting sheets)
+ * ```
+ *
+ * ## Lanes 進捗表示
+ *
+ * ### ヘッダー: 加重平均による集計
+ *
+ * Phase 2/3 のヘッダーは全子タスク（シート）の進捗を加重平均で集計する。
+ *
+ * 以前の実装では `Math.min()` で最遅シートのページ生成進捗のみを表示していたが、
+ * シートごとに「生成完了→行送信中」「まだ生成中」「送信完了」と異なるサブフェーズに
+ * いる場合、ヘッダーの数値と各レーンの表示が乖離する問題があった。
+ * 例: ヘッダー「174/755 (23%)」だが Page List は既に Sent、
+ * Referrers RT は Sending rows 7500/13256 — ヘッダーが全体を見ていない。
+ *
+ * 解決策として、各シートの進捗を 0.0〜1.0 の fraction で管理し、
+ * 全シートの平均値をヘッダーに表示する:
+ *
+ * - **生成フェーズ**: `(pageNum / max) * 0.5` → 0%〜50%
+ * - **送信フェーズ**: `0.5 + (sentRows / totalRows) * 0.5` → 50%〜100%
+ *   (`sendRowsInChunks` の `onProgress` コールバック経由)
+ * - **完了**: 1.0 (100%)
+ *
+ * 生成と送信を 50:50 で重み付けしている理由:
+ * - 生成だけで 0〜100% にすると、全シートが生成完了（100%）になっても
+ *   送信が残っている状態で「100%」と表示されてしまう
+ * - 送信（特に大量行のチャンク送信）は Google API の
+ *   レート制限により生成と同程度の時間がかかるため、等配分が妥当
+ *
+ * ### レーン: フェーズ遷移に応じた状態表示
+ *
+ * - **アクティブフェーズ内で完了 + 将来フェーズあり**: `c.green("Sent (N rows)")`
+ * - **アクティブフェーズ内で完了 + 全フェーズ完了**: `c.green("Done (N rows)")`
+ * - **非アクティブ + 完了済み**: `c.dim` で同じテキスト（色だけ変化、Waiting に戻らない）
+ * - **非アクティブ + 未着手**: `c.dim("Waiting...")`
+ * @param params - シート作成に必要なパラメータ
+ */
+export declare function createSheets(params: CreateSheetsParams): Promise<void>;