@d-zero/archaeologist 2.0.0 → 3.0.1

package/README.md

@@ -12,16 +12,22 @@
 ## CLI
 
 ```sh
-npx @d-zero/archaeologist -f <filepath> [--limit <number>] [--debug]
+npx @d-zero/archaeologist -f <listfile> [options]
 ```
 
 URLリストを持つファイルを指定して実行します。
 
 ### オプション
 
-- `-f, --file <filepath>`: URLリストを持つファイルのパス（必須）
+- `-f, --listfile <filepath>`: URLリストを持つファイルのパス（必須）
+- `-t, --type <types>`: 比較タイプの指定（`image,dom,text`、カンマ区切り）
+- `-s, --selector <selector>`: 比較対象を限定するCSSセレクター
+- `-i, --ignore <selector>`: 無視するCSSセレクター
+- `--devices <devices>`: デバイス指定（`desktop,mobile,tablet`、カンマ区切り）
+- `--freeze <filepath>`: フリーズモード用ファイルパス
 - `--limit <number>`: 並列実行数の上限（デフォルト: 10）
 - `--debug`: デバッグモード（デフォルト: false）
+- `--verbose`: 詳細ログモード（デフォルト: false）
 
 ### ファイルフォーマット
 
@@ -53,7 +59,7 @@ https://example.com/xyz/001
 
 ## ページフック
 
-[Frontmatter](https://jekyllrb.com/docs/front-matter/)の`hooks`に配列としてスクリプトファイルのパスを渡すと、ページを開いた後（厳密にはPuppetterの`waitUntil: 'networkidle0'`のタイミング直後）にそれらのスクリプトを実行します。スクリプトは配列の順番通りに逐次実行されます。
+[Frontmatter](https://jekyllrb.com/docs/front-matter/)の`hooks`に配列としてスクリプトファイルのパスを渡すと、ページを開いた後（厳密にはPuppeteerの`waitUntil: 'networkidle0'`のタイミング直後）にそれらのスクリプトを実行します。スクリプトは配列の順番通りに逐次実行されます。
 
 ```txt
 ---

package/dist/analyze-child-process.d.ts

@@ -4,6 +4,9 @@ export type ChildProcessParams = {
     list: readonly URLPair[];
     dir: string;
     useOldMode: boolean;
-    htmlDiffOnly: boolean;
+    types?: readonly string[];
+    selector?: string;
+    ignore?: string;
+    devices?: readonly string[];
     hooks?: readonly PageHook[];
 };

package/dist/analyze-child-process.js

@@ -4,11 +4,13 @@ import { createChildProcess } from '@d-zero/puppeteer-dealer';
 import { delay } from '@d-zero/shared/delay';
 import c from 'ansi-colors';
 import { diffImages } from './modules/diff-images.js';
+import { diffText } from './modules/diff-text.js';
 import { diffTree } from './modules/diff-tree.js';
 import { getData } from './modules/get-data.js';
+import { normalizeTextDocument } from './modules/normalize-text-document.js';
 import { score } from './utils.js';
 createChildProcess((param) => {
-    const { list, dir, htmlDiffOnly } = param;
+    const { list, dir, types = ['image', 'dom', 'text'], selector, ignore, devices, } = param;
     return {
         async eachPage({ page, url: urlA, index }, logger) {
             const urlPair = list.find(([url]) => url === urlA);
@@ -18,7 +20,10 @@ createChildProcess((param) => {
             const dataPair = [];
             for (const url of urlPair) {
                 const data = await getData(page, url, {
-                    htmlDiffOnly,
+                    htmlDiffOnly: !types.includes('image'),
+                    selector,
+                    ignore,
+                    devices,
                 }, logger);
                 dataPair.push(data);
                 await delay(600);
@@ -36,47 +41,71 @@ createChildProcess((param) => {
                 if (!screenshotB) {
                     throw new Error(`Failed to get screenshotB: ${id}`);
                 }
-                const imageDiff = await diffImages(screenshotA, screenshotB, (phase, data) => {
-                    switch (phase) {
-                        case 'create': {
-                            logger(`${sizeName} ${outputUrl} 🖼️ Create images`);
-                            break;
-                        }
-                        case 'resize': {
-                            const { width, height } = data;
-                            logger(`${sizeName} ${outputUrl} ↔️ Resize images to ${width}x${height}`);
-                            break;
-                        }
-                        case 'diff': {
-                            logger(`${sizeName} ${outputUrl} 📊 Compare images`);
-                            break;
+                let image = null;
+                if (types.includes('image')) {
+                    const imageDiff = await diffImages(screenshotA, screenshotB, (phase, data) => {
+                        switch (phase) {
+                            case 'create': {
+                                logger(`${sizeName} ${outputUrl} 🖼️ Create images`);
+                                break;
+                            }
+                            case 'resize': {
+                                const { width, height } = data;
+                                logger(`${sizeName} ${outputUrl} ↔️ Resize images to ${width}x${height}`);
+                                break;
+                            }
+                            case 'diff': {
+                                logger(`${sizeName} ${outputUrl} 📊 Compare images`);
+                                break;
+                            }
                         }
+                    });
+                    if (imageDiff) {
+                        logger(`${sizeName} ${outputUrl} 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
+                        await delay(1500);
+                        await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
+                        await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
+                        const outFilePath = path.resolve(dir, `${id}_diff.png`);
+                        logger(`${sizeName} ${outputUrl} 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
+                        await writeFile(outFilePath, imageDiff.images.diff);
+                        image = {
+                            matches: imageDiff.matches,
+                            file: outFilePath,
+                        };
                     }
-                });
-                let image = null;
-                if (imageDiff) {
-                    logger(`${sizeName} ${outputUrl} 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
-                    await delay(1500);
-                    await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
-                    await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
-                    const outFilePath = path.resolve(dir, `${id}_diff.png`);
-                    logger(`${sizeName} ${outputUrl} 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
-                    await writeFile(outFilePath, imageDiff.images.diff);
-                    image = {
-                        matches: imageDiff.matches,
+                }
+                let dom = null;
+                if (types.includes('dom')) {
+                    const htmlDiff = diffTree(a.url, b.url, screenshotA.domTree, screenshotB.domTree);
+                    const outFilePath = path.resolve(dir, `${id}_html.diff`);
+                    await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
+                    dom = {
+                        matches: htmlDiff.matches,
+                        diff: htmlDiff.changed ? htmlDiff.result : null,
+                        file: outFilePath,
+                    };
+                }
+                let text = null;
+                if (types.includes('text')) {
+                    const contentA = normalizeTextDocument(screenshotA.text.textContent);
+                    const contentB = normalizeTextDocument(screenshotB.text.textContent);
+                    const altTextListA = screenshotA.text.altTextList.join('\n');
+                    const altTextListB = screenshotB.text.altTextList.join('\n');
+                    const textA = `${contentA}\n\n${altTextListA}`;
+                    const textB = `${contentB}\n\n${altTextListB}`;
+                    const textDiff = diffText(a.url, b.url, textA, textB);
+                    const outFilePath = path.resolve(dir, `${id}_text.diff`);
+                    await writeFile(outFilePath, `${textDiff.phrases.result}\n\n${textDiff.tokens.result}`, { encoding: 'utf8' });
+                    text = {
+                        matches: textDiff.tokens.matches,
+                        diff: textDiff.tokens.changed ? textDiff.tokens.result : null,
                         file: outFilePath,
                     };
                 }
-                const htmlDiff = diffTree(a.url, b.url, screenshotA.domTree, screenshotB.domTree);
-                const outFilePath = path.resolve(dir, `${id}_html.diff`);
-                await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
                 screenshotResult[name] = {
                     image,
-                    dom: {
-                        matches: htmlDiff.matches,
-                        diff: htmlDiff.changed ? htmlDiff.result : null,
-                        file: outFilePath,
-                    },
+                    dom,
+                    text,
                 };
             }
             const result = {

package/dist/analyze-main-process.d.ts

@@ -1,7 +1,8 @@
 import type { AnalyzeOptions, URLPair } from './types.js';
+import type { DealOptions } from '@d-zero/dealer';
 /**
  *
  * @param list
  * @param options
  */
-export declare function analyze(list: readonly URLPair[], options?: AnalyzeOptions): Promise<void>;
+export declare function analyze(list: readonly URLPair[], options?: AnalyzeOptions & DealOptions): Promise<void>;

package/dist/analyze-main-process.js

@@ -1,7 +1,8 @@
-import { mkdir } from 'node:fs/promises';
+import { writeFile, mkdir } from 'node:fs/promises';
 import path from 'node:path';
 import { createProcess, deal } from '@d-zero/puppeteer-dealer';
 import c from 'ansi-colors';
+import stripAnsi from 'strip-ansi';
 import { analyzeUrlList } from './modules/analize-url.js';
 import { score } from './utils.js';
 /**
@@ -22,25 +23,37 @@ export async function analyze(list, options) {
             list,
             dir,
             useOldMode,
-            htmlDiffOnly: options?.htmlDiffOnly ?? false,
+            types: options?.types,
+            selector: options?.selector,
+            ignore: options?.ignore,
+            devices: options?.devices,
             hooks: options?.hooks ?? [],
         }, {
             ...options,
             headless: useOldMode ? 'shell' : true,
         });
-    }, (result) => {
-        results.push(result);
+    }, {
+        ...options,
+        each(result) {
+            results.push(result);
+        },
     });
     const output = [];
     for (const result of results) {
         output.push(c.gray(`${result.target.join(' vs ')}`));
-        for (const [sizeName, { image, dom }] of Object.entries(result.screenshots)) {
+        for (const [sizeName, { image, dom, text }] of Object.entries(result.screenshots)) {
             if (image) {
                 const { matches, file } = image;
                 output.push(`  ${c.bgMagenta(` ${sizeName} `)} ${score(matches, 0.9)} ${file}`);
             }
-            output.push(`  ${c.bgBlueBright(' HTML ')}: ${score(dom.matches, 0.995)} ${dom.file}`);
+            if (dom) {
+                output.push(`  ${c.bgBlueBright(' HTML ')}: ${score(dom.matches, 0.995)} ${dom.file}`);
+            }
+            if (text) {
+                output.push(`  ${c.bgGreenBright(' TEXT ')}: ${score(text.matches, 0.995)} ${text.file}`);
+            }
         }
     }
+    await writeFile(path.resolve(dir, 'RESULT.txt'), stripAnsi(output.join('\n').replaceAll(dir, '.')), 'utf8');
     process.stdout.write(output.join('\n') + '\n');
 }

package/dist/cli.js

@@ -1,32 +1,52 @@
 #!/usr/bin/env node
-import minimist from 'minimist';
+import { createCLI, parseCommonOptions, parseList } from '@d-zero/cli-core';
 import { analyze } from './analyze-main-process.js';
 import { freeze } from './freeze-main-process.js';
 import { readConfig } from './read-config.js';
-const cli = minimist(process.argv.slice(2), {
-    alias: {
+const { options, hasConfigFile } = createCLI({
+    aliases: {
         f: 'listfile',
+        t: 'type',
+        s: 'selector',
+        i: 'ignore',
+    },
+    usage: ['Usage: archaeologist -f <listfile> [--limit <number>]'],
+    parseArgs: (cli) => ({
+        ...parseCommonOptions(cli),
+        listfile: cli.listfile,
+        type: cli.type,
+        freeze: cli.freeze,
+        selector: cli.selector,
+        ignore: cli.ignore,
+        devices: cli.devices ??
+            // Alias for devices
+            cli.device,
+    }),
+    validateArgs: (options) => {
+        return !!(options.listfile?.length || options.freeze?.length);
     },
 });
-if (cli.listfile?.length) {
-    const { pairList, hooks } = await readConfig(cli.listfile);
+if (hasConfigFile) {
+    const { pairList, hooks } = await readConfig(options.listfile);
     await analyze(pairList, {
         hooks,
-        limit: cli.limit ? Number.parseInt(cli.limit) : undefined,
-        debug: !!cli.debug,
-        htmlDiffOnly: !!cli.htmlDiffOnly,
+        types: options.type ? parseList(options.type) : undefined,
+        selector: options.selector,
+        ignore: options.ignore,
+        devices: options.devices ? parseList(options.devices) : undefined,
+        limit: options.limit,
+        debug: options.debug,
+        verbose: options.verbose,
     });
     process.exit(0);
 }
-if (cli.freeze) {
-    const { pairList, hooks } = await readConfig(cli.freeze);
+if (options.freeze) {
+    const { pairList, hooks } = await readConfig(options.freeze);
     const list = pairList.map(([urlA]) => urlA);
     await freeze(list, {
         hooks,
-        limit: cli.limit ? Number.parseInt(cli.limit) : undefined,
-        debug: !!cli.debug,
+        limit: options.limit,
+        debug: options.debug,
     });
     process.exit(0);
 }
-process.stderr.write('Usage: archaeologist -f <listfile> [--limit <number>]\n');
-process.exit(1);

package/dist/modules/diff-text.d.ts

@@ -0,0 +1,21 @@
+/**
+ *
+ * @param urlA
+ * @param urlB
+ * @param phraseA
+ * @param phraseB
+ */
+export declare function diffText(urlA: string, urlB: string, phraseA: string, phraseB: string): {
+    phrases: {
+        changed: boolean;
+        maxLine: number;
+        matches: number;
+        result: string;
+    };
+    tokens: {
+        changed: boolean;
+        maxLine: number;
+        matches: number;
+        result: string;
+    };
+};

package/dist/modules/diff-text.js

@@ -0,0 +1,45 @@
+import { getTokenizer } from 'kuromojin';
+import { diffTree } from './diff-tree.js';
+const tokenizer = await getTokenizer();
+/**
+ *
+ * @param text
+ */
+function tokenList(text) {
+    return tokenizer
+        .tokenize(text)
+        .filter((token) => token.surface_form.trim() !== '')
+        .map((token) => `${token.surface_form}:${token.pos}:${token.pos_detail_1}`);
+}
+/**
+ *
+ * @param tokens
+ */
+function frequencyMap(tokens) {
+    const map = new Map();
+    for (const token of tokens) {
+        map.set(token, (map.get(token) ?? 0) + 1);
+    }
+    return map
+        .entries()
+        .map(([token, frequency]) => `${token} x${frequency}`)
+        .toArray()
+        .toSorted((a, b) => a.localeCompare(b));
+}
+/**
+ *
+ * @param urlA
+ * @param urlB
+ * @param phraseA
+ * @param phraseB
+ */
+export function diffText(urlA, urlB, phraseA, phraseB) {
+    const tokensA = tokenList(phraseA);
+    const tokensB = tokenList(phraseB);
+    const frequencyMapA = frequencyMap(tokensA).join('\n');
+    const frequencyMapB = frequencyMap(tokensB).join('\n');
+    return {
+        phrases: diffTree(urlA, urlB, phraseA, phraseB),
+        tokens: diffTree(urlA, urlB, frequencyMapA, frequencyMapB),
+    };
+}

package/dist/modules/get-data.d.ts

@@ -4,6 +4,9 @@ import type { Page } from 'puppeteer';
 export interface GetDataOptions {
     readonly hooks?: readonly PageHook[];
     readonly htmlDiffOnly?: boolean;
+    readonly selector?: string;
+    readonly ignore?: string;
+    readonly devices?: readonly string[];
 }
 /**
  *

package/dist/modules/get-data.js

@@ -1,4 +1,5 @@
 import { distill } from '@d-zero/html-distiller';
+import { defaultSizes } from '@d-zero/puppeteer-page-scan';
 import { screenshotListener, screenshot } from '@d-zero/puppeteer-screenshot';
 /**
  *
@@ -9,19 +10,19 @@ import { screenshotListener, screenshot } from '@d-zero/puppeteer-screenshot';
  */
 export async function getData(page, url, options, update) {
     const htmlDiffOnly = options.htmlDiffOnly ?? false;
+    const devices = options.devices ?? ['desktop', 'mobile'];
+    const sizes = {};
+    for (const device of devices) {
+        // @ts-ignore
+        sizes[device] = defaultSizes[device];
+    }
     const screenshots = await screenshot(page, url, {
-        sizes: {
-            desktop: {
-                width: 1280,
-            },
-            mobile: {
-                width: 375,
-                resolution: 2,
-            },
-        },
+        sizes,
         hooks: options?.hooks ?? [],
         listener: screenshotListener(update),
         domOnly: htmlDiffOnly,
+        selector: options.selector,
+        ignore: options.ignore,
     });
     const data = { url, screenshots: {} };
     for (const [sizeName, screenshot] of Object.entries(screenshots)) {

package/dist/modules/normalize-text-document.d.ts

@@ -0,0 +1,5 @@
+/**
+ *
+ * @param text
+ */
+export declare function normalizeTextDocument(text: string): string;

package/dist/modules/normalize-text-document.js

@@ -0,0 +1,15 @@
+/**
+ *
+ * @param text
+ */
+export function normalizeTextDocument(text) {
+    return (text
+        .trim()
+        // Spaces
+        .replaceAll(/\s+/g, '\n')
+        // Periods
+        .replaceAll('。', '。\n')
+        // Newlines
+        .replaceAll(/\n+/g, '\n')
+        .trim());
+}

package/dist/parse-types.d.ts

@@ -0,0 +1,5 @@
+/**
+ *
+ * @param typeQuery
+ */
+export declare function parseTypes(typeQuery: string): string[];

package/dist/parse-types.js

@@ -0,0 +1,8 @@
+/**
+ *
+ * @param typeQuery
+ */
+export function parseTypes(typeQuery) {
+    const types = typeQuery.split(',').map((type) => type.trim());
+    return types;
+}

package/dist/read-config.js

@@ -1,17 +1,12 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
 import { readPageHooks } from '@d-zero/puppeteer-page-scan';
 import { toList } from '@d-zero/readtext/list';
-import fm from 'front-matter';
+import { readConfigFile } from '@d-zero/shared/config-reader';
 /**
  *
  * @param filePath
  */
 export async function readConfig(filePath) {
-    const fileContent = await fs.readFile(filePath, 'utf8');
-    const content = 
-    // @ts-ignore
-    fm(fileContent);
+    const { content, baseDir } = await readConfigFile(filePath);
     const urlList = toList(content.body);
     const pairList = urlList.map((urlStr) => {
         const url = new URL(urlStr);
@@ -20,7 +15,6 @@ export async function readConfig(filePath) {
             `${content.attributes.comparisonHost}${url.pathname}${url.search}`,
         ];
     });
-    const baseDir = path.dirname(filePath);
     const hooks = await readPageHooks(content.attributes?.hooks ?? [], baseDir);
     return {
         pairList,

package/dist/types.d.ts

@@ -13,7 +13,8 @@ export type Result = {
 };
 export type MediaResult = {
     image: ImageResult | null;
-    dom: DOMResult;
+    dom: DOMResult | null;
+    text: TextResult | null;
 };
 export type ImageResult = {
     matches: number;
@@ -24,10 +25,18 @@ export type DOMResult = {
     diff: string | null;
     file: string;
 };
+export type TextResult = {
+    matches: number;
+    diff: string | null;
+    file: string;
+};
 export interface ArchaeologistOptions extends AnalyzeOptions {
 }
 export interface AnalyzeOptions extends GeneralOptions {
-    readonly htmlDiffOnly?: boolean;
+    readonly types?: readonly string[];
+    readonly selector?: string;
+    readonly ignore?: string;
+    readonly devices?: readonly string[];
 }
 export interface FreezeOptions extends GeneralOptions {
 }

package/package.json

@@ -1,10 +1,9 @@
 {
 	"name": "@d-zero/archaeologist",
-	"version": "2.0.0",
+	"version": "3.0.1",
 	"description": "Uncover visual and HTML differences in web pages with precision",
 	"author": "D-ZERO",
 	"license": "MIT",
-	"private": false,
 	"publishConfig": {
 		"access": "public"
 	},
@@ -15,9 +14,7 @@
 			"types": "./dist/index.d.ts"
 		}
 	},
-	"bin": {
-		"archaeologist": "./dist/cli.js"
-	},
+	"bin": "./dist/cli.js",
 	"files": [
 		"dist"
 	],
@@ -27,27 +24,30 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/fs": "0.2.0",
-		"@d-zero/html-distiller": "1.0.3",
-		"@d-zero/puppeteer-dealer": "0.4.0",
-		"@d-zero/puppeteer-page-scan": "4.0.0",
-		"@d-zero/puppeteer-screenshot": "3.0.1",
-		"@d-zero/readtext": "1.1.3",
-		"@d-zero/shared": "0.8.0",
+		"@d-zero/cli-core": "1.1.0",
+		"@d-zero/fs": "0.2.1",
+		"@d-zero/html-distiller": "1.0.4",
+		"@d-zero/puppeteer-dealer": "0.5.1",
+		"@d-zero/puppeteer-page-scan": "4.0.2",
+		"@d-zero/puppeteer-screenshot": "3.1.1",
+		"@d-zero/readtext": "1.1.5",
+		"@d-zero/shared": "0.9.1",
 		"ansi-colors": "4.1.3",
-		"diff": "8.0.1",
+		"diff": "8.0.2",
 		"front-matter": "4.0.2",
 		"jimp": "1.6.0",
+		"kuromojin": "3.0.1",
 		"minimist": "1.2.8",
 		"parse-diff": "0.11.1",
 		"pixelmatch": "7.1.0",
 		"pngjs": "7.0.0",
-		"puppeteer": "24.9.0"
+		"puppeteer": "24.10.2",
+		"strip-ansi": "7.1.0"
 	},
 	"devDependencies": {
 		"@types/diff": "8.0.0",
 		"@types/pixelmatch": "5.2.6",
 		"@types/pngjs": "6.0.5"
 	},
-	"gitHead": "4e9cc7b87e0fef91b6f2d4edfb66ca9134b2491b"
+	"gitHead": "1218a023e62c79efeece6350d561f2e1906be7ea"
 }