@d-zero/archaeologist 1.1.3 → 3.0.0

package/README.md

@@ -12,16 +12,22 @@
 ## CLI
 
 ```sh
-npx @d-zero/archaeologist -f <filepath> [--limit <number>] [--debug]
+npx @d-zero/archaeologist -f <listfile> [options]
 ```
 
 URLリストを持つファイルを指定して実行します。
 
 ### オプション
 
-- `-f, --file <filepath>`: URLリストを持つファイルのパス（必須）
+- `-f, --listfile <filepath>`: URLリストを持つファイルのパス（必須）
+- `-t, --type <types>`: 比較タイプの指定（`image,dom,text`、カンマ区切り）
+- `-s, --selector <selector>`: 比較対象を限定するCSSセレクター
+- `-i, --ignore <selector>`: 無視するCSSセレクター
+- `--devices <devices>`: デバイス指定（`desktop,mobile,tablet`、カンマ区切り）
+- `--freeze <filepath>`: フリーズモード用ファイルパス
 - `--limit <number>`: 並列実行数の上限（デフォルト: 10）
 - `--debug`: デバッグモード（デフォルト: false）
+- `--verbose`: 詳細ログモード（デフォルト: false）
 
 ### ファイルフォーマット
 
@@ -53,7 +59,7 @@ https://example.com/xyz/001
 
 ## ページフック
 
-[Frontmatter](https://jekyllrb.com/docs/front-matter/)の`hooks`に配列としてスクリプトファイルのパスを渡すと、ページを開いた後（厳密にはPuppetterの`waitUntil: 'networkidle0'`のタイミング直後）にそれらのスクリプトを実行します。スクリプトは配列の順番通りに逐次実行されます。
+[Frontmatter](https://jekyllrb.com/docs/front-matter/)の`hooks`に配列としてスクリプトファイルのパスを渡すと、ページを開いた後（厳密にはPuppeteerの`waitUntil: 'networkidle0'`のタイミング直後）にそれらのスクリプトを実行します。スクリプトは配列の順番通りに逐次実行されます。
 
 ```txt
 ---

package/dist/analyze-child-process.d.ts

@@ -0,0 +1,12 @@
+import type { URLPair } from './types.js';
+import type { PageHook } from '@d-zero/puppeteer-page-scan';
+export type ChildProcessParams = {
+    list: readonly URLPair[];
+    dir: string;
+    useOldMode: boolean;
+    types?: readonly string[];
+    selector?: string;
+    ignore?: string;
+    devices?: readonly string[];
+    hooks?: readonly PageHook[];
+};

package/dist/analyze-child-process.js

@@ -0,0 +1,118 @@
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { createChildProcess } from '@d-zero/puppeteer-dealer';
+import { delay } from '@d-zero/shared/delay';
+import c from 'ansi-colors';
+import { diffImages } from './modules/diff-images.js';
+import { diffText } from './modules/diff-text.js';
+import { diffTree } from './modules/diff-tree.js';
+import { getData } from './modules/get-data.js';
+import { normalizeTextDocument } from './modules/normalize-text-document.js';
+import { score } from './utils.js';
+createChildProcess((param) => {
+    const { list, dir, types = ['image', 'dom', 'text'], selector, ignore, devices, } = param;
+    return {
+        async eachPage({ page, url: urlA, index }, logger) {
+            const urlPair = list.find(([url]) => url === urlA);
+            if (!urlPair) {
+                throw new Error(`Failed to find urlPair: ${urlA}`);
+            }
+            const dataPair = [];
+            for (const url of urlPair) {
+                const data = await getData(page, url, {
+                    htmlDiffOnly: !types.includes('image'),
+                    selector,
+                    ignore,
+                    devices,
+                }, logger);
+                dataPair.push(data);
+                await delay(600);
+            }
+            const [a, b] = dataPair;
+            if (!a || !b) {
+                throw new Error('Failed to get screenshots');
+            }
+            const screenshotResult = {};
+            const outputUrl = 'vs ' + c.gray(urlPair[1]);
+            for (const [name, screenshotA] of Object.entries(a.screenshots)) {
+                const screenshotB = b.screenshots[name];
+                const sizeName = c.bgMagenta(` ${name} `);
+                const id = `${index}_${name}`;
+                if (!screenshotB) {
+                    throw new Error(`Failed to get screenshotB: ${id}`);
+                }
+                let image = null;
+                if (types.includes('image')) {
+                    const imageDiff = await diffImages(screenshotA, screenshotB, (phase, data) => {
+                        switch (phase) {
+                            case 'create': {
+                                logger(`${sizeName} ${outputUrl} 🖼️ Create images`);
+                                break;
+                            }
+                            case 'resize': {
+                                const { width, height } = data;
+                                logger(`${sizeName} ${outputUrl} ↔️ Resize images to ${width}x${height}`);
+                                break;
+                            }
+                            case 'diff': {
+                                logger(`${sizeName} ${outputUrl} 📊 Compare images`);
+                                break;
+                            }
+                        }
+                    });
+                    if (imageDiff) {
+                        logger(`${sizeName} ${outputUrl} 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
+                        await delay(1500);
+                        await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
+                        await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
+                        const outFilePath = path.resolve(dir, `${id}_diff.png`);
+                        logger(`${sizeName} ${outputUrl} 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
+                        await writeFile(outFilePath, imageDiff.images.diff);
+                        image = {
+                            matches: imageDiff.matches,
+                            file: outFilePath,
+                        };
+                    }
+                }
+                let dom = null;
+                if (types.includes('dom')) {
+                    const htmlDiff = diffTree(a.url, b.url, screenshotA.domTree, screenshotB.domTree);
+                    const outFilePath = path.resolve(dir, `${id}_html.diff`);
+                    await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
+                    dom = {
+                        matches: htmlDiff.matches,
+                        diff: htmlDiff.changed ? htmlDiff.result : null,
+                        file: outFilePath,
+                    };
+                }
+                let text = null;
+                if (types.includes('text')) {
+                    const contentA = normalizeTextDocument(screenshotA.text.textContent);
+                    const contentB = normalizeTextDocument(screenshotB.text.textContent);
+                    const altTextListA = screenshotA.text.altTextList.join('\n');
+                    const altTextListB = screenshotB.text.altTextList.join('\n');
+                    const textA = `${contentA}\n\n${altTextListA}`;
+                    const textB = `${contentB}\n\n${altTextListB}`;
+                    const textDiff = diffText(a.url, b.url, textA, textB);
+                    const outFilePath = path.resolve(dir, `${id}_text.diff`);
+                    await writeFile(outFilePath, `${textDiff.phrases.result}\n\n${textDiff.tokens.result}`, { encoding: 'utf8' });
+                    text = {
+                        matches: textDiff.tokens.matches,
+                        diff: textDiff.tokens.changed ? textDiff.tokens.result : null,
+                        file: outFilePath,
+                    };
+                }
+                screenshotResult[name] = {
+                    image,
+                    dom,
+                    text,
+                };
+            }
+            const result = {
+                target: [a.url, b.url],
+                screenshots: screenshotResult,
+            };
+            return result;
+        },
+    };
+});

package/dist/analyze-main-process.d.ts

@@ -0,0 +1,8 @@
+import type { AnalyzeOptions, URLPair } from './types.js';
+import type { DealOptions } from '@d-zero/dealer';
+/**
+ *
+ * @param list
+ * @param options
+ */
+export declare function analyze(list: readonly URLPair[], options?: AnalyzeOptions & DealOptions): Promise<void>;

package/dist/analyze-main-process.js

@@ -0,0 +1,59 @@
+import { writeFile, mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { createProcess, deal } from '@d-zero/puppeteer-dealer';
+import c from 'ansi-colors';
+import stripAnsi from 'strip-ansi';
+import { analyzeUrlList } from './modules/analize-url.js';
+import { score } from './utils.js';
+/**
+ *
+ * @param list
+ * @param options
+ */
+export async function analyze(list, options) {
+    const results = [];
+    const dir = path.resolve(process.cwd(), '.archaeologist');
+    await mkdir(dir, { recursive: true }).catch(() => { });
+    const urlInfo = analyzeUrlList(list);
+    const useOldMode = urlInfo.hasAuth || urlInfo.hasNoSSL;
+    await deal(list.map(([urlA]) => ({ id: null, url: urlA })), (_, done, total) => {
+        return `${c.bold.magenta('🕵️  Archaeologist')} ${done}/${total}`;
+    }, () => {
+        return createProcess(path.resolve(import.meta.dirname, 'analyze-child-process.js'), {
+            list,
+            dir,
+            useOldMode,
+            types: options?.types,
+            selector: options?.selector,
+            ignore: options?.ignore,
+            devices: options?.devices,
+            hooks: options?.hooks ?? [],
+        }, {
+            ...options,
+            headless: useOldMode ? 'shell' : true,
+        });
+    }, {
+        ...options,
+        each(result) {
+            results.push(result);
+        },
+    });
+    const output = [];
+    for (const result of results) {
+        output.push(c.gray(`${result.target.join(' vs ')}`));
+        for (const [sizeName, { image, dom, text }] of Object.entries(result.screenshots)) {
+            if (image) {
+                const { matches, file } = image;
+                output.push(`  ${c.bgMagenta(` ${sizeName} `)} ${score(matches, 0.9)} ${file}`);
+            }
+            if (dom) {
+                output.push(`  ${c.bgBlueBright(' HTML ')}: ${score(dom.matches, 0.995)} ${dom.file}`);
+            }
+            if (text) {
+                output.push(`  ${c.bgGreenBright(' TEXT ')}: ${score(text.matches, 0.995)} ${text.file}`);
+            }
+        }
+    }
+    await writeFile(path.resolve(dir, 'RESULT.txt'), stripAnsi(output.join('\n').replaceAll(dir, '.')), 'utf8');
+    process.stdout.write(output.join('\n') + '\n');
+}

package/dist/cli.js

@@ -1,21 +1,52 @@
 #!/usr/bin/env node
-import minimist from 'minimist';
-import { archaeologist } from './archaeologist.js';
+import { createCLI, parseCommonOptions, parseList } from '@d-zero/cli-core';
+import { analyze } from './analyze-main-process.js';
+import { freeze } from './freeze-main-process.js';
 import { readConfig } from './read-config.js';
-const cli = minimist(process.argv.slice(2), {
-    alias: {
+const { options, hasConfigFile } = createCLI({
+    aliases: {
         f: 'listfile',
+        t: 'type',
+        s: 'selector',
+        i: 'ignore',
+    },
+    usage: ['Usage: archaeologist -f <listfile> [--limit <number>]'],
+    parseArgs: (cli) => ({
+        ...parseCommonOptions(cli),
+        listfile: cli.listfile,
+        type: cli.type,
+        freeze: cli.freeze,
+        selector: cli.selector,
+        ignore: cli.ignore,
+        devices: cli.devices ??
+            // Alias for devices
+            cli.device,
+    }),
+    validateArgs: (options) => {
+        return !!(options.listfile?.length || options.freeze?.length);
     },
 });
-if (cli.listfile?.length) {
-    const { pairList, hooks } = await readConfig(cli.listfile);
-    await archaeologist(pairList, {
+if (hasConfigFile) {
+    const { pairList, hooks } = await readConfig(options.listfile);
+    await analyze(pairList, {
+        hooks,
+        types: options.type ? parseList(options.type) : undefined,
+        selector: options.selector,
+        ignore: options.ignore,
+        devices: options.devices ? parseList(options.devices) : undefined,
+        limit: options.limit,
+        debug: options.debug,
+        verbose: options.verbose,
+    });
+    process.exit(0);
+}
+if (options.freeze) {
+    const { pairList, hooks } = await readConfig(options.freeze);
+    const list = pairList.map(([urlA]) => urlA);
+    await freeze(list, {
         hooks,
-        limit: cli.limit ? Number.parseInt(cli.limit) : undefined,
-        debug: !!cli.debug,
-        htmlDiffOnly: !!cli.htmlDiffOnly,
+        limit: options.limit,
+        debug: options.debug,
     });
     process.exit(0);
 }
-process.stderr.write('Usage: archaeologist -f <listfile> [--limit <number>]\n');
-process.exit(1);

package/dist/freeze-child-process.d.ts

@@ -0,0 +1,3 @@
+export type ChildProcessParams = {
+    dir: string;
+};

package/dist/freeze-child-process.js

@@ -0,0 +1,22 @@
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { createChildProcess } from '@d-zero/puppeteer-dealer';
+import { delay } from '@d-zero/shared/delay';
+import { getData } from './modules/get-data.js';
+createChildProcess((param) => {
+    const { dir } = param;
+    return {
+        async eachPage({ page, id, url }, logger) {
+            const data = await getData(page, url, {}, logger);
+            await delay(600);
+            for (const size of Object.values(data.screenshots)) {
+                const jsonFile = path.resolve(dir, `${id}_${size.id}.html`);
+                const ssFile = path.resolve(dir, `${id}_${size.id}.png`);
+                await writeFile(jsonFile, size.dom, 'utf8');
+                if (size.binary) {
+                    await writeFile(ssFile, size.binary);
+                }
+            }
+        },
+    };
+});

package/dist/freeze-main-process.d.ts

@@ -0,0 +1,7 @@
+import type { FreezeOptions } from './types.js';
+/**
+ *
+ * @param list
+ * @param options
+ */
+export declare function freeze(list: readonly string[], options?: FreezeOptions): Promise<string>;

package/dist/freeze-main-process.js

@@ -0,0 +1,34 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { zip } from '@d-zero/fs/zip';
+import { createProcess, deal } from '@d-zero/puppeteer-dealer';
+import { timestamp } from '@d-zero/shared/timestamp';
+import c from 'ansi-colors';
+import { analyzeUrlList } from './modules/analize-url.js';
+/**
+ *
+ * @param list
+ * @param options
+ */
+export async function freeze(list, options) {
+    const name = `${timestamp('YYYYMMDD')}.archae`;
+    const dir = path.resolve(process.cwd(), `.${name}`);
+    await mkdir(dir, { recursive: true }).catch(() => { });
+    const urlInfo = analyzeUrlList(list);
+    const useOldMode = urlInfo.hasAuth || urlInfo.hasNoSSL;
+    await deal(list.map((url) => ({ id: null, url })), (_, done, total) => {
+        return `${c.bold.magenta('🕵️  Archaeologist Freeze❄️')} ${done}/${total}`;
+    }, () => {
+        return createProcess(path.resolve(import.meta.dirname, 'freeze-child-process.js'), {
+            dir,
+        }, {
+            ...options,
+            headless: useOldMode ? 'shell' : true,
+        });
+    });
+    const urlListPath = path.resolve(dir, '_URL_LIST.json');
+    await writeFile(urlListPath, JSON.stringify(list, null, '\t'), 'utf8');
+    const zipPath = path.resolve(process.cwd(), `${name}.zip`);
+    await zip(zipPath, dir);
+    return zipPath;
+}

package/dist/index.d.ts

@@ -1,2 +1,2 @@
-export { archaeologist } from './archaeologist.js';
+export { analyze as archaeologist } from './analyze-main-process.js';
 export * from './types.js';

package/dist/index.js

@@ -1,2 +1,2 @@
-export { archaeologist } from './archaeologist.js';
+export { analyze as archaeologist } from './analyze-main-process.js';
 export * from './types.js';

package/dist/modules/analize-url.d.ts

@@ -0,0 +1,11 @@
+import type { URLPair } from '../types.js';
+type AnalyzedUrlList = {
+    hasAuth: boolean;
+    hasNoSSL: boolean;
+};
+/**
+ *
+ * @param list
+ */
+export declare function analyzeUrlList(list: readonly (URLPair | string)[]): AnalyzedUrlList;
+export {};

package/dist/{analize-url.js → modules/analize-url.js}

@@ -8,6 +8,16 @@ export function analyzeUrlList(list) {
         hasNoSSL: false,
     };
     for (const urlPair of list) {
+        if (typeof urlPair === 'string') {
+            const urlObj = new URL(urlPair);
+            if (urlObj.username || urlObj.password) {
+                result.hasAuth = true;
+            }
+            if (urlObj.protocol === 'http:') {
+                result.hasNoSSL = true;
+            }
+            continue;
+        }
         for (const url of urlPair) {
             const urlObj = new URL(url);
             if (urlObj.username || urlObj.password) {

package/dist/modules/diff-text.d.ts

@@ -0,0 +1,21 @@
+/**
+ *
+ * @param urlA
+ * @param urlB
+ * @param phraseA
+ * @param phraseB
+ */
+export declare function diffText(urlA: string, urlB: string, phraseA: string, phraseB: string): {
+    phrases: {
+        changed: boolean;
+        maxLine: number;
+        matches: number;
+        result: string;
+    };
+    tokens: {
+        changed: boolean;
+        maxLine: number;
+        matches: number;
+        result: string;
+    };
+};

package/dist/modules/diff-text.js

@@ -0,0 +1,45 @@
+import { getTokenizer } from 'kuromojin';
+import { diffTree } from './diff-tree.js';
+const tokenizer = await getTokenizer();
+/**
+ *
+ * @param text
+ */
+function tokenList(text) {
+    return tokenizer
+        .tokenize(text)
+        .filter((token) => token.surface_form.trim() !== '')
+        .map((token) => `${token.surface_form}:${token.pos}:${token.pos_detail_1}`);
+}
+/**
+ *
+ * @param tokens
+ */
+function frequencyMap(tokens) {
+    const map = new Map();
+    for (const token of tokens) {
+        map.set(token, (map.get(token) ?? 0) + 1);
+    }
+    return map
+        .entries()
+        .map(([token, frequency]) => `${token} x${frequency}`)
+        .toArray()
+        .toSorted((a, b) => a.localeCompare(b));
+}
+/**
+ *
+ * @param urlA
+ * @param urlB
+ * @param phraseA
+ * @param phraseB
+ */
+export function diffText(urlA, urlB, phraseA, phraseB) {
+    const tokensA = tokenList(phraseA);
+    const tokensB = tokenList(phraseB);
+    const frequencyMapA = frequencyMap(tokensA).join('\n');
+    const frequencyMapB = frequencyMap(tokensB).join('\n');
+    return {
+        phrases: diffTree(urlA, urlB, phraseA, phraseB),
+        tokens: diffTree(urlA, urlB, frequencyMapA, frequencyMapB),
+    };
+}

package/dist/{get-data.d.ts → modules/get-data.d.ts}

@@ -1,9 +1,12 @@
-import type { PageData } from './types.js';
-import type { Page } from '@d-zero/puppeteer-page';
+import type { PageData } from '../types.js';
 import type { PageHook } from '@d-zero/puppeteer-screenshot';
+import type { Page } from 'puppeteer';
 export interface GetDataOptions {
     readonly hooks?: readonly PageHook[];
     readonly htmlDiffOnly?: boolean;
+    readonly selector?: string;
+    readonly ignore?: string;
+    readonly devices?: readonly string[];
 }
 /**
  *

package/dist/{get-data.js → modules/get-data.js}

@@ -1,4 +1,5 @@
 import { distill } from '@d-zero/html-distiller';
+import { defaultSizes } from '@d-zero/puppeteer-page-scan';
 import { screenshotListener, screenshot } from '@d-zero/puppeteer-screenshot';
 /**
  *
@@ -9,19 +10,19 @@ import { screenshotListener, screenshot } from '@d-zero/puppeteer-screenshot';
  */
 export async function getData(page, url, options, update) {
     const htmlDiffOnly = options.htmlDiffOnly ?? false;
+    const devices = options.devices ?? ['desktop', 'mobile'];
+    const sizes = {};
+    for (const device of devices) {
+        // @ts-ignore
+        sizes[device] = defaultSizes[device];
+    }
     const screenshots = await screenshot(page, url, {
-        sizes: {
-            desktop: {
-                width: 1280,
-            },
-            mobile: {
-                width: 375,
-                resolution: 2,
-            },
-        },
+        sizes,
         hooks: options?.hooks ?? [],
         listener: screenshotListener(update),
         domOnly: htmlDiffOnly,
+        selector: options.selector,
+        ignore: options.ignore,
     });
     const data = { url, screenshots: {} };
     for (const [sizeName, screenshot] of Object.entries(screenshots)) {

package/dist/modules/normalize-text-document.d.ts

@@ -0,0 +1,5 @@
+/**
+ *
+ * @param text
+ */
+export declare function normalizeTextDocument(text: string): string;

package/dist/modules/normalize-text-document.js

@@ -0,0 +1,15 @@
+/**
+ *
+ * @param text
+ */
+export function normalizeTextDocument(text) {
+    return (text
+        .trim()
+        // Spaces
+        .replaceAll(/\s+/g, '\n')
+        // Periods
+        .replaceAll('。', '。\n')
+        // Newlines
+        .replaceAll(/\n+/g, '\n')
+        .trim());
+}

package/dist/parse-types.d.ts

@@ -0,0 +1,5 @@
+/**
+ *
+ * @param typeQuery
+ */
+export declare function parseTypes(typeQuery: string): string[];

package/dist/parse-types.js

@@ -0,0 +1,8 @@
+/**
+ *
+ * @param typeQuery
+ */
+export function parseTypes(typeQuery) {
+    const types = typeQuery.split(',').map((type) => type.trim());
+    return types;
+}

package/dist/read-config.js

@@ -1,17 +1,12 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
 import { readPageHooks } from '@d-zero/puppeteer-page-scan';
 import { toList } from '@d-zero/readtext/list';
-import fm from 'front-matter';
+import { readConfigFile } from '@d-zero/shared/config-reader';
 /**
  *
  * @param filePath
  */
 export async function readConfig(filePath) {
-    const fileContent = await fs.readFile(filePath, 'utf8');
-    const content = 
-    // @ts-ignore
-    fm(fileContent);
+    const { content, baseDir } = await readConfigFile(filePath);
     const urlList = toList(content.body);
     const pairList = urlList.map((urlStr) => {
         const url = new URL(urlStr);
@@ -20,7 +15,6 @@ export async function readConfig(filePath) {
             `${content.attributes.comparisonHost}${url.pathname}${url.search}`,
         ];
     });
-    const baseDir = path.dirname(filePath);
     const hooks = await readPageHooks(content.attributes?.hooks ?? [], baseDir);
     return {
         pairList,

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 export type { PageHook } from '@d-zero/puppeteer-screenshot';
-import type { Screenshot } from '@d-zero/puppeteer-screenshot';
+import type { PageHook, Screenshot } from '@d-zero/puppeteer-screenshot';
 export type PageData = {
     url: string;
     screenshots: Record<string, Screenshot & {
@@ -13,7 +13,8 @@ export type Result = {
 };
 export type MediaResult = {
     image: ImageResult | null;
-    dom: DOMResult;
+    dom: DOMResult | null;
+    text: TextResult | null;
 };
 export type ImageResult = {
     matches: number;
@@ -24,3 +25,23 @@ export type DOMResult = {
     diff: string | null;
     file: string;
 };
+export type TextResult = {
+    matches: number;
+    diff: string | null;
+    file: string;
+};
+export interface ArchaeologistOptions extends AnalyzeOptions {
+}
+export interface AnalyzeOptions extends GeneralOptions {
+    readonly types?: readonly string[];
+    readonly selector?: string;
+    readonly ignore?: string;
+    readonly devices?: readonly string[];
+}
+export interface FreezeOptions extends GeneralOptions {
+}
+interface GeneralOptions {
+    readonly hooks: readonly PageHook[];
+    readonly limit?: number;
+    readonly debug?: boolean;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/archaeologist",
-	"version": "1.1.3",
+	"version": "3.0.0",
 	"description": "Uncover visual and HTML differences in web pages with precision",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -27,25 +27,30 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/html-distiller": "1.0.2",
-		"@d-zero/puppeteer-dealer": "0.3.0",
-		"@d-zero/puppeteer-page-scan": "3.0.0",
-		"@d-zero/puppeteer-screenshot": "3.0.0",
-		"@d-zero/readtext": "1.1.2",
+		"@d-zero/cli-core": "1.1.0",
+		"@d-zero/fs": "0.2.0",
+		"@d-zero/html-distiller": "1.0.3",
+		"@d-zero/puppeteer-dealer": "0.5.0",
+		"@d-zero/puppeteer-page-scan": "4.0.1",
+		"@d-zero/puppeteer-screenshot": "3.1.0",
+		"@d-zero/readtext": "1.1.4",
+		"@d-zero/shared": "0.9.0",
 		"ansi-colors": "4.1.3",
-		"diff": "7.0.0",
+		"diff": "8.0.2",
 		"front-matter": "4.0.2",
 		"jimp": "1.6.0",
+		"kuromojin": "3.0.1",
 		"minimist": "1.2.8",
 		"parse-diff": "0.11.1",
 		"pixelmatch": "7.1.0",
-		"pngjs": "7.0.0"
+		"pngjs": "7.0.0",
+		"puppeteer": "24.10.1",
+		"strip-ansi": "7.1.0"
 	},
 	"devDependencies": {
-		"@d-zero/puppeteer-page": "0.3.0",
-		"@types/diff": "7.0.1",
+		"@types/diff": "8.0.0",
 		"@types/pixelmatch": "5.2.6",
 		"@types/pngjs": "6.0.5"
 	},
-	"gitHead": "e4fd17857e31022d121527b00fd7f009dbdb2142"
+	"gitHead": "04c6969564182c36ee38ef41e78130936dfa4863"
 }

package/dist/analize-url.d.ts

@@ -1,11 +0,0 @@
-import type { URLPair } from './types.js';
-type AnalyzedUrlList = {
-    hasAuth: boolean;
-    hasNoSSL: boolean;
-};
-/**
- *
- * @param list
- */
-export declare function analyzeUrlList(list: readonly URLPair[]): AnalyzedUrlList;
-export {};

package/dist/analyze.d.ts

@@ -1,14 +0,0 @@
-import type { Result, URLPair } from './types.js';
-import type { PageHook } from '@d-zero/puppeteer-screenshot';
-export interface AnalyzeOptions {
-    readonly hooks: readonly PageHook[];
-    readonly htmlDiffOnly?: boolean;
-    readonly limit?: number;
-    readonly debug?: boolean;
-}
-/**
- *
- * @param list
- * @param options
- */
-export declare function analyze(list: readonly URLPair[], options?: AnalyzeOptions): Promise<Result[]>;

package/dist/analyze.js

@@ -1,111 +0,0 @@
-import { writeFile, mkdir } from 'node:fs/promises';
-import path from 'node:path';
-import { deal } from '@d-zero/puppeteer-dealer';
-import c from 'ansi-colors';
-import { analyzeUrlList } from './analize-url.js';
-import { diffImages } from './diff-images.js';
-import { diffTree } from './diff-tree.js';
-import { getData } from './get-data.js';
-import { score } from './output-utils.js';
-/**
- *
- * @param list
- * @param options
- */
-export async function analyze(list, options) {
-    const results = [];
-    const dir = path.resolve(process.cwd(), '.archaeologist');
-    await mkdir(dir, { recursive: true }).catch(() => { });
-    const urlInfo = analyzeUrlList(list);
-    const useOldMode = urlInfo.hasAuth || urlInfo.hasNoSSL;
-    await deal(list.map(([urlA]) => ({ id: null, url: urlA })), (_, done, total) => {
-        return `${c.bold.magenta('🕵️  Archaeologist')} ${done}/${total}`;
-    }, {
-        async deal(page, _, urlA, logger, index) {
-            const urlPair = list.find(([url]) => url === urlA);
-            if (!urlPair) {
-                throw new Error(`Failed to find urlPair: ${urlA}`);
-            }
-            const dataPair = [];
-            for (const url of urlPair) {
-                const data = await getData(page, url, {
-                    ...options,
-                }, logger);
-                dataPair.push(data);
-                await delay(600);
-            }
-            const [a, b] = dataPair;
-            if (!a || !b) {
-                throw new Error('Failed to get screenshots');
-            }
-            const screenshotResult = {};
-            const outputUrl = 'vs ' + c.gray(urlPair[1]);
-            for (const [name, screenshotA] of Object.entries(a.screenshots)) {
-                const screenshotB = b.screenshots[name];
-                const sizeName = c.bgMagenta(` ${name} `);
-                const id = `${index}_${name}`;
-                if (!screenshotB) {
-                    throw new Error(`Failed to get screenshotB: ${id}`);
-                }
-                const imageDiff = await diffImages(screenshotA, screenshotB, (phase, data) => {
-                    switch (phase) {
-                        case 'create': {
-                            logger(`${sizeName} ${outputUrl} 🖼️ Create images`);
-                            break;
-                        }
-                        case 'resize': {
-                            const { width, height } = data;
-                            logger(`${sizeName} ${outputUrl} ↔️ Resize images to ${width}x${height}`);
-                            break;
-                        }
-                        case 'diff': {
-                            logger(`${sizeName} ${outputUrl} 📊 Compare images`);
-                            break;
-                        }
-                    }
-                });
-                let image = null;
-                if (imageDiff) {
-                    logger(`${sizeName} ${outputUrl} 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
-                    await delay(1500);
-                    await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
-                    await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
-                    const outFilePath = path.resolve(dir, `${id}_diff.png`);
-                    logger(`${sizeName} ${outputUrl} 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
-                    await writeFile(outFilePath, imageDiff.images.diff);
-                    image = {
-                        matches: imageDiff.matches,
-                        file: outFilePath,
-                    };
-                }
-                const htmlDiff = diffTree(a.url, b.url, screenshotA.domTree, screenshotB.domTree);
-                const outFilePath = path.resolve(dir, `${id}_html.diff`);
-                await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
-                screenshotResult[name] = {
-                    image,
-                    dom: {
-                        matches: htmlDiff.matches,
-                        diff: htmlDiff.changed ? htmlDiff.result : null,
-                        file: outFilePath,
-                    },
-                };
-            }
-            const result = {
-                target: [a.url, b.url],
-                screenshots: screenshotResult,
-            };
-            results.push(result);
-        },
-    }, {
-        ...options,
-        headless: useOldMode ? 'shell' : true,
-    });
-    return results;
-}
-/**
- *
- * @param ms
- */
-function delay(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-}

package/dist/archaeologist.d.ts

@@ -1,10 +0,0 @@
-import type { AnalyzeOptions } from './analyze.js';
-import type { URLPair } from './types.js';
-export interface ArchaeologistOptions extends AnalyzeOptions {
-}
-/**
- *
- * @param list
- * @param options
- */
-export declare function archaeologist(list: readonly URLPair[], options?: ArchaeologistOptions): Promise<void>;

package/dist/archaeologist.js

@@ -1,23 +0,0 @@
-import c from 'ansi-colors';
-import { analyze } from './analyze.js';
-import { score } from './output-utils.js';
-/**
- *
- * @param list
- * @param options
- */
-export async function archaeologist(list, options) {
-    const results = await analyze(list, options);
-    const output = [];
-    for (const result of results) {
-        output.push(c.gray(`${result.target.join(' vs ')}`));
-        for (const [sizeName, { image, dom }] of Object.entries(result.screenshots)) {
-            if (image) {
-                const { matches, file } = image;
-                output.push(`  ${c.bgMagenta(` ${sizeName} `)} ${score(matches, 0.9)} ${file}`);
-            }
-            output.push(`  ${c.bgBlueBright(' HTML ')}: ${score(dom.matches, 0.995)} ${dom.file}`);
-        }
-    }
-    process.stdout.write(output.join('\n') + '\n');
-}

package/dist/freeze.d.ts

@@ -1,13 +0,0 @@
-import type { PageData } from './types.js';
-import type { PageHook } from '@d-zero/puppeteer-screenshot';
-export interface FreezeOptions {
-    readonly hooks: readonly PageHook[];
-    readonly limit?: number;
-    readonly debug?: boolean;
-}
-/**
- *
- * @param list
- * @param options
- */
-export declare function freeze(list: readonly string[], options?: FreezeOptions): Promise<PageData[]>;

package/dist/freeze.js

@@ -1,34 +0,0 @@
-import { mkdir } from 'node:fs/promises';
-import path from 'node:path';
-import { deal } from '@d-zero/puppeteer-dealer';
-import { delay } from '@d-zero/shared/delay';
-import c from 'ansi-colors';
-import { analyzeUrlList } from './analize-url.js';
-import { getData } from './get-data.js';
-/**
- *
- * @param list
- * @param options
- */
-export async function freeze(list, options) {
-    const results = [];
-    const dir = path.resolve(process.cwd(), '.archaeologist');
-    await mkdir(dir, { recursive: true }).catch(() => { });
-    const urlInfo = analyzeUrlList(list);
-    const useOldMode = urlInfo.hasAuth || urlInfo.hasNoSSL;
-    await deal(list.map((url) => ({ id: null, url })), (_, done, total) => {
-        return `${c.bold.magenta('🕵️  Archaeologist Freeze❄️')} ${done}/${total}`;
-    }, {
-        async deal(page, _, url, logger) {
-            const data = await getData(page, url, {
-                ...options,
-            }, logger);
-            await delay(600);
-            results.push(data);
-        },
-    }, {
-        ...options,
-        headless: useOldMode ? 'shell' : true,
-    });
-    return results;
-}

package/dist/output-utils.d.ts

@@ -1,6 +0,0 @@
-/**
- *
- * @param matches
- * @param threshold
- */
-export declare function score(matches: number, threshold: number): string;

package/dist/output-utils.js

@@ -1,11 +0,0 @@
-import c from 'ansi-colors';
-/**
- *
- * @param matches
- * @param threshold
- */
-export function score(matches, threshold) {
-    const color = matches > threshold ? c.green : c.red;
-    const num = (matches * 100).toFixed(1);
-    return c.bold(color(`${num}%`));
-}