@d-zero/archaeologist 1.0.3 → 1.1.1

package/README.md

@@ -12,11 +12,19 @@
 ## CLI
 
 ```sh
-npx @d-zero/archaeologist -f <filepath>
+npx @d-zero/archaeologist -f <filepath> [--limit <number>] [--debug]
 ```
 
 URLリストを持つファイルを指定して実行します。
 
+### オプション
+
+- `-f, --file <filepath>`: URLリストを持つファイルのパス（必須）
+- `--limit <number>`: 並列実行数の上限（デフォルト: 10）
+- `--debug`: デバッグモード（デフォルト: false）
+
+### ファイルフォーマット
+
 ファイルの先頭には比較対象のホストを指定します。[Frontmatter](https://jekyllrb.com/docs/front-matter/)形式で`comparisonHost`に指定します。
 
 ```txt

package/dist/analyze.d.ts

@@ -1,3 +1,9 @@
 import type { Result, URLPair } from './types.js';
 import type { PageHook } from '@d-zero/puppeteer-screenshot';
-export declare function analyze(list: readonly URLPair[], hooks: readonly PageHook[]): Promise<Result[]>;
+export interface AnalyzeOptions {
+    readonly hooks: readonly PageHook[];
+    readonly htmlDiffOnly?: boolean;
+    readonly limit?: number;
+    readonly debug?: boolean;
+}
+export declare function analyze(list: readonly URLPair[], options?: AnalyzeOptions): Promise<Result[]>;

package/dist/analyze.js

@@ -8,7 +8,7 @@ import { diffImages } from './diff-images.js';
 import { diffTree } from './diff-tree.js';
 import { getData } from './get-data.js';
 import { label, score } from './output-utils.js';
-export async function analyze(list, hooks) {
+export async function analyze(list, options) {
     const urlInfo = analyzeUrlList(list);
     const useOldMode = urlInfo.hasAuth && urlInfo.hasNoSSL;
     const browser = await puppeteer.launch({
@@ -35,40 +35,9 @@ export async function analyze(list, hooks) {
         return async () => {
             const dataPair = [];
             for (const url of urlPair) {
-                const data = await getData(page, url, hooks, (phase, data) => {
-                    const outputUrl = c.gray(url);
-                    const sizeName = label(data.name);
-                    switch (phase) {
-                        case 'setViewport': {
-                            const { width } = data;
-                            update(`%braille% ${outputUrl} ${sizeName}: ↔️ Change viewport size to ${width}px`);
-                            break;
-                        }
-                        case 'load': {
-                            const { type } = data;
-                            update(`%braille% ${outputUrl} ${sizeName}: %earth% ${type === 'open' ? 'Open' : 'Reload'} page`);
-                            break;
-                        }
-                        case 'hook': {
-                            const { message } = data;
-                            update(`%braille% ${outputUrl} ${sizeName}: ${message}`);
-                            break;
-                        }
-                        case 'scroll': {
-                            update(`%braille% ${outputUrl} ${sizeName}: %propeller% Scroll the page`);
-                            break;
-                        }
-                        case 'screenshotStart': {
-                            update(`%braille% ${outputUrl} ${sizeName}: 📸 Take a screenshot`);
-                            break;
-                        }
-                        case 'screenshotEnd': {
-                            const { binary } = data;
-                            update(`%braille% ${outputUrl} ${sizeName}: 📸 Screenshot taken (${binary.length} bytes)`);
-                            break;
-                        }
-                    }
-                });
+                const data = await getData(page, url, {
+                    ...options,
+                }, update);
                 dataPair.push(data);
                 await delay(600);
             }
@@ -102,33 +71,41 @@ export async function analyze(list, hooks) {
                         }
                     }
                 });
-                update(`%braille% ${outputUrl} ${sizeName}: 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
-                await delay(1500);
-                await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
-                await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
-                const outFilePath = path.resolve(dir, `${id}_diff.png`);
-                update(`%braille% ${outputUrl} ${sizeName}: 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
-                await writeFile(outFilePath, imageDiff.images.diff);
+                let image = null;
+                if (imageDiff) {
+                    update(`%braille% ${outputUrl} ${sizeName}: 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
+                    await delay(1500);
+                    await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
+                    await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
+                    const outFilePath = path.resolve(dir, `${id}_diff.png`);
+                    update(`%braille% ${outputUrl} ${sizeName}: 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
+                    await writeFile(outFilePath, imageDiff.images.diff);
+                    image = {
+                        matches: imageDiff.matches,
+                        file: outFilePath,
+                    };
+                }
+                const htmlDiff = diffTree(a.url, b.url, screenshotA.domTree, screenshotB.domTree);
+                const outFilePath = path.resolve(dir, `${id}_html.diff`);
+                await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
                 screenshotResult[name] = {
-                    matches: imageDiff.matches,
-                    file: outFilePath,
+                    image,
+                    dom: {
+                        matches: htmlDiff.matches,
+                        diff: htmlDiff.changed ? htmlDiff.result : null,
+                        file: outFilePath,
+                    },
                 };
             }
-            const htmlDiff = diffTree(a.serializedHtml, b.serializedHtml);
-            const outFilePath = path.resolve(dir, `${index}_html.diff`);
-            await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
             const result = {
                 target: [a.url, b.url],
                 screenshots: screenshotResult,
-                html: {
-                    matches: htmlDiff.matches,
-                    diff: htmlDiff.changed ? htmlDiff.result : null,
-                    file: outFilePath,
-                },
             };
             results.push(result);
         };
     }, {
+        limit: options?.limit,
+        debug: options?.debug,
         header(_, done, total) {
             return `${c.bold.magenta('🕵️  Archaeologist')} ${done}/${total}`;
         },

package/dist/archaeologist.d.ts

@@ -1,3 +1,5 @@
+import type { AnalyzeOptions } from './analyze.js';
 import type { URLPair } from './types.js';
-import type { PageHook } from '@d-zero/puppeteer-screenshot';
-export declare function archaeologist(list: readonly URLPair[], pageHooks?: readonly PageHook[]): Promise<void>;
+export interface ArchaeologistOptions extends AnalyzeOptions {
+}
+export declare function archaeologist(list: readonly URLPair[], options?: ArchaeologistOptions): Promise<void>;

package/dist/archaeologist.js

@@ -1,15 +1,18 @@
 import c from 'ansi-colors';
 import { analyze } from './analyze.js';
 import { label, score } from './output-utils.js';
-export async function archaeologist(list, pageHooks) {
-    const results = await analyze(list, pageHooks ?? []);
+export async function archaeologist(list, options) {
+    const results = await analyze(list, options);
     const output = [];
     for (const result of results) {
         output.push(c.gray(`${result.target.join(' vs ')}`));
-        for (const [sizeName, { matches, file }] of Object.entries(result.screenshots)) {
-            output.push(`  ${label(sizeName)} ${score(matches, 0.9)} ${file}`);
+        for (const [sizeName, { image, dom }] of Object.entries(result.screenshots)) {
+            if (image) {
+                const { matches, file } = image;
+                output.push(`  ${label(sizeName)} ${score(matches, 0.9)} ${file}`);
+            }
+            output.push(`  ${label('HTML', c.bgBlueBright)}: ${score(dom.matches, 0.995)} ${dom.file}`);
         }
-        output.push(`  ${label('HTML', c.bgBlueBright)}: ${score(result.html.matches, 0.995)} ${result.html.file}`);
     }
     process.stdout.write(output.join('\n') + '\n');
 }

package/dist/cli.js

@@ -7,10 +7,15 @@ const cli = minimist(process.argv.slice(2), {
         f: 'listfile',
     },
 });
-if (cli.listfile) {
-    const { pairList, pageHooks } = await readConfig(cli.listfile);
-    await archaeologist(pairList, pageHooks);
+if (cli.listfile?.length) {
+    const { pairList, hooks } = await readConfig(cli.listfile);
+    await archaeologist(pairList, {
+        hooks,
+        limit: cli.limit ? Number.parseInt(cli.limit) : undefined,
+        debug: !!cli.debug,
+        htmlDiffOnly: !!cli.htmlDiffOnly,
+    });
     process.exit(0);
 }
-process.stdout.write('Usage: archaeologist -f <listfile>\n');
+process.stderr.write('Usage: archaeologist -f <listfile> [--limit <number>]\n');
 process.exit(1);

package/dist/diff-images.d.ts

@@ -1,19 +1,18 @@
-/// <reference types="node" resolution-mode="require"/>
 import type { Screenshot } from '@d-zero/puppeteer-screenshot';
 export type DiffImagesPhase = {
     create: {
-        a: Buffer;
-        b: Buffer;
+        a: Uint8Array;
+        b: Uint8Array;
     };
     resize: {
-        a: Buffer;
-        b: Buffer;
+        a: Uint8Array;
+        b: Uint8Array;
         width: number;
         height: number;
     };
     diff: {
-        a: Buffer;
-        b: Buffer;
+        a: Uint8Array;
+        b: Uint8Array;
     };
 };
 type DiffImagesListener = (phase: keyof DiffImagesPhase, data: DiffImagesPhase[keyof DiffImagesPhase]) => void;
@@ -24,5 +23,5 @@ export declare function diffImages(dataA: Screenshot, dataB: Screenshot, listene
         b: Buffer;
         diff: Buffer;
     };
-}>;
+} | null>;
 export {};

package/dist/diff-images.js

@@ -1,15 +1,18 @@
-import Jimp from 'jimp';
+import { Jimp, HorizontalAlign, VerticalAlign, JimpMime } from 'jimp';
 import pixelmatch from 'pixelmatch';
 import { PNG } from 'pngjs';
 export async function diffImages(dataA, dataB, listener) {
+    if (!dataA.binary || !dataB.binary) {
+        return null;
+    }
     listener('create', { a: dataA.binary, b: dataB.binary });
-    const imgA = PNG.sync.read(dataA.binary);
-    const imgB = PNG.sync.read(dataB.binary);
+    const imgA = PNG.sync.read(Buffer.from(dataA.binary));
+    const imgB = PNG.sync.read(Buffer.from(dataB.binary));
     const width = Math.max(imgA.width, imgB.width);
     const height = Math.max(imgA.height, imgB.height);
     listener('resize', { a: dataA.binary, b: dataB.binary, width, height });
-    const resizedA = await resizeImg(dataA.binary, width, height);
-    const resizedB = await resizeImg(dataB.binary, width, height);
+    const resizedA = await resizeImg(Buffer.from(dataA.binary), width, height);
+    const resizedB = await resizeImg(Buffer.from(dataB.binary), width, height);
     listener('diff', { a: resizedA, b: resizedB });
     const imgA_ = PNG.sync.read(resizedA);
     const imgB_ = PNG.sync.read(resizedB);
@@ -30,12 +33,10 @@ export async function diffImages(dataA, dataB, listener) {
 }
 async function resizeImg(bin, width, height) {
     const img = await Jimp.read(bin);
-    img.contain(width, height, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
-    return new Promise((resolve, reject) => {
-        img.getBuffer(Jimp.MIME_PNG, (err, buffer) => {
-            if (err)
-                reject(err);
-            resolve(buffer);
-        });
+    img.contain({
+        w: width,
+        h: height,
+        align: HorizontalAlign.LEFT | VerticalAlign.TOP,
     });
+    return img.getBuffer(JimpMime.png);
 }

package/dist/diff-tree.d.ts

@@ -1,7 +1,6 @@
-export declare function diffTree(dataA: string, dataB: string): {
+export declare function diffTree(urlA: string, urlB: string, dataA: string, dataB: string): {
     changed: boolean;
     maxLine: number;
-    changedLines: number;
     matches: number;
     result: string;
 };

package/dist/diff-tree.js

@@ -1,32 +1,18 @@
-import { diffLines } from 'diff';
-export function diffTree(dataA, dataB) {
-    const changes = diffLines(dataA, dataB);
+import { createTwoFilesPatch } from 'diff';
+import parse from 'parse-diff';
+export function diffTree(urlA, urlB, dataA, dataB) {
+    const result = createTwoFilesPatch(urlA, urlB, dataA, dataB);
+    const info = parse(result)[0];
+    if (!info) {
+        throw new Error('Failed to parse diff');
+    }
     const lineA = dataA.split('\n').length;
     const lineB = dataB.split('\n').length;
     const maxLine = Math.max(lineA, lineB);
-    let changedLines = 0;
-    const result = changes
-        .map((change) => {
-        if (change.added) {
-            changedLines++;
-            return `+${change.value}`;
-        }
-        if (change.removed) {
-            changedLines++;
-            return `-${change.value}`;
-        }
-        return change.value
-            .split('\n')
-            .map((line) => (line.trim() ? ` ${line}` : line))
-            .join('\n');
-    })
-        .filter(Boolean)
-        .join('');
     return {
-        changed: changedLines > 0,
+        changed: dataA !== dataB,
         maxLine,
-        changedLines,
-        matches: 1 - changedLines / maxLine,
+        matches: 1 - Math.abs((info.additions - info.deletions) / maxLine),
         result,
     };
 }

package/dist/get-data.d.ts

@@ -1,4 +1,8 @@
 import type { PageData } from './types.js';
-import type { Listener, PageHook } from '@d-zero/puppeteer-screenshot';
+import type { PageHook } from '@d-zero/puppeteer-screenshot';
 import type { Page } from 'puppeteer';
-export declare function getData(page: Page, url: string, hooks: readonly PageHook[], listener: Listener): Promise<PageData>;
+export interface GetDataOptions {
+    readonly hooks?: readonly PageHook[];
+    readonly htmlDiffOnly?: boolean;
+}
+export declare function getData(page: Page, url: string, options: GetDataOptions, update: (log: string) => void): Promise<PageData>;

package/dist/get-data.js

@@ -1,6 +1,7 @@
 import { distill } from '@d-zero/html-distiller';
-import { screenshot } from '@d-zero/puppeteer-screenshot';
-export async function getData(page, url, hooks, listener) {
+import { screenshotListener, screenshot } from '@d-zero/puppeteer-screenshot';
+export async function getData(page, url, options, update) {
+    const htmlDiffOnly = options.htmlDiffOnly ?? false;
     const screenshots = await screenshot(page, url, {
         sizes: {
             desktop: {
@@ -11,15 +12,16 @@ export async function getData(page, url, hooks, listener) {
                 resolution: 2,
             },
         },
-        hooks,
-        listener,
+        hooks: options?.hooks ?? [],
+        listener: screenshotListener(update),
+        domOnly: htmlDiffOnly,
     });
-    const html = await page.content();
-    const serializedHtmlTree = distill(html).tree;
-    const serializedHtml = JSON.stringify(serializedHtmlTree, null, 2);
-    return {
-        url,
-        serializedHtml,
-        screenshots,
-    };
+    const data = { url, screenshots: {} };
+    for (const [sizeName, screenshot] of Object.entries(screenshots)) {
+        data.screenshots[sizeName] = {
+            ...screenshot,
+            domTree: JSON.stringify(distill(screenshot.dom).tree, null, 2),
+        };
+    }
+    return data;
 }

package/dist/read-config.d.ts

@@ -1,4 +1,4 @@
 export declare function readConfig(filePath: string): Promise<{
     pairList: [string, string][];
-    pageHooks: import("@d-zero/puppeteer-screenshot").PageHook[];
+    hooks: import("@d-zero/puppeteer-page-scan").PageHook[];
 }>;

package/dist/read-config.js

@@ -1,7 +1,8 @@
 import fs from 'node:fs/promises';
+import path from 'node:path';
+import { readPageHooks } from '@d-zero/puppeteer-page-scan';
 import { toList } from '@d-zero/readtext/list';
 import fm from 'front-matter';
-import { readHooks } from './read-hooks.js';
 export async function readConfig(filePath) {
     const fileContent = await fs.readFile(filePath, 'utf8');
     const content = 
@@ -15,9 +16,10 @@ export async function readConfig(filePath) {
             `${content.attributes.comparisonHost}${url.pathname}${url.search}`,
         ];
     });
-    const pageHooks = await readHooks(content.attributes?.hooks ?? [], filePath);
+    const baseDir = path.dirname(filePath);
+    const hooks = await readPageHooks(content.attributes?.hooks ?? [], baseDir);
     return {
         pairList,
-        pageHooks,
+        hooks,
     };
 }

package/dist/types.d.ts

@@ -2,19 +2,25 @@ export type { PageHook } from '@d-zero/puppeteer-screenshot';
 import type { Screenshot } from '@d-zero/puppeteer-screenshot';
 export type PageData = {
     url: string;
-    serializedHtml: string;
-    screenshots: Record<string, Screenshot>;
+    screenshots: Record<string, Screenshot & {
+        domTree: string;
+    }>;
 };
 export type URLPair = readonly [urlA: string, urlB: string];
 export type Result = {
     target: [urlA: string, urlB: string];
-    screenshots: Record<string, {
-        matches: number;
-        file: string;
-    }>;
-    html: {
-        matches: number;
-        diff: string | null;
-        file: string;
-    };
+    screenshots: Record<string, MediaResult>;
+};
+export type MediaResult = {
+    image: ImageResult | null;
+    dom: DOMResult;
+};
+export type ImageResult = {
+    matches: number;
+    file: string;
+};
+export type DOMResult = {
+    matches: number;
+    diff: string | null;
+    file: string;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/archaeologist",
-	"version": "1.0.3",
+	"version": "1.1.1",
 	"description": "Uncover visual and HTML differences in web pages with precision",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -26,24 +26,25 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/dealer": "1.0.1",
+		"@d-zero/dealer": "1.1.0",
 		"@d-zero/html-distiller": "1.0.0",
-		"@d-zero/puppeteer-screenshot": "1.0.3",
-		"@d-zero/readtext": "1.0.2",
+		"@d-zero/puppeteer-page-scan": "1.0.0",
+		"@d-zero/puppeteer-screenshot": "1.2.0",
+		"@d-zero/readtext": "1.1.0",
 		"ansi-colors": "4.1.3",
-		"diff": "5.2.0",
+		"diff": "7.0.0",
 		"front-matter": "4.0.2",
-		"jimp": "0.22.12",
+		"jimp": "1.6.0",
 		"minimist": "1.2.8",
-		"pixelmatch": "5.3.0",
+		"parse-diff": "0.11.1",
+		"pixelmatch": "6.0.0",
 		"pngjs": "7.0.0",
-		"puppeteer": "22.11.0"
+		"puppeteer": "23.5.0"
 	},
 	"devDependencies": {
-		"@types/diff": "5.2.1",
+		"@types/diff": "5.2.2",
 		"@types/pixelmatch": "5.2.6",
-		"@types/pngjs": "6.0.5",
-		"puppeteer": "22.11.0"
+		"@types/pngjs": "6.0.5"
 	},
-	"gitHead": "8181fac55ec45eb13c5a25e4187b7b3621f2e34c"
+	"gitHead": "e8f65086bf7c316dda6667f1173da8585a5ef19c"
 }