@d-zero/archaeologist 1.0.0-alpha.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 D-ZERO Co., Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,34 @@
+# `@d-zero/archaeologist`
+
+**🕵️ Archaeologist: アーキオロジスト**
+
+ウェブサイトの本番環境と開発環境や、新旧のページの比較するためのツールです。
+
+- Puppeteerを実行してページのスクリーンショットを撮影します
+- スクリーンショットはデスクトップとモバイルの2つのサイズでそれぞれ撮影します
+- スクリーンショットは画像差分（ビジュアルリグレッション）を検出・出力します
+- HTMLの差分も検出します
+
+## CLI
+
+```sh
+npx @d-zero/archaeologist -f <filepath>
+```
+
+URLリストを持つファイルを指定して実行します。
+
+ファイルの先頭には比較対象のホストを指定します。[Frontmatter](https://jekyllrb.com/docs/front-matter/)形式で`comparisonHost`に指定します。
+
+```txt
+---
+comparisonHost: https://stage.example.com
+---
+
+https://example.com
+https://example.com/a
+https://example.com/b
+https://example.com/c
+https://example.com/xyz/001
+```
+
+実行した結果は`.archaeologist`ディレクトリに保存されます。

package/dist/analyze.d.ts

@@ -0,0 +1,2 @@
+import type { Result, URLPair } from './types.js';
+export declare function analyze(list: readonly URLPair[]): Promise<Result[]>;

package/dist/analyze.js

@@ -0,0 +1,132 @@
+import { writeFile, mkdir } from 'node:fs/promises';
+import path from 'node:path';
+import { deal } from '@d-zero/dealer';
+import c from 'ansi-colors';
+import puppeteer from 'puppeteer';
+import { diffImages } from './diff-images.js';
+import { diffTree } from './diff-tree.js';
+import { getData } from './get-data.js';
+import { label, score } from './output-utils.js';
+export async function analyze(list) {
+    const browser = await puppeteer.launch({
+        args: [
+            //
+            '--lang=ja',
+            '--no-zygote',
+            '--ignore-certificate-errors',
+        ],
+    });
+    const results = [];
+    const dir = path.resolve(process.cwd(), '.archaeologist');
+    await mkdir(dir, { recursive: true }).catch(() => { });
+    await deal(list, 
+    //
+    async (urlPair, update, index) => {
+        const page = await browser.newPage();
+        page.setDefaultNavigationTimeout(0);
+        await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36');
+        await page.setExtraHTTPHeaders({
+            'Accept-Language': 'ja-JP',
+        });
+        return async () => {
+            const dataPair = [];
+            for (const url of urlPair) {
+                const data = await getData(page, url, (phase, data) => {
+                    const outputUrl = c.gray(url);
+                    const sizeName = label(data.name);
+                    switch (phase) {
+                        case 'setViewport': {
+                            const { width } = data;
+                            update(`%braille% ${outputUrl} ${sizeName}: ↔️ Change viewport size to ${width}px`);
+                            break;
+                        }
+                        case 'load': {
+                            const { type } = data;
+                            update(`%braille% ${outputUrl} ${sizeName}: %earth% ${type === 'open' ? 'Open' : 'Reload'} page`);
+                            break;
+                        }
+                        case 'scroll': {
+                            update(`%braille% ${outputUrl} ${sizeName}: %propeller% Scroll the page`);
+                            break;
+                        }
+                        case 'screenshotStart': {
+                            update(`%braille% ${outputUrl} ${sizeName}: 📸 Take a screenshot`);
+                            break;
+                        }
+                        case 'screenshotEnd': {
+                            const { binary } = data;
+                            update(`%braille% ${outputUrl} ${sizeName}: 📸 Screenshot taken (${binary.length} bytes)`);
+                            break;
+                        }
+                    }
+                });
+                dataPair.push(data);
+                await delay(600);
+            }
+            const [a, b] = dataPair;
+            if (!a || !b) {
+                throw new Error('Failed to get screenshots');
+            }
+            const screenshotResult = {};
+            const outputUrl = c.gray(urlPair.join(' vs '));
+            for (const [name, screenshotA] of Object.entries(a.screenshots)) {
+                const screenshotB = b.screenshots[name];
+                const sizeName = label(name);
+                const id = `${index}_${name}`;
+                if (!screenshotB) {
+                    throw new Error(`Failed to get screenshotB: ${id}`);
+                }
+                const imageDiff = await diffImages(screenshotA, screenshotB, (phase, data) => {
+                    switch (phase) {
+                        case 'create': {
+                            update(`%braille% ${outputUrl} ${sizeName}: 🖼️ Create images`);
+                            break;
+                        }
+                        case 'resize': {
+                            const { width, height } = data;
+                            update(`%braille% ${outputUrl} ${sizeName}: ↔️ Resize images to ${width}x${height}`);
+                            break;
+                        }
+                        case 'diff': {
+                            update(`%braille% ${outputUrl} ${sizeName}: 📊 Compare images`);
+                            break;
+                        }
+                    }
+                });
+                update(`%braille% ${outputUrl} ${sizeName}: 🧩 Matches ${score(imageDiff.matches, 0.9)}`);
+                await delay(1500);
+                await writeFile(path.resolve(dir, `${id}_a.png`), imageDiff.images.a);
+                await writeFile(path.resolve(dir, `${id}_b.png`), imageDiff.images.b);
+                const outFilePath = path.resolve(dir, `${id}_diff.png`);
+                update(`%braille% ${outputUrl} ${sizeName}: 📊 Save diff image to ${path.relative(dir, outFilePath)}`);
+                await writeFile(outFilePath, imageDiff.images.diff);
+                screenshotResult[name] = {
+                    matches: imageDiff.matches,
+                    file: outFilePath,
+                };
+            }
+            const htmlDiff = diffTree(a.serializedHtml, b.serializedHtml);
+            const outFilePath = path.resolve(dir, `${index}_html.diff`);
+            await writeFile(outFilePath, htmlDiff.result, { encoding: 'utf8' });
+            const result = {
+                target: [a.url, b.url],
+                screenshots: screenshotResult,
+                html: {
+                    matches: htmlDiff.matches,
+                    diff: htmlDiff.changed ? htmlDiff.result : null,
+                    file: outFilePath,
+                },
+            };
+            results.push(result);
+        };
+    }, {
+        header(_, done, total) {
+            return `${c.bold.magenta('🕵️  Archaeologist')} ${done}/${total}`;
+        },
+    });
+    await browser.close();
+    return results;
+}
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/archaeologist.d.ts

@@ -0,0 +1,2 @@
+import type { URLPair } from './types.js';
+export declare function archaeologist(list: readonly URLPair[]): Promise<void>;

package/dist/archaeologist.js

@@ -0,0 +1,15 @@
+import c from 'ansi-colors';
+import { analyze } from './analyze.js';
+import { label, score } from './output-utils.js';
+export async function archaeologist(list) {
+    const results = await analyze(list);
+    const output = [];
+    for (const result of results) {
+        output.push(c.gray(`${result.target.join(' vs ')}`));
+        for (const [sizeName, { matches, file }] of Object.entries(result.screenshots)) {
+            output.push(`  ${label(sizeName)} ${score(matches, 0.9)} ${file}`);
+        }
+        output.push(`  ${label('HTML', c.bgBlueBright)}: ${score(result.html.matches, 0.995)} ${result.html.file}`);
+    }
+    process.stdout.write(output.join('\n') + '\n');
+}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import { toList } from '@d-zero/readtext/list';
+import fm from 'front-matter';
+import minimist from 'minimist';
+import { archaeologist } from './archaeologist.js';
+const cli = minimist(process.argv.slice(2), {
+    alias: {
+        f: 'listfile',
+    },
+});
+if (cli.listfile) {
+    const fileContent = await fs.readFile(cli.listfile, 'utf8');
+    const content = 
+    // @ts-ignore
+    fm(fileContent);
+    const urlList = toList(content.body);
+    const pairList = urlList.map((urlStr) => {
+        const url = new URL(urlStr);
+        return [
+            url.toString(),
+            `${content.attributes.comparisonHost}${url.pathname}${url.search}`,
+        ];
+    });
+    await archaeologist(pairList);
+    process.exit(0);
+}
+process.stdout.write('Usage: archaeologist -f <listfile>\n');
+process.exit(1);

package/dist/diff-images.d.ts

@@ -0,0 +1,28 @@
+/// <reference types="node" resolution-mode="require"/>
+import type { Screenshot } from '@d-zero/puppeteer-screenshot';
+export type DiffImagesPhase = {
+    create: {
+        a: Buffer;
+        b: Buffer;
+    };
+    resize: {
+        a: Buffer;
+        b: Buffer;
+        width: number;
+        height: number;
+    };
+    diff: {
+        a: Buffer;
+        b: Buffer;
+    };
+};
+type DiffImagesListener = (phase: keyof DiffImagesPhase, data: DiffImagesPhase[keyof DiffImagesPhase]) => void;
+export declare function diffImages(dataA: Screenshot, dataB: Screenshot, listener: DiffImagesListener): Promise<{
+    matches: number;
+    images: {
+        a: Buffer;
+        b: Buffer;
+        diff: Buffer;
+    };
+}>;
+export {};

package/dist/diff-images.js

@@ -0,0 +1,41 @@
+import Jimp from 'jimp';
+import pixelmatch from 'pixelmatch';
+import { PNG } from 'pngjs';
+export async function diffImages(dataA, dataB, listener) {
+    listener('create', { a: dataA.binary, b: dataB.binary });
+    const imgA = PNG.sync.read(dataA.binary);
+    const imgB = PNG.sync.read(dataB.binary);
+    const width = Math.max(imgA.width, imgB.width);
+    const height = Math.max(imgA.height, imgB.height);
+    listener('resize', { a: dataA.binary, b: dataB.binary, width, height });
+    const resizedA = await resizeImg(dataA.binary, width, height);
+    const resizedB = await resizeImg(dataB.binary, width, height);
+    listener('diff', { a: resizedA, b: resizedB });
+    const imgA_ = PNG.sync.read(resizedA);
+    const imgB_ = PNG.sync.read(resizedB);
+    const diffImage = new PNG({ width, height });
+    const matcheBytes = pixelmatch(imgA_.data, imgB_.data, diffImage.data, width, height);
+    const matches = 1 - matcheBytes / (width * height);
+    const imageABuffer = PNG.sync.write(imgA_);
+    const imageBBuffer = PNG.sync.write(imgB_);
+    const imageDiffBuffer = PNG.sync.write(diffImage);
+    return {
+        matches,
+        images: {
+            a: imageABuffer,
+            b: imageBBuffer,
+            diff: imageDiffBuffer,
+        },
+    };
+}
+async function resizeImg(bin, width, height) {
+    const img = await Jimp.read(bin);
+    img.contain(width, height, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
+    return new Promise((resolve, reject) => {
+        img.getBuffer(Jimp.MIME_PNG, (err, buffer) => {
+            if (err)
+                reject(err);
+            resolve(buffer);
+        });
+    });
+}

package/dist/diff-tree-color.d.ts

@@ -0,0 +1 @@
+export declare function diffTreeColor(diff: string): string;

package/dist/diff-tree-color.js

@@ -0,0 +1,15 @@
+import c from 'ansi-colors';
+export function diffTreeColor(diff) {
+    return diff
+        .split('\n')
+        .map((line) => {
+        if (line.startsWith('+')) {
+            return c.green(line);
+        }
+        if (line.startsWith('-')) {
+            return c.red(line);
+        }
+        return c.gray(line);
+    })
+        .join('\n');
+}

package/dist/diff-tree.d.ts

@@ -0,0 +1,7 @@
+export declare function diffTree(dataA: string, dataB: string): {
+    changed: boolean;
+    maxLine: number;
+    changedLines: number;
+    matches: number;
+    result: string;
+};

package/dist/diff-tree.js

@@ -0,0 +1,32 @@
+import { diffLines } from 'diff';
+export function diffTree(dataA, dataB) {
+    const changes = diffLines(dataA, dataB);
+    const lineA = dataA.split('\n').length;
+    const lineB = dataB.split('\n').length;
+    const maxLine = Math.max(lineA, lineB);
+    let changedLines = 0;
+    const result = changes
+        .map((change) => {
+        if (change.added) {
+            changedLines++;
+            return `+${change.value}`;
+        }
+        if (change.removed) {
+            changedLines++;
+            return `-${change.value}`;
+        }
+        return change.value
+            .split('\n')
+            .map((line) => (line.trim() ? ` ${line}` : line))
+            .join('\n');
+    })
+        .filter(Boolean)
+        .join('');
+    return {
+        changed: changedLines > 0,
+        maxLine,
+        changedLines,
+        matches: 1 - changedLines / maxLine,
+        result,
+    };
+}

package/dist/get-data.d.ts

@@ -0,0 +1,4 @@
+import type { PageData } from './types.js';
+import type { Listener } from '@d-zero/puppeteer-screenshot';
+import type { Page } from 'puppeteer';
+export declare function getData(page: Page, url: string, listener: Listener): Promise<PageData>;

package/dist/get-data.js

@@ -0,0 +1,24 @@
+import { distill } from '@d-zero/html-distiller';
+import { screenshot } from '@d-zero/puppeteer-screenshot';
+export async function getData(page, url, listener) {
+    const screenshots = await screenshot(page, url, {
+        sizes: {
+            desktop: {
+                width: 1280,
+            },
+            mobile: {
+                width: 375,
+                resolution: 2,
+            },
+        },
+        listener,
+    });
+    const html = await page.content();
+    const serializedHtmlTree = distill(html).tree;
+    const serializedHtml = JSON.stringify(serializedHtmlTree, null, 2);
+    return {
+        url,
+        serializedHtml,
+        screenshots,
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { archaeologist } from './archaeologist.js';

package/dist/index.js

@@ -0,0 +1 @@
+export { archaeologist } from './archaeologist.js';

package/dist/output-utils.d.ts

@@ -0,0 +1,3 @@
+import c from 'ansi-colors';
+export declare function label(str: string, color?: c.StyleFunction): string;
+export declare function score(matches: number, threshold: number): string;

package/dist/output-utils.js

@@ -0,0 +1,9 @@
+import c from 'ansi-colors';
+export function label(str, color = c.bgMagenta) {
+    return color(` ${str} `);
+}
+export function score(matches, threshold) {
+    const color = matches > threshold ? c.green : c.red;
+    const num = (matches * 100).toFixed(1);
+    return c.bold(color(`${num}%`));
+}

package/dist/types.d.ts

@@ -0,0 +1,19 @@
+import type { Screenshot } from '@d-zero/puppeteer-screenshot';
+export type PageData = {
+    url: string;
+    serializedHtml: string;
+    screenshots: Record<string, Screenshot>;
+};
+export type URLPair = readonly [urlA: string, urlB: string];
+export type Result = {
+    target: [urlA: string, urlB: string];
+    screenshots: Record<string, {
+        matches: number;
+        file: string;
+    }>;
+    html: {
+        matches: number;
+        diff: string | null;
+        file: string;
+    };
+};

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/utils.d.ts

@@ -0,0 +1,2 @@
+export declare function label(str: string): string;
+export declare function score(matches: number, threshold: number): string;

package/dist/utils.js

@@ -0,0 +1,9 @@
+import c from 'ansi-colors';
+export function label(str) {
+    return c.bgMagenta(` ${str} `);
+}
+export function score(matches, threshold) {
+    const color = matches > threshold ? c.green : c.red;
+    const num = (matches * 100).toFixed(1);
+    return c.bold(color(`${num}%`));
+}

package/package.json

@@ -0,0 +1,49 @@
+{
+	"name": "@d-zero/archaeologist",
+	"version": "1.0.0-alpha.2",
+	"description": "Uncover visual and HTML differences in web pages with precision",
+	"author": "D-ZERO",
+	"license": "MIT",
+	"private": false,
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": "./dist/index.js",
+			"types": "./dist/index.d.ts"
+		}
+	},
+	"bin": {
+		"archaeologist": "./dist/cli.js"
+	},
+	"files": [
+		"dist"
+	],
+	"scripts": {
+		"build": "tsc",
+		"clean": "tsc --build --clean"
+	},
+	"dependencies": {
+		"@d-zero/dealer": "^1.0.0-alpha.2",
+		"@d-zero/html-distiller": "^1.0.0-alpha.2",
+		"@d-zero/puppeteer-screenshot": "^1.0.0-alpha.2",
+		"@d-zero/readtext": "^1.0.0-alpha.2",
+		"ansi-colors": "4.1.3",
+		"diff": "5.2.0",
+		"front-matter": "4.0.2",
+		"jimp": "0.22.12",
+		"minimist": "1.2.8",
+		"pixelmatch": "5.3.0",
+		"pngjs": "7.0.0",
+		"puppeteer": "22.6.4"
+	},
+	"devDependencies": {
+		"@types/diff": "5.0.9",
+		"@types/pixelmatch": "5.2.6",
+		"@types/pngjs": "6.0.4",
+		"puppeteer": "22.6.4"
+	},
+	"gitHead": "802df4fdb85892c8fab3bf1b376755e25ca2ca47"
+}