@d-zero/replicator 0.2.0 → 0.4.0

package/README.md

@@ -1,36 +1,80 @@
-# @d-zero/replicator
+# `@d-zero/replicator`
 
-Replicate web pages with all their resources to local directories
+ウェブページとそのリソースをレスポンシブ画像対応でローカルディレクトリに複製するツールです。
 
-## Installation
+## インストール
 
 ```bash
 npm install @d-zero/replicator
 ```
 
-## Usage
+## 使い方
 
 ### CLI
 
 ```bash
-npx @d-zero/replicator <url> -o <output-directory>
+npx @d-zero/replicator <url> -o <output-directory> [options]
 ```
 
-### Programmatic
+#### オプション
+
+- `-o, --output <dir>`: 出力ディレクトリ（必須）
+- `-t, --timeout <ms>`: リクエストタイムアウト（ミリ秒、デフォルト: 30000）
+- `-d, --devices <devices>`: デバイスプリセット（カンマ区切り、デフォルト: desktop-compact,mobile）
+- `-v, --verbose`: 詳細ログモード
+
+#### 利用可能なデバイスプリセット
+
+- `desktop`: 1400px幅
+- `tablet`: 768px幅
+- `mobile`: 375px幅（2倍解像度）
+- `desktop-hd`: 1920px幅
+- `desktop-compact`: 1280px幅
+- `mobile-large`: 414px幅（3倍解像度）
+- `mobile-small`: 320px幅（2倍解像度）
+
+#### 使用例
+
+```bash
+# デフォルトデバイス（desktop-compact, mobile）
+npx @d-zero/replicator https://example.com -o ./output
+
+# カスタムデバイス指定
+npx @d-zero/replicator https://example.com -o ./output --devices desktop,tablet,mobile
+
+# タイムアウト指定
+npx @d-zero/replicator https://example.com -o ./output --timeout 60000
+```
+
+### プログラマティック使用
 
 ```typescript
 import { replicate } from '@d-zero/replicator';
 
+// デフォルトデバイス
 await replicate('https://example.com', './output');
+
+// カスタムデバイス
+await replicate('https://example.com', './output', {
+	devices: {
+		desktop: { width: 1400 },
+		mobile: { width: 375, resolution: 2 },
+	},
+	timeout: 30000,
+	verbose: true,
+});
 ```
 
-## Features
+## 機能
 
-- Download HTML pages preserving directory structure
-- Fetch all related resources (CSS, JS, images, etc.)
-- Maintain relative links between resources
-- Support for same-host resources only
-- Preserve original file extensions and paths
+- **レスポンシブ画像対応**: 複数のデバイス幅で`<picture>`要素やメディアクエリのリソースを取得
+- **遅延読み込み対応**: ページを自動スクロールして`loading=lazy`や`IntersectionObserver`ベースのコンテンツを取得
+- **マルチデバイスシミュレーション**: 様々なデバイス幅と解像度をシミュレートして包括的なリソース取得を実現
+- HTMLページのディレクトリ構造を保持してダウンロード
+- 関連するすべてのリソース（CSS、JS、画像など）を取得
+- リソース間の相対リンクを維持
+- 同一ホストのリソースのみサポート
+- 元のファイル拡張子とパスを保持
 
 ## License
 

package/dist/cli.js

@@ -1,15 +1,36 @@
 #!/usr/bin/env node
-import { createCLI, parseCommonOptions } from '@d-zero/cli-core';
+import { createCLI, parseCommonOptions, parseList } from '@d-zero/cli-core';
+import { parseDevicesOption } from '@d-zero/puppeteer-page-scan';
 import { replicate } from './index.js';
 const { options, args } = createCLI({
     aliases: {
         o: 'output',
         v: 'verbose',
+        t: 'timeout',
+        d: 'devices',
     },
-    usage: ['Usage: replicator <url> -o <output-directory> [--verbose]'],
+    usage: [
+        'Usage: replicator <url> -o <output-directory> [options]',
+        '',
+        'Options:',
+        '  -o, --output <dir>        Output directory (required)',
+        '  -t, --timeout <ms>        Request timeout in milliseconds (default: 30000)',
+        '  -d, --devices <devices>   Device presets (comma-separated, default: desktop-compact,mobile)',
+        '  -v, --verbose             Enable verbose logging',
+        '',
+        'Available device presets:',
+        '  desktop, tablet, mobile, desktop-hd, desktop-compact, mobile-large, mobile-small',
+        '',
+        'Examples:',
+        '  replicator https://example.com -o ./output',
+        '  replicator https://example.com -o ./output --devices desktop,tablet',
+        '  replicator https://example.com -o ./output --timeout 60000',
+    ],
     parseArgs: (cli) => ({
         ...parseCommonOptions(cli),
         output: cli.output,
+        timeout: cli.timeout ? Number(cli.timeout) : undefined,
+        devices: cli.devices,
     }),
     validateArgs: (options, cli) => {
         return !!(cli._.length > 0 && options.output);
@@ -23,8 +44,12 @@ if (!url || typeof url !== 'string') {
     process.exit(1);
 }
 try {
+    const deviceNames = options.devices ? parseList(options.devices) : undefined;
+    const devices = parseDevicesOption(deviceNames);
     await replicate(url, outputDir, {
         verbose: options.verbose ?? false,
+        timeout: options.timeout,
+        devices,
     });
     // eslint-disable-next-line no-console
     console.log(`✅ Successfully replicated ${url} to ${outputDir}`);

package/dist/index.js

@@ -1,6 +1,7 @@
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
 import { URL } from 'node:url';
+import { beforePageScan, devicePresets } from '@d-zero/puppeteer-page-scan';
 import { launch } from 'puppeteer';
 /**
  *
@@ -9,7 +10,12 @@ import { launch } from 'puppeteer';
  * @param options
  */
 export async function replicate(url, outputDir, options = {}) {
-    const { verbose = false, userAgent, timeout = 30_000 } = options;
+    const { verbose = false, timeout = 30_000, devices } = options;
+    const defaultSizes = {
+        'desktop-compact': devicePresets['desktop-compact'],
+        mobile: devicePresets.mobile,
+    };
+    const targetSizes = devices ?? defaultSizes;
     const log = (message) => {
         if (verbose) {
             // eslint-disable-next-line no-console
@@ -22,39 +28,105 @@ export async function replicate(url, outputDir, options = {}) {
         console.log(message);
     };
     const baseUrl = new URL(url);
-    const resources = [];
+    const allResources = [];
     progress(`🚀 Starting replication of ${url}`);
     log(`   Output directory: ${outputDir}`);
+    log(`   Device sizes: ${Object.keys(targetSizes).join(', ')}`);
     progress(`🌐 Launching browser...`);
     const browser = await launch({
         headless: true,
         timeout,
     });
-    progress(`📄 Creating new page...`);
-    const page = await browser.newPage();
-    if (userAgent) {
-        log(`   Setting user agent: ${userAgent}`);
-        await page.setUserAgent(userAgent);
+    try {
+        // Process each device size
+        for (const [sizeName, sizeConfig] of Object.entries(targetSizes)) {
+            const { width } = sizeConfig;
+            const resolution = 'resolution' in sizeConfig ? sizeConfig.resolution : undefined;
+            progress(`📱 Processing ${sizeName} (${width}px${resolution ? `, ${resolution}x` : ''})...`);
+            const page = await browser.newPage();
+            const sizeResources = [];
+            try {
+                await processPageForSize(page, url, baseUrl, sizeResources, {
+                    sizeName,
+                    width,
+                    resolution,
+                    timeout,
+                    log,
+                    progress,
+                });
+                // Merge resources, avoiding duplicates
+                for (const resource of sizeResources) {
+                    const existing = allResources.find((r) => r.url === resource.url);
+                    if (!existing) {
+                        allResources.push(resource);
+                    }
+                    else if (!existing.content && resource.content) {
+                        existing.content = resource.content;
+                    }
+                }
+            }
+            finally {
+                await page.close().catch((error) => {
+                    log(`⚠️ Warning: Failed to close page for ${sizeName}: ${error instanceof Error ? error.message : String(error)}`);
+                });
+            }
+        }
+        const resourceCount = allResources.length;
+        const downloadedCount = allResources.filter((r) => r.content).length;
+        progress(`📄 Found ${resourceCount} total resources (${downloadedCount} downloaded successfully)`);
+        // Ensure output directory exists
+        progress(`📁 Creating output directory...`);
+        await fs.mkdir(outputDir, { recursive: true });
+        // Save all resources
+        progress(`💾 Saving files to disk...`);
+        const savedCount = await saveResources(allResources, outputDir, log, progress);
+        progress(`🎉 Replication complete! ${savedCount} files saved to ${outputDir}`);
+    }
+    finally {
+        progress(`🔧 Cleaning up browser...`);
+        await browser.close().catch((error) => {
+            // Log browser close errors but don't throw them
+            log(`⚠️ Warning: Failed to close browser: ${error instanceof Error ? error.message : String(error)}`);
+        });
     }
-    progress(`🔍 Setting up resource detection...`);
-    // Collect all requests
+}
+/**
+ * Process a page for a specific device size
+ * @param page
+ * @param url
+ * @param baseUrl
+ * @param resources
+ * @param options
+ * @param options.sizeName
+ * @param options.width
+ * @param options.resolution
+ * @param options.timeout
+ * @param options.log
+ * @param options.progress
+ */
+async function processPageForSize(page, url, baseUrl, resources, options) {
+    const { sizeName, width, resolution, log, progress } = options;
     const requestPromises = [];
+    // Set up resource detection
     page.on('request', (request) => {
         const requestUrl = request.url();
         const requestUrlObj = new URL(requestUrl);
         // Only handle same-host resources
         if (requestUrlObj.hostname === baseUrl.hostname) {
-            log(`📥 Intercepting: ${requestUrl}`);
+            log(`📥 [${sizeName}] Intercepting: ${requestUrl}`);
             const localPath = urlToLocalPath(requestUrl);
             const resourceType = getResourceType(requestUrl);
-            resources.push({
-                url: requestUrl,
-                localPath,
-                type: resourceType,
-            });
+            // Check if this resource is already tracked
+            if (!resources.some((r) => r.url === requestUrl)) {
+                resources.push({
+                    url: requestUrl,
+                    localPath,
+                    type: resourceType,
+                });
+            }
         }
         else {
-            log(`🚫 Skipping external resource: ${requestUrl}`);
+            log(`🚫 [${sizeName}] Skipping external resource: ${requestUrl}`);
         }
     });
     page.on('response', (response) => {
@@ -69,47 +141,67 @@ export async function replicate(url, outputDir, options = {}) {
                         .buffer()
                         .then((buffer) => {
                         resource.content = buffer;
-                        log(`✅ Downloaded: ${responseUrl}`);
+                        log(`✅ [${sizeName}] Downloaded: ${responseUrl}`);
                     })
                         .catch((error) => {
                         const errorMessage = error instanceof Error ? error.message : String(error);
-                        log(`❌ Failed to download: ${responseUrl} - ${errorMessage}`);
-                        // Don't rethrow here as this would break the entire operation
-                        // Individual resource failures should not stop the whole process
+                        log(`❌ [${sizeName}] Failed to download: ${responseUrl} - ${errorMessage}`);
                     });
                 }
                 else if (resource) {
-                    log(`❌ Resource failed (${response.status()}): ${responseUrl}`);
+                    log(`❌ [${sizeName}] Resource failed (${response.status()}): ${responseUrl}`);
                 }
             })();
             requestPromises.push(promise);
         }
     });
-    try {
-        // Navigate to the page
-        progress(`📡 Navigating to ${url}...`);
-        await page.goto(url, { waitUntil: 'networkidle2', timeout });
-        progress(`⏳ Waiting for all resources to load...`);
-        // Wait for all downloads to complete
-        await Promise.all(requestPromises);
-        const resourceCount = resources.length;
-        const downloadedCount = resources.filter((r) => r.content).length;
-        progress(`📄 Found ${resourceCount} resources (${downloadedCount} downloaded successfully)`);
-        // Ensure output directory exists
-        progress(`📁 Creating output directory...`);
-        await fs.mkdir(outputDir, { recursive: true });
-        // Save all resources
-        progress(`💾 Saving files to disk...`);
-        const savedCount = await saveResources(resources, outputDir, log, progress);
-        progress(`🎉 Replication complete! ${savedCount} files saved to ${outputDir}`);
-    }
-    finally {
-        progress(`🔧 Cleaning up browser...`);
-        await browser.close().catch((error) => {
-            // Log browser close errors but don't throw them
-            log(`⚠️ Warning: Failed to close browser: ${error instanceof Error ? error.message : String(error)}`);
-        });
-    }
+    // Set viewport and navigate using beforePageScan (which includes scrolling)
+    progress(`📡 [${sizeName}] Setting viewport and navigating...`);
+    await beforePageScan(page, url, {
+        name: sizeName,
+        width,
+        resolution,
+        listener: (phase, data) => {
+            switch (phase) {
+                case 'setViewport': {
+                    const setViewportData = data;
+                    log(`📱 [${sizeName}] Viewport set: ${setViewportData.width}px${setViewportData.resolution ? ` @ ${setViewportData.resolution}x` : ''}`);
+                    break;
+                }
+                case 'load': {
+                    const loadData = data;
+                    log(`📄 [${sizeName}] Page loaded (${loadData.type})`);
+                    break;
+                }
+                case 'scroll': {
+                    const scrollData = data;
+                    switch (scrollData.message) {
+                        case 'Start scrolling': {
+                            log(`📜 [${sizeName}] Starting scroll to trigger lazy loading...`);
+                            break;
+                        }
+                        case 'End of page': {
+                            log(`📜 [${sizeName}] Scroll completed (${scrollData.scrollY}/${scrollData.scrollHeight}px)`);
+                            break;
+                        }
+                        case 'Scrolling': {
+                            const progress = Math.round((scrollData.scrollY / scrollData.scrollHeight) * 100);
+                            log(`📜 [${sizeName}] Scrolling progress: ${progress}% (${scrollData.scrollY}/${scrollData.scrollHeight}px)`);
+                            break;
+                        }
+                        // No default
+                    }
+                    break;
+                }
+            }
+        },
+    });
+    progress(`⏳ [${sizeName}] Waiting for all resources to load...`);
+    // Wait for all downloads to complete
+    await Promise.all(requestPromises);
+    const resourceCount = resources.length;
+    const downloadedCount = resources.filter((r) => r.content).length;
+    progress(`📄 [${sizeName}] Found ${resourceCount} resources (${downloadedCount} downloaded)`);
 }
 /**
  *

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.2.0",
+	"version": "0.4.0",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,14 +24,16 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.1.0",
-		"@d-zero/shared": "0.9.1",
+		"@d-zero/cli-core": "1.1.1",
+		"@d-zero/puppeteer-page-scan": "4.1.0",
+		"@d-zero/puppeteer-scroll": "3.0.6",
+		"@d-zero/shared": "0.9.2",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
-		"puppeteer": "24.12.0"
+		"puppeteer": "24.18.0"
 	},
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "7cc778738d8c811adb69cee528655e12eba52e87"
+	"gitHead": "8081edac801400fed7c0b7ebeccc0ce66ccfe131"
 }