@d-zero/replicator 0.9.6-alpha.2 → 0.10.0

package/README.md

@@ -22,7 +22,8 @@ npx @d-zero/replicator <url...> -o <output-directory> [options]
 - `-o, --output <dir>`: 出力ディレクトリ（必須）
 - `-t, --timeout <ms>`: リクエストタイムアウト（ミリ秒、デフォルト: 30000）
 - `-d, --devices <devices>`: デバイスプリセット（カンマ区切り、デフォルト: desktop-compact,mobile）
-- `-l, --limit <number>`: 並列処理数の上限（デフォルト: 3）
+- `-l, --limit <number>`: ページスキャン（Phase 1）の並列処理数の上限（デフォルト: 3）
+- `--download-limit <number>`: リソースダウンロード（Phase 2）の並列処理数の上限（デフォルト: 10）
 - `--interval <ms>`: 並列実行間の間隔（デフォルト: なし）
   - 数値または"min-max"形式でランダム範囲を指定可能
 - `--only <type>`: ダウンロード対象を限定（`page` または `resource`）
@@ -92,7 +93,8 @@ await replicate({
 		'https://example.com/page3',
 	],
 	outputDir: './output',
-	limit: 2, // 最大2つのURLを同時処理
+	limit: 2, // ページスキャン（Phase 1）は最大2並列
+	downloadLimit: 5, // リソースダウンロード（Phase 2）は最大5並列
 });
 
 // カスタムデバイス

package/dist/cli.js

@@ -23,7 +23,8 @@ const { options, args } = createCLI({
         '  -o, --output <dir>        Output directory (required)',
         '  -t, --timeout <ms>        Request timeout in milliseconds (default: 30000)',
         '  -d, --devices <devices>   Device presets (comma-separated, default: desktop-compact,mobile)',
-        '  -l, --limit <number>      Parallel execution limit (default: 3)',
+        '  -l, --limit <number>      Parallel execution limit for page scan (default: 3)',
+        '  --download-limit <number> Parallel execution limit for resource download (default: 10)',
         '  --interval <ms>           Interval between parallel executions (default: none)',
         '                             Format: number or "min-max" for random range',
         '  --only <type>             Download only specified type: page or resource',
@@ -47,6 +48,7 @@ const { options, args } = createCLI({
         timeout: cli.timeout ? Number(cli.timeout) : undefined,
         devices: cli.devices,
         limit: cli.limit ? Number(cli.limit) : undefined,
+        downloadLimit: cli['download-limit'] ? Number(cli['download-limit']) : undefined,
         only: cli.only,
         auth: cli.auth,
     }),
@@ -77,6 +79,7 @@ try {
         timeout: options.timeout,
         devices,
         limit: options.limit,
+        downloadLimit: options.downloadLimit,
         only: options.only,
         interval: options.interval,
         username,

package/dist/index.js

@@ -103,7 +103,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
  * @param options - Replication options
  */
 export async function replicate(options) {
-    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, interval, username, password, } = options;
+    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, downloadLimit = 10, only, interval, username, password, } = options;
     if (urls.length === 0) {
         throw new Error('At least one URL is required');
     }
@@ -128,6 +128,7 @@ export async function replicate(options) {
     progress(c.bold.cyan(`🌐 Replicating ${urls.length} URL(s)`));
     progress(c.gray(`   Output: ${outputDir}`));
     progress(c.gray(`   Parallel limit: ${limit}`));
+    progress(c.gray(`   Download limit: ${downloadLimit}`));
     progress('');
     // Phase 1: Collect resource metadata from all URLs
     let allEncodedUrls;
@@ -152,7 +153,7 @@ export async function replicate(options) {
     // Use the first URL as base URL for constructing full URLs
     const baseUrl = urls[0];
     // Download all resources
-    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval, username, password);
+    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval, username, password, downloadLimit);
     progress('');
     progress(c.bold.green(`✅ Replication complete!`));
     progress(c.gray(`   All resources saved to: ${outputDir}`));

package/dist/resource-downloader.d.ts

@@ -19,5 +19,6 @@ import type { DelayOptions } from '@d-zero/shared/delay';
  * @param interval
  * @param username
  * @param password
+ * @param limit
  */
-export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions, username?: string, password?: string): Promise<void>;
+export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions, username?: string, password?: string, limit?: number): Promise<void>;

package/dist/resource-downloader.js

@@ -23,8 +23,9 @@ import c from 'ansi-colors';
  * @param interval
  * @param username
  * @param password
+ * @param limit
  */
-export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval, username, password) {
+export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval, username, password, limit = 10) {
     const uniqueResources = new Map();
     // Parse all encoded pathnames
     for (const encodedPath of encodedPaths) {
@@ -110,7 +111,7 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
             update(c.green('✅ Downloaded'));
         };
     }, {
-        limit: 10,
+        limit,
         verbose,
         interval,
         header: (progress, done, total, limit) => {

package/dist/types.d.ts

@@ -9,6 +9,7 @@ export interface ReplicateOptions {
         resolution?: number;
     }>;
     limit?: number;
+    downloadLimit?: number;
     only?: 'page' | 'resource';
     interval?: number | DelayOptions;
     username?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.9.6-alpha.2",
+	"version": "0.10.0",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,12 +24,12 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.3.8-alpha.2",
-		"@d-zero/dealer": "1.7.4-alpha.2",
-		"@d-zero/puppeteer-dealer": "0.7.10-alpha.2",
-		"@d-zero/puppeteer-page-scan": "4.4.10-alpha.2",
-		"@d-zero/puppeteer-scroll": "3.1.18-alpha.2",
-		"@d-zero/shared": "0.21.3-alpha.2",
+		"@d-zero/cli-core": "1.3.8",
+		"@d-zero/dealer": "1.7.4",
+		"@d-zero/puppeteer-dealer": "0.7.10",
+		"@d-zero/puppeteer-page-scan": "4.4.10",
+		"@d-zero/puppeteer-scroll": "3.1.18",
+		"@d-zero/shared": "0.21.3",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
 		"puppeteer": "24.37.5"
@@ -42,5 +42,5 @@
 		"url": "https://github.com/d-zero-dev/tools.git",
 		"directory": "packages/@d-zero/replicator"
 	},
-	"gitHead": "74b262438ed5e263d89a066029adfadf1ec27e0b"
+	"gitHead": "f6d9cef8a6a5caefe14781155c14605157dc59c2"
 }