@d-zero/replicator 0.8.7 → 0.9.0

package/README.md

@@ -26,6 +26,7 @@ npx @d-zero/replicator <url...> -o <output-directory> [options]
 - `--interval <ms>`: 並列実行間の間隔（デフォルト: なし）
   - 数値または"min-max"形式でランダム範囲を指定可能
 - `--only <type>`: ダウンロード対象を限定（`page` または `resource`）
+- `-a, --auth <user:pass>`: Basic認証の認証情報（`ユーザー名:パスワード` 形式）
 - `-v, --verbose`: 詳細ログモード
 
 ##### `--only` オプション
@@ -67,6 +68,9 @@ npx @d-zero/replicator https://example.com -o ./output --only page
 
 # リソースのみダウンロード（HTMLを除外）
 npx @d-zero/replicator https://example.com -o ./output --only resource
+
+# Basic認証が必要なページ
+npx @d-zero/replicator https://example.com -o ./output -a username:password
 ```
 
 ### プログラマティック使用
@@ -116,10 +120,19 @@ await replicate({
 	outputDir: './output',
 	only: 'resource',
 });
+
+// Basic認証が必要なページ
+await replicate({
+	urls: ['https://example.com'],
+	outputDir: './output',
+	username: 'username',
+	password: 'password',
+});
 ```
 
 ## 機能
 
+- **Basic認証対応**: `--auth user:pass`オプションでBasic認証が必要なページにアクセス可能
 - **並列処理**: 複数のURLを並列で効率的に処理
 - **メモリ効率**: リソースを直接ディスクに保存してメモリ使用量を最小化
 - **選択的ダウンロード**: `--only`オプションでHTMLページのみまたはリソースのみをダウンロード可能

package/dist/child-process.js

@@ -3,7 +3,7 @@ import { beforePageScan, devicePresets } from '@d-zero/puppeteer-page-scan';
 import { scrollAllOver } from '@d-zero/puppeteer-scroll';
 import { encodeResourcePath } from '@d-zero/shared/encode-resource-path';
 createChildProcess((param) => {
-    const { devices, timeout } = param;
+    const { devices, timeout, username, password } = param;
     return {
         async eachPage({ page, url }, logger) {
             const resourcePaths = new Set();
@@ -38,6 +38,9 @@ createChildProcess((param) => {
                 resourcePaths.add(encodeResourcePath(resourceUrlObj, mimeType));
             };
             page.on('response', responseHandler);
+            if (username && password) {
+                await page.authenticate({ username, password });
+            }
             const defaultSizes = {
                 'desktop-compact': devicePresets['desktop-compact'],
                 mobile: devicePresets.mobile,

package/dist/cli.js

@@ -14,6 +14,7 @@ const { options, args } = createCLI({
         t: 'timeout',
         d: 'devices',
         l: 'limit',
+        a: 'auth',
     },
     usage: [
         'Usage: replicator <url1> [url2...] -o <output-directory> [options]',
@@ -26,6 +27,7 @@ const { options, args } = createCLI({
         '  --interval <ms>           Interval between parallel executions (default: none)',
         '                             Format: number or "min-max" for random range',
         '  --only <type>             Download only specified type: page or resource',
+        '  -a, --auth <user:pass>    Credentials for Basic authentication',
         '  -v, --verbose             Enable verbose logging',
         '',
         'Available device presets:',
@@ -46,6 +48,7 @@ const { options, args } = createCLI({
         devices: cli.devices,
         limit: cli.limit ? Number(cli.limit) : undefined,
         only: cli.only,
+        auth: cli.auth,
     }),
     validateArgs: (options, cli) => {
         if (options.only && options.only !== 'page' && options.only !== 'resource') {
@@ -64,6 +67,9 @@ if (urls.length === 0) {
 try {
     const deviceNames = options.devices ? parseList(options.devices) : undefined;
     const devices = parseDevicesOption(deviceNames);
+    const colonIndex = options.auth ? options.auth.indexOf(':') : -1;
+    const username = colonIndex >= 0 ? options.auth.slice(0, colonIndex) : undefined;
+    const password = colonIndex >= 0 ? options.auth.slice(colonIndex + 1) : undefined;
     await replicate({
         urls,
         outputDir,
@@ -73,6 +79,8 @@ try {
         limit: options.limit,
         only: options.only,
         interval: options.interval,
+        username,
+        password,
     });
 }
 catch (error) {

package/dist/index.js

@@ -33,9 +33,11 @@ function collectPageUrlsOnly(urls, progress) {
  * @param limit - Parallel execution limit
  * @param progress - Progress logger function
  * @param interval
+ * @param username
+ * @param password
  * @returns Set of encoded URLs
  */
-async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval) {
+async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval, username, password) {
     progress(c.bold.yellow('📡 Phase 1: Collecting resource metadata...'));
     const results = [];
     await deal(urls.map((url) => ({ id: null, url })), (_, done, total) => {
@@ -44,6 +46,8 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
     }, () => createProcess(path.resolve(import.meta.dirname, 'child-process.js'), {
         devices: targetSizes,
         timeout,
+        username,
+        password,
     }, {}), {
         verbose,
         limit,
@@ -99,7 +103,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
  * @param options - Replication options
  */
 export async function replicate(options) {
-    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, interval, } = options;
+    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, interval, username, password, } = options;
     if (urls.length === 0) {
         throw new Error('At least one URL is required');
     }
@@ -134,7 +138,7 @@ export async function replicate(options) {
         }
         case 'resource':
         case undefined: {
-            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval);
+            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval, username, password);
             break;
         }
         default: {
@@ -148,7 +152,7 @@ export async function replicate(options) {
     // Use the first URL as base URL for constructing full URLs
     const baseUrl = urls[0];
     // Download all resources
-    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval);
+    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval, username, password);
     progress('');
     progress(c.bold.green(`✅ Replication complete!`));
     progress(c.gray(`   All resources saved to: ${outputDir}`));

package/dist/resource-downloader.d.ts

@@ -17,5 +17,7 @@ import type { DelayOptions } from '@d-zero/shared/delay';
  * @param verbose
  * @param only
  * @param interval
+ * @param username
+ * @param password
  */
-export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions): Promise<void>;
+export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions, username?: string, password?: string): Promise<void>;

package/dist/resource-downloader.js

@@ -21,8 +21,10 @@ import c from 'ansi-colors';
  * @param verbose
  * @param only
  * @param interval
+ * @param username
+ * @param password
  */
-export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval) {
+export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval, username, password) {
     const uniqueResources = new Map();
     // Parse all encoded pathnames
     for (const encodedPath of encodedPaths) {
@@ -46,6 +48,9 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
         }
     }
     const tasks = [...uniqueResources.values()];
+    const authHeader = username && password
+        ? `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`
+        : undefined;
     if (tasks.length === 0) {
         logger(c.yellow('⚠️  No resources to download'));
         return;
@@ -60,7 +65,9 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
         setLineHeader(lineHeader);
         return async () => {
             update('Fetching%dots%');
-            const response = await fetch(task.url).catch((error) => {
+            const response = await fetch(task.url, {
+                headers: authHeader ? { Authorization: authHeader } : {},
+            }).catch((error) => {
                 update(c.red(`❌ Fetch failed: ${error.message}`));
                 failed++;
                 return null;

package/dist/types.d.ts

@@ -11,6 +11,8 @@ export interface ReplicateOptions {
     limit?: number;
     only?: 'page' | 'resource';
     interval?: number | DelayOptions;
+    username?: string;
+    password?: string;
 }
 export interface Resource {
     url: string;
@@ -22,6 +24,8 @@ export interface ChildProcessInput {
         resolution?: number;
     }>;
     timeout?: number;
+    username?: string;
+    password?: string;
 }
 export interface ChildProcessResult {
     url: string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.8.7",
+	"version": "0.9.0",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -25,8 +25,8 @@
 	},
 	"dependencies": {
 		"@d-zero/cli-core": "1.3.4",
-		"@d-zero/dealer": "1.6.4",
-		"@d-zero/puppeteer-dealer": "0.7.4",
+		"@d-zero/dealer": "1.6.5",
+		"@d-zero/puppeteer-dealer": "0.7.5",
 		"@d-zero/puppeteer-page-scan": "4.4.5",
 		"@d-zero/puppeteer-scroll": "3.1.13",
 		"@d-zero/shared": "0.20.1",
@@ -37,5 +37,5 @@
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "a6b5eb0a0a327c003053f7c25be4c075ed319c76"
+	"gitHead": "31410767ae6beff5c5dbe21a824406a4e6716868"
 }