@d-zero/replicator 0.6.0 → 0.7.0

package/dist/child-process.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -18,6 +18,8 @@ const { options, args } = createCLI({
         '  -t, --timeout <ms>        Request timeout in milliseconds (default: 30000)',
         '  -d, --devices <devices>   Device presets (comma-separated, default: desktop-compact,mobile)',
         '  -l, --limit <number>      Parallel execution limit (default: 3)',
+        '  --interval <ms>           Interval between parallel executions (default: none)',
+        '                             Format: number or "min-max" for random range',
         '  --only <type>             Download only specified type: page or resource',
         '  -v, --verbose             Enable verbose logging',
         '',
@@ -65,6 +67,7 @@ try {
         devices,
         limit: options.limit,
         only: options.only,
+        interval: options.interval,
     });
 }
 catch (error) {

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+import type { ReplicateOptions } from './types.js';
+/**
+ * Replicate web pages with all their resources to local directories
+ *
+ * ## Architecture
+ *
+ * This implementation uses a two-phase architecture for memory efficiency:
+ *
+ * ### Phase 1: Metadata Collection
+ * - Each URL is processed in a separate child process using puppeteer-dealer
+ * - Child processes scan pages with Puppeteer and collect resource URLs
+ * - For URLs ending with '/' (e.g., https://example.com/), MIME type is captured
+ * and encoded as "url:::MIME/type" format
+ * - Only metadata (URLs + MIME types) is returned to parent - no buffer data
+ *
+ * ### Phase 2: Resource Download
+ * - Parent process aggregates all metadata and removes duplicates
+ * - Parses encoded URLs to determine correct local paths
+ * - Downloads resources via fetch() and immediately writes to disk
+ * - No resource content is kept in memory
+ *
+ * This approach minimizes memory usage by avoiding duplicate I/O operations
+ * and keeping buffer data out of inter-process communication.
+ * @param options - Replication options
+ */
+export declare function replicate(options: ReplicateOptions): Promise<void>;

package/dist/index.js

@@ -54,9 +54,10 @@ function collectPageUrlsOnly(urls, progress) {
  * @param verbose - Enable verbose logging
  * @param limit - Parallel execution limit
  * @param progress - Progress logger function
+ * @param interval
  * @returns Set of encoded URLs
  */
-async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress) {
+async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval) {
     progress(c.bold.yellow('📡 Phase 1: Collecting resource metadata...'));
     const results = [];
     await deal(urls.map((url) => ({ id: null, url })), (_, done, total) => {
@@ -68,6 +69,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
     }, {}), {
         verbose,
         limit,
+        interval,
         each: (result) => {
             results.push(result);
         },
@@ -119,7 +121,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
  * @param options - Replication options
  */
 export async function replicate(options) {
-    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, } = options;
+    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, interval, } = options;
     if (urls.length === 0) {
         throw new Error('At least one URL is required');
     }
@@ -154,7 +156,7 @@ export async function replicate(options) {
         }
         case 'resource':
         case undefined: {
-            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress);
+            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval);
             break;
         }
         default: {
@@ -168,7 +170,7 @@ export async function replicate(options) {
     // Use the first URL as base URL for constructing full URLs
     const baseUrl = urls[0];
     // Download all resources
-    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only);
+    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval);
     progress('');
     progress(c.bold.green(`✅ Replication complete!`));
     progress(c.gray(`   All resources saved to: ${outputDir}`));

package/dist/resource-downloader.d.ts

@@ -0,0 +1,21 @@
+import type { DelayOptions } from '@d-zero/shared/delay';
+/**
+ * Download and save resources to disk
+ * @param encodedPaths - Array of encoded pathnames
+ * @param baseUrl - Base URL to construct full URLs
+ * @param outputDir - Output directory
+ * @param logger - Logger function
+ * @param verbose - Enable verbose output
+ * @param only - Download only specified type: page or resource
+ */
+/**
+ *
+ * @param encodedPaths
+ * @param baseUrl
+ * @param outputDir
+ * @param logger
+ * @param verbose
+ * @param only
+ * @param interval
+ */
+export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions): Promise<void>;

package/dist/resource-downloader.js

@@ -35,7 +35,17 @@ function parseEncodedPath(encodedPath, baseUrl) {
  * @param verbose - Enable verbose output
  * @param only - Download only specified type: page or resource
  */
-export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only) {
+/**
+ *
+ * @param encodedPaths
+ * @param baseUrl
+ * @param outputDir
+ * @param logger
+ * @param verbose
+ * @param only
+ * @param interval
+ */
+export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval) {
     const uniqueResources = new Map();
     // Parse all encoded pathnames
     for (const encodedPath of encodedPaths) {
@@ -67,13 +77,14 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
     logger('');
     let downloaded = 0;
     let failed = 0;
-    await deal(tasks, (task, update, index) => {
+    await deal(tasks, (task, update, index, setLineHeader) => {
         const fileId = index.toString().padStart(4, '0');
         const lineHeader = `%braille% ${c.bgWhite(` ${fileId} `)} ${c.gray(task.localPath)}: `;
+        setLineHeader(lineHeader);
         return async () => {
-            update(`${lineHeader}Fetching%dots%`);
+            update('Fetching%dots%');
             const response = await fetch(task.url).catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Fetch failed: ${error.message}`)}`);
+                update(c.red(`❌ Fetch failed: ${error.message}`));
                 failed++;
                 return null;
             });
@@ -81,30 +92,30 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
                 return;
             }
             if (!response.ok) {
-                update(`${lineHeader}${c.red(`❌ HTTP ${response.status}`)}`);
+                update(c.red(`❌ HTTP ${response.status}`));
                 failed++;
                 return;
             }
-            update(`${lineHeader}Reading content%dots%`);
+            update('Reading content%dots%');
             const content = Buffer.from(await response.arrayBuffer());
             const fullPath = path.join(outputDir, task.localPath);
             const dir = path.dirname(fullPath);
-            update(`${lineHeader}Creating directory%dots%`);
+            update('Creating directory%dots%');
             const mkdirSuccess = await mkdir(dir, { recursive: true })
                 .then(() => true)
                 .catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Failed to create directory: ${error.message}`)}`);
+                update(c.red(`❌ Failed to create directory: ${error.message}`));
                 failed++;
                 return false;
             });
             if (!mkdirSuccess) {
                 return;
             }
-            update(`${lineHeader}Writing file%dots%`);
+            update('Writing file%dots%');
             const writeSuccess = await writeFile(fullPath, content)
                 .then(() => true)
                 .catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Failed to write: ${error.message}`)}`);
+                update(c.red(`❌ Failed to write: ${error.message}`));
                 failed++;
                 return false;
             });
@@ -112,11 +123,12 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
                 return;
             }
             downloaded++;
-            update(`${lineHeader}${c.green('✅ Downloaded')}`);
+            update(c.green('✅ Downloaded'));
         };
     }, {
         limit: 10,
         verbose,
+        interval,
         header: (progress, done, total, limit) => {
             const percentage = Math.round(progress * 100);
             if (progress === 1) {

package/dist/types.d.ts

@@ -0,0 +1,29 @@
+import type { DelayOptions } from '@d-zero/shared/delay';
+export interface ReplicateOptions {
+    urls: string[];
+    outputDir: string;
+    verbose?: boolean;
+    timeout?: number;
+    devices?: Record<string, {
+        width: number;
+        resolution?: number;
+    }>;
+    limit?: number;
+    only?: 'page' | 'resource';
+    interval?: number | DelayOptions;
+}
+export interface Resource {
+    url: string;
+    localPath: string;
+}
+export interface ChildProcessInput {
+    devices?: Record<string, {
+        width: number;
+        resolution?: number;
+    }>;
+    timeout?: number;
+}
+export interface ChildProcessResult {
+    url: string;
+    encodedUrls: string[];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.6.0",
+	"version": "0.7.0",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,18 +24,18 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.1.3",
-		"@d-zero/dealer": "1.3.2",
-		"@d-zero/puppeteer-dealer": "0.5.9",
-		"@d-zero/puppeteer-page-scan": "4.2.5",
-		"@d-zero/puppeteer-scroll": "3.0.11",
-		"@d-zero/shared": "0.12.0",
+		"@d-zero/cli-core": "1.2.0",
+		"@d-zero/dealer": "1.4.0",
+		"@d-zero/puppeteer-dealer": "0.5.10",
+		"@d-zero/puppeteer-page-scan": "4.2.6",
+		"@d-zero/puppeteer-scroll": "3.1.0",
+		"@d-zero/shared": "0.13.0",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
-		"puppeteer": "24.26.1"
+		"puppeteer": "24.27.0"
 	},
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "fe6d98ee0108b0e53848f28a74e4e08875e31a78"
+	"gitHead": "85abd39686d2ce02c7b8db071437464d212dd982"
 }