@d-zero/replicator 0.6.1 → 0.7.1

package/dist/cli.js

@@ -18,6 +18,8 @@ const { options, args } = createCLI({
         '  -t, --timeout <ms>        Request timeout in milliseconds (default: 30000)',
         '  -d, --devices <devices>   Device presets (comma-separated, default: desktop-compact,mobile)',
         '  -l, --limit <number>      Parallel execution limit (default: 3)',
+        '  --interval <ms>           Interval between parallel executions (default: none)',
+        '                             Format: number or "min-max" for random range',
         '  --only <type>             Download only specified type: page or resource',
         '  -v, --verbose             Enable verbose logging',
         '',
@@ -65,6 +67,7 @@ try {
         devices,
         limit: options.limit,
         only: options.only,
+        interval: options.interval,
     });
 }
 catch (error) {

package/dist/index.js

@@ -54,9 +54,10 @@ function collectPageUrlsOnly(urls, progress) {
  * @param verbose - Enable verbose logging
  * @param limit - Parallel execution limit
  * @param progress - Progress logger function
+ * @param interval
  * @returns Set of encoded URLs
  */
-async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress) {
+async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval) {
     progress(c.bold.yellow('📡 Phase 1: Collecting resource metadata...'));
     const results = [];
     await deal(urls.map((url) => ({ id: null, url })), (_, done, total) => {
@@ -68,6 +69,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
     }, {}), {
         verbose,
         limit,
+        interval,
         each: (result) => {
             results.push(result);
         },
@@ -119,7 +121,7 @@ async function collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit
  * @param options - Replication options
  */
 export async function replicate(options) {
-    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, } = options;
+    const { urls, outputDir, verbose = false, timeout = 30_000, devices, limit = 3, only, interval, } = options;
     if (urls.length === 0) {
         throw new Error('At least one URL is required');
     }
@@ -154,7 +156,7 @@ export async function replicate(options) {
         }
         case 'resource':
         case undefined: {
-            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress);
+            allEncodedUrls = await collectAllResourceUrls(urls, targetSizes, timeout, verbose, limit, progress, interval);
             break;
         }
         default: {
@@ -168,7 +170,7 @@ export async function replicate(options) {
     // Use the first URL as base URL for constructing full URLs
     const baseUrl = urls[0];
     // Download all resources
-    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only);
+    await downloadResources([...allEncodedUrls], baseUrl, outputDir, progress, verbose, only, interval);
     progress('');
     progress(c.bold.green(`✅ Replication complete!`));
     progress(c.gray(`   All resources saved to: ${outputDir}`));

package/dist/resource-downloader.d.ts

@@ -1,3 +1,4 @@
+import type { DelayOptions } from '@d-zero/shared/delay';
 /**
  * Download and save resources to disk
  * @param encodedPaths - Array of encoded pathnames
@@ -7,4 +8,14 @@
  * @param verbose - Enable verbose output
  * @param only - Download only specified type: page or resource
  */
-export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource'): Promise<void>;
+/**
+ *
+ * @param encodedPaths
+ * @param baseUrl
+ * @param outputDir
+ * @param logger
+ * @param verbose
+ * @param only
+ * @param interval
+ */
+export declare function downloadResources(encodedPaths: string[], baseUrl: string, outputDir: string, logger: (message: string) => void, verbose?: boolean, only?: 'page' | 'resource', interval?: number | DelayOptions): Promise<void>;

package/dist/resource-downloader.js

@@ -35,7 +35,17 @@ function parseEncodedPath(encodedPath, baseUrl) {
  * @param verbose - Enable verbose output
  * @param only - Download only specified type: page or resource
  */
-export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only) {
+/**
+ *
+ * @param encodedPaths
+ * @param baseUrl
+ * @param outputDir
+ * @param logger
+ * @param verbose
+ * @param only
+ * @param interval
+ */
+export async function downloadResources(encodedPaths, baseUrl, outputDir, logger, verbose = false, only, interval) {
     const uniqueResources = new Map();
     // Parse all encoded pathnames
     for (const encodedPath of encodedPaths) {
@@ -67,13 +77,14 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
     logger('');
     let downloaded = 0;
     let failed = 0;
-    await deal(tasks, (task, update, index) => {
+    await deal(tasks, (task, update, index, setLineHeader) => {
         const fileId = index.toString().padStart(4, '0');
         const lineHeader = `%braille% ${c.bgWhite(` ${fileId} `)} ${c.gray(task.localPath)}: `;
+        setLineHeader(lineHeader);
         return async () => {
-            update(`${lineHeader}Fetching%dots%`);
+            update('Fetching%dots%');
             const response = await fetch(task.url).catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Fetch failed: ${error.message}`)}`);
+                update(c.red(`❌ Fetch failed: ${error.message}`));
                 failed++;
                 return null;
             });
@@ -81,30 +92,30 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
                 return;
             }
             if (!response.ok) {
-                update(`${lineHeader}${c.red(`❌ HTTP ${response.status}`)}`);
+                update(c.red(`❌ HTTP ${response.status}`));
                 failed++;
                 return;
             }
-            update(`${lineHeader}Reading content%dots%`);
+            update('Reading content%dots%');
             const content = Buffer.from(await response.arrayBuffer());
             const fullPath = path.join(outputDir, task.localPath);
             const dir = path.dirname(fullPath);
-            update(`${lineHeader}Creating directory%dots%`);
+            update('Creating directory%dots%');
             const mkdirSuccess = await mkdir(dir, { recursive: true })
                 .then(() => true)
                 .catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Failed to create directory: ${error.message}`)}`);
+                update(c.red(`❌ Failed to create directory: ${error.message}`));
                 failed++;
                 return false;
             });
             if (!mkdirSuccess) {
                 return;
             }
-            update(`${lineHeader}Writing file%dots%`);
+            update('Writing file%dots%');
             const writeSuccess = await writeFile(fullPath, content)
                 .then(() => true)
                 .catch((error) => {
-                update(`${lineHeader}${c.red(`❌ Failed to write: ${error.message}`)}`);
+                update(c.red(`❌ Failed to write: ${error.message}`));
                 failed++;
                 return false;
             });
@@ -112,11 +123,12 @@ export async function downloadResources(encodedPaths, baseUrl, outputDir, logger
                 return;
             }
             downloaded++;
-            update(`${lineHeader}${c.green('✅ Downloaded')}`);
+            update(c.green('✅ Downloaded'));
         };
     }, {
         limit: 10,
         verbose,
+        interval,
         header: (progress, done, total, limit) => {
             const percentage = Math.round(progress * 100);
             if (progress === 1) {

package/dist/types.d.ts

@@ -1,3 +1,4 @@
+import type { DelayOptions } from '@d-zero/shared/delay';
 export interface ReplicateOptions {
     urls: string[];
     outputDir: string;
@@ -9,6 +10,7 @@ export interface ReplicateOptions {
     }>;
     limit?: number;
     only?: 'page' | 'resource';
+    interval?: number | DelayOptions;
 }
 export interface Resource {
     url: string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.6.1",
+	"version": "0.7.1",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,18 +24,18 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.1.3",
-		"@d-zero/dealer": "1.3.2",
-		"@d-zero/puppeteer-dealer": "0.5.9",
-		"@d-zero/puppeteer-page-scan": "4.2.5",
-		"@d-zero/puppeteer-scroll": "3.0.11",
-		"@d-zero/shared": "0.12.0",
+		"@d-zero/cli-core": "1.2.1",
+		"@d-zero/dealer": "1.4.1",
+		"@d-zero/puppeteer-dealer": "0.5.11",
+		"@d-zero/puppeteer-page-scan": "4.2.7",
+		"@d-zero/puppeteer-scroll": "3.1.1",
+		"@d-zero/shared": "0.14.0",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
-		"puppeteer": "24.26.1"
+		"puppeteer": "24.27.0"
 	},
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "bf9cfd1370947212367041214f68c8e8e6317b86"
+	"gitHead": "2397932a3e3fe5f0631c0d540bdcac732fc71f19"
 }