@d-zero/replicator 0.2.0 → 0.3.0

package/dist/index.js

@@ -1,6 +1,7 @@
 import { promises as fs } from 'node:fs';
 import path from 'node:path';
 import { URL } from 'node:url';
+import { beforePageScan } from '@d-zero/puppeteer-page-scan';
 import { launch } from 'puppeteer';
 /**
  *
@@ -9,7 +10,12 @@ import { launch } from 'puppeteer';
  * @param options
  */
 export async function replicate(url, outputDir, options = {}) {
-    const { verbose = false, userAgent, timeout = 30_000 } = options;
+    const { verbose = false, timeout = 30_000, sizes } = options;
+    const defaultSizes = {
+        desktop: { width: 1280 },
+        mobile: { width: 375, resolution: 2 },
+    };
+    const targetSizes = sizes ?? defaultSizes;
     const log = (message) => {
         if (verbose) {
             // eslint-disable-next-line no-console
@@ -22,39 +28,105 @@ export async function replicate(url, outputDir, options = {}) {
         console.log(message);
     };
     const baseUrl = new URL(url);
-    const resources = [];
+    const allResources = [];
     progress(`🚀 Starting replication of ${url}`);
     log(`   Output directory: ${outputDir}`);
+    log(`   Device sizes: ${Object.keys(targetSizes).join(', ')}`);
     progress(`🌐 Launching browser...`);
     const browser = await launch({
         headless: true,
         timeout,
     });
-    progress(`📄 Creating new page...`);
-    const page = await browser.newPage();
-    if (userAgent) {
-        log(`   Setting user agent: ${userAgent}`);
-        await page.setUserAgent(userAgent);
+    try {
+        // Process each device size
+        for (const [sizeName, sizeConfig] of Object.entries(targetSizes)) {
+            const { width } = sizeConfig;
+            const resolution = 'resolution' in sizeConfig ? sizeConfig.resolution : undefined;
+            progress(`📱 Processing ${sizeName} (${width}px${resolution ? `, ${resolution}x` : ''})...`);
+            const page = await browser.newPage();
+            const sizeResources = [];
+            try {
+                await processPageForSize(page, url, baseUrl, sizeResources, {
+                    sizeName,
+                    width,
+                    resolution,
+                    timeout,
+                    log,
+                    progress,
+                });
+                // Merge resources, avoiding duplicates
+                for (const resource of sizeResources) {
+                    const existing = allResources.find((r) => r.url === resource.url);
+                    if (!existing) {
+                        allResources.push(resource);
+                    }
+                    else if (!existing.content && resource.content) {
+                        existing.content = resource.content;
+                    }
+                }
+            }
+            finally {
+                await page.close().catch((error) => {
+                    log(`⚠️ Warning: Failed to close page for ${sizeName}: ${error instanceof Error ? error.message : String(error)}`);
+                });
+            }
+        }
+        const resourceCount = allResources.length;
+        const downloadedCount = allResources.filter((r) => r.content).length;
+        progress(`📄 Found ${resourceCount} total resources (${downloadedCount} downloaded successfully)`);
+        // Ensure output directory exists
+        progress(`📁 Creating output directory...`);
+        await fs.mkdir(outputDir, { recursive: true });
+        // Save all resources
+        progress(`💾 Saving files to disk...`);
+        const savedCount = await saveResources(allResources, outputDir, log, progress);
+        progress(`🎉 Replication complete! ${savedCount} files saved to ${outputDir}`);
+    }
+    finally {
+        progress(`🔧 Cleaning up browser...`);
+        await browser.close().catch((error) => {
+            // Log browser close errors but don't throw them
+            log(`⚠️ Warning: Failed to close browser: ${error instanceof Error ? error.message : String(error)}`);
+        });
     }
-    progress(`🔍 Setting up resource detection...`);
-    // Collect all requests
+}
+/**
+ * Process a page for a specific device size
+ * @param page
+ * @param url
+ * @param baseUrl
+ * @param resources
+ * @param options
+ * @param options.sizeName
+ * @param options.width
+ * @param options.resolution
+ * @param options.timeout
+ * @param options.log
+ * @param options.progress
+ */
+async function processPageForSize(page, url, baseUrl, resources, options) {
+    const { sizeName, width, resolution, log, progress } = options;
     const requestPromises = [];
+    // Set up resource detection
     page.on('request', (request) => {
         const requestUrl = request.url();
         const requestUrlObj = new URL(requestUrl);
         // Only handle same-host resources
         if (requestUrlObj.hostname === baseUrl.hostname) {
-            log(`📥 Intercepting: ${requestUrl}`);
+            log(`📥 [${sizeName}] Intercepting: ${requestUrl}`);
             const localPath = urlToLocalPath(requestUrl);
             const resourceType = getResourceType(requestUrl);
-            resources.push({
-                url: requestUrl,
-                localPath,
-                type: resourceType,
-            });
+            // Check if this resource is already tracked
+            if (!resources.some((r) => r.url === requestUrl)) {
+                resources.push({
+                    url: requestUrl,
+                    localPath,
+                    type: resourceType,
+                });
+            }
         }
         else {
-            log(`🚫 Skipping external resource: ${requestUrl}`);
+            log(`🚫 [${sizeName}] Skipping external resource: ${requestUrl}`);
         }
     });
     page.on('response', (response) => {
@@ -69,47 +141,67 @@ export async function replicate(url, outputDir, options = {}) {
                         .buffer()
                         .then((buffer) => {
                         resource.content = buffer;
-                        log(`✅ Downloaded: ${responseUrl}`);
+                        log(`✅ [${sizeName}] Downloaded: ${responseUrl}`);
                     })
                         .catch((error) => {
                         const errorMessage = error instanceof Error ? error.message : String(error);
-                        log(`❌ Failed to download: ${responseUrl} - ${errorMessage}`);
-                        // Don't rethrow here as this would break the entire operation
-                        // Individual resource failures should not stop the whole process
+                        log(`❌ [${sizeName}] Failed to download: ${responseUrl} - ${errorMessage}`);
                     });
                 }
                 else if (resource) {
-                    log(`❌ Resource failed (${response.status()}): ${responseUrl}`);
+                    log(`❌ [${sizeName}] Resource failed (${response.status()}): ${responseUrl}`);
                 }
             })();
             requestPromises.push(promise);
         }
     });
-    try {
-        // Navigate to the page
-        progress(`📡 Navigating to ${url}...`);
-        await page.goto(url, { waitUntil: 'networkidle2', timeout });
-        progress(`⏳ Waiting for all resources to load...`);
-        // Wait for all downloads to complete
-        await Promise.all(requestPromises);
-        const resourceCount = resources.length;
-        const downloadedCount = resources.filter((r) => r.content).length;
-        progress(`📄 Found ${resourceCount} resources (${downloadedCount} downloaded successfully)`);
-        // Ensure output directory exists
-        progress(`📁 Creating output directory...`);
-        await fs.mkdir(outputDir, { recursive: true });
-        // Save all resources
-        progress(`💾 Saving files to disk...`);
-        const savedCount = await saveResources(resources, outputDir, log, progress);
-        progress(`🎉 Replication complete! ${savedCount} files saved to ${outputDir}`);
-    }
-    finally {
-        progress(`🔧 Cleaning up browser...`);
-        await browser.close().catch((error) => {
-            // Log browser close errors but don't throw them
-            log(`⚠️ Warning: Failed to close browser: ${error instanceof Error ? error.message : String(error)}`);
-        });
-    }
+    // Set viewport and navigate using beforePageScan (which includes scrolling)
+    progress(`📡 [${sizeName}] Setting viewport and navigating...`);
+    await beforePageScan(page, url, {
+        name: sizeName,
+        width,
+        resolution,
+        listener: (phase, data) => {
+            switch (phase) {
+                case 'setViewport': {
+                    const setViewportData = data;
+                    log(`📱 [${sizeName}] Viewport set: ${setViewportData.width}px${setViewportData.resolution ? ` @ ${setViewportData.resolution}x` : ''}`);
+                    break;
+                }
+                case 'load': {
+                    const loadData = data;
+                    log(`📄 [${sizeName}] Page loaded (${loadData.type})`);
+                    break;
+                }
+                case 'scroll': {
+                    const scrollData = data;
+                    switch (scrollData.message) {
+                        case 'Start scrolling': {
+                            log(`📜 [${sizeName}] Starting scroll to trigger lazy loading...`);
+                            break;
+                        }
+                        case 'End of page': {
+                            log(`📜 [${sizeName}] Scroll completed (${scrollData.scrollY}/${scrollData.scrollHeight}px)`);
+                            break;
+                        }
+                        case 'Scrolling': {
+                            const progress = Math.round((scrollData.scrollY / scrollData.scrollHeight) * 100);
+                            log(`📜 [${sizeName}] Scrolling progress: ${progress}% (${scrollData.scrollY}/${scrollData.scrollHeight}px)`);
+                            break;
+                        }
+                        // No default
+                    }
+                    break;
+                }
+            }
+        },
+    });
+    progress(`⏳ [${sizeName}] Waiting for all resources to load...`);
+    // Wait for all downloads to complete
+    await Promise.all(requestPromises);
+    const resourceCount = resources.length;
+    const downloadedCount = resources.filter((r) => r.content).length;
+    progress(`📄 [${sizeName}] Found ${resourceCount} resources (${downloadedCount} downloaded)`);
 }
 /**
  *

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.2.0",
+	"version": "0.3.0",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,14 +24,16 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.1.0",
-		"@d-zero/shared": "0.9.1",
+		"@d-zero/cli-core": "1.1.1",
+		"@d-zero/puppeteer-page-scan": "4.0.4",
+		"@d-zero/puppeteer-scroll": "3.0.5",
+		"@d-zero/shared": "0.9.2",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
-		"puppeteer": "24.12.0"
+		"puppeteer": "24.17.1"
 	},
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "7cc778738d8c811adb69cee528655e12eba52e87"
+	"gitHead": "6fa0fd190c54351762a70039cd8dfcafc51caa0b"
 }