@d-zero/replicator 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 D-ZERO Co., Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,37 @@
+# @d-zero/replicator
+
+Replicate web pages with all their resources to local directories
+
+## Installation
+
+```bash
+npm install @d-zero/replicator
+```
+
+## Usage
+
+### CLI
+
+```bash
+npx @d-zero/replicator <url> -o <output-directory>
+```
+
+### Programmatic
+
+```typescript
+import { replicate } from '@d-zero/replicator';
+
+await replicate('https://example.com', './output');
+```
+
+## Features
+
+- Download HTML pages preserving directory structure
+- Fetch all related resources (CSS, JS, images, etc.)
+- Maintain relative links between resources
+- Support for same-host resources only
+- Preserve original file extensions and paths
+
+## License
+
+MIT

package/dist/cli.js

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+import { createCLI, parseCommonOptions } from '@d-zero/cli-core';
+import { replicate } from './index.js';
+const { options, args } = createCLI({
+    aliases: {
+        o: 'output',
+        v: 'verbose',
+    },
+    usage: ['Usage: replicator <url> -o <output-directory> [--verbose]'],
+    parseArgs: (cli) => ({
+        ...parseCommonOptions(cli),
+        output: cli.output,
+    }),
+    validateArgs: (options, cli) => {
+        return !!(cli._.length > 0 && options.output);
+    },
+});
+const url = args[0];
+const outputDir = options.output;
+if (!url || typeof url !== 'string') {
+    // eslint-disable-next-line no-console
+    console.error('❌ Error: URL is required');
+    process.exit(1);
+}
+try {
+    await replicate(url, outputDir, {
+        verbose: options.verbose ?? false,
+    });
+    // eslint-disable-next-line no-console
+    console.log(`✅ Successfully replicated ${url} to ${outputDir}`);
+}
+catch (error) {
+    if (error instanceof Error) {
+        // eslint-disable-next-line no-console
+        console.error('❌ Error:', error.message);
+        if (options.verbose) {
+            // eslint-disable-next-line no-console
+            console.error('Stack trace:', error.stack);
+        }
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.error('❌ Unknown error:', error);
+    }
+    process.exit(1);
+}

package/dist/index.js

@@ -0,0 +1,195 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { URL } from 'node:url';
+import { launch } from 'puppeteer';
+/**
+ *
+ * @param url
+ * @param outputDir
+ * @param options
+ */
+export async function replicate(url, outputDir, options = {}) {
+    const { verbose = false, userAgent, timeout = 30_000 } = options;
+    const log = (message) => {
+        if (verbose) {
+            // eslint-disable-next-line no-console
+            console.log(message);
+        }
+    };
+    // Always show these key progress messages
+    const progress = (message) => {
+        // eslint-disable-next-line no-console
+        console.log(message);
+    };
+    const baseUrl = new URL(url);
+    const resources = [];
+    progress(`🚀 Starting replication of ${url}`);
+    log(`   Output directory: ${outputDir}`);
+    progress(`🌐 Launching browser...`);
+    const browser = await launch({
+        headless: true,
+        timeout,
+    });
+    progress(`📄 Creating new page...`);
+    const page = await browser.newPage();
+    if (userAgent) {
+        log(`   Setting user agent: ${userAgent}`);
+        await page.setUserAgent(userAgent);
+    }
+    progress(`🔍 Setting up resource detection...`);
+    // Collect all requests
+    const requestPromises = [];
+    page.on('request', (request) => {
+        const requestUrl = request.url();
+        const requestUrlObj = new URL(requestUrl);
+        // Only handle same-host resources
+        if (requestUrlObj.hostname === baseUrl.hostname) {
+            log(`📥 Intercepting: ${requestUrl}`);
+            const localPath = urlToLocalPath(requestUrl);
+            const resourceType = getResourceType(requestUrl);
+            resources.push({
+                url: requestUrl,
+                localPath,
+                type: resourceType,
+            });
+        }
+        else {
+            log(`🚫 Skipping external resource: ${requestUrl}`);
+        }
+    });
+    page.on('response', (response) => {
+        const responseUrl = response.url();
+        const responseUrlObj = new URL(responseUrl);
+        // Only handle same-host resources
+        if (responseUrlObj.hostname === baseUrl.hostname) {
+            const promise = (async () => {
+                const resource = resources.find((r) => r.url === responseUrl);
+                if (resource && response.ok()) {
+                    await response
+                        .buffer()
+                        .then((buffer) => {
+                        resource.content = buffer;
+                        log(`✅ Downloaded: ${responseUrl}`);
+                    })
+                        .catch((error) => {
+                        const errorMessage = error instanceof Error ? error.message : String(error);
+                        log(`❌ Failed to download: ${responseUrl} - ${errorMessage}`);
+                        // Don't rethrow here as this would break the entire operation
+                        // Individual resource failures should not stop the whole process
+                    });
+                }
+                else if (resource) {
+                    log(`❌ Resource failed (${response.status()}): ${responseUrl}`);
+                }
+            })();
+            requestPromises.push(promise);
+        }
+    });
+    try {
+        // Navigate to the page
+        progress(`📡 Navigating to ${url}...`);
+        await page.goto(url, { waitUntil: 'networkidle2', timeout });
+        progress(`⏳ Waiting for all resources to load...`);
+        // Wait for all downloads to complete
+        await Promise.all(requestPromises);
+        const resourceCount = resources.length;
+        const downloadedCount = resources.filter((r) => r.content).length;
+        progress(`📄 Found ${resourceCount} resources (${downloadedCount} downloaded successfully)`);
+        // Ensure output directory exists
+        progress(`📁 Creating output directory...`);
+        await fs.mkdir(outputDir, { recursive: true });
+        // Save all resources
+        progress(`💾 Saving files to disk...`);
+        const savedCount = await saveResources(resources, outputDir, log, progress);
+        progress(`🎉 Replication complete! ${savedCount} files saved to ${outputDir}`);
+    }
+    finally {
+        progress(`🔧 Cleaning up browser...`);
+        await browser.close().catch((error) => {
+            // Log browser close errors but don't throw them
+            log(`⚠️ Warning: Failed to close browser: ${error instanceof Error ? error.message : String(error)}`);
+        });
+    }
+}
+/**
+ *
+ * @param resources
+ * @param outputDir
+ * @param log
+ * @param progress
+ */
+async function saveResources(resources, outputDir, log, progress) {
+    let savedCount = 0;
+    const totalResources = resources.filter((r) => r.content).length;
+    for (const resource of resources) {
+        if (resource.content) {
+            const fullPath = path.join(outputDir, resource.localPath);
+            const dir = path.dirname(fullPath);
+            try {
+                await fs.mkdir(dir, { recursive: true });
+                await fs.writeFile(fullPath, resource.content);
+                savedCount++;
+                // Show progress every 10 files or for the last file
+                if (savedCount % 10 === 0 || savedCount === totalResources) {
+                    progress(`   Saved ${savedCount}/${totalResources} files...`);
+                }
+                log(`💾 Saved: ${resource.localPath}`);
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                log(`❌ Failed to save ${resource.localPath}: ${errorMessage}`);
+                progress(`   ⚠️  Failed to save ${resource.localPath}`);
+                // Continue with other resources instead of failing completely
+            }
+        }
+    }
+    return savedCount;
+}
+/**
+ *
+ * @param url
+ */
+function urlToLocalPath(url) {
+    const urlObj = new URL(url);
+    let pathname = urlObj.pathname;
+    // Remove leading slash
+    if (pathname.startsWith('/')) {
+        pathname = pathname.slice(1);
+    }
+    // If path is empty or ends with /, treat as index.html
+    if (pathname === '' || pathname.endsWith('/')) {
+        pathname = pathname + 'index.html';
+    }
+    // If no extension, add .html
+    if (!pathname.includes('.')) {
+        pathname = pathname + '.html';
+    }
+    return pathname;
+}
+/**
+ *
+ * @param url
+ */
+function getResourceType(url) {
+    const urlObj = new URL(url);
+    const pathname = urlObj.pathname.toLowerCase();
+    if (pathname.endsWith('.html') ||
+        pathname.endsWith('.htm') ||
+        pathname === '/' ||
+        !pathname.includes('.')) {
+        return 'html';
+    }
+    if (pathname.endsWith('.css')) {
+        return 'css';
+    }
+    if (pathname.endsWith('.js') || pathname.endsWith('.mjs')) {
+        return 'js';
+    }
+    if (/\.(?:jpg|jpeg|png|gif|svg|webp|ico)$/.test(pathname)) {
+        return 'image';
+    }
+    if (/\.(?:woff|woff2|ttf|otf|eot)$/.test(pathname)) {
+        return 'font';
+    }
+    return 'other';
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,37 @@
+{
+	"name": "@d-zero/replicator",
+	"version": "0.2.0",
+	"description": "Replicate web pages with all their resources to local directories",
+	"author": "D-ZERO",
+	"license": "MIT",
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"exports": {
+		".": {
+			"import": "./dist/index.js",
+			"types": "./dist/index.d.ts"
+		}
+	},
+	"bin": "./dist/cli.js",
+	"files": [
+		"dist"
+	],
+	"scripts": {
+		"build": "tsc",
+		"watch": "tsc --watch",
+		"clean": "tsc --build --clean"
+	},
+	"dependencies": {
+		"@d-zero/cli-core": "1.1.0",
+		"@d-zero/shared": "0.9.1",
+		"ansi-colors": "4.1.3",
+		"minimist": "1.2.8",
+		"puppeteer": "24.12.0"
+	},
+	"devDependencies": {
+		"@types/minimist": "1.2.5"
+	},
+	"gitHead": "7cc778738d8c811adb69cee528655e12eba52e87"
+}