@d-zero/replicator 0.7.0 → 0.7.2

package/dist/child-process.js

@@ -1,28 +1,7 @@
 import { createChildProcess } from '@d-zero/puppeteer-dealer';
 import { beforePageScan, devicePresets } from '@d-zero/puppeteer-page-scan';
 import { scrollAllOver } from '@d-zero/puppeteer-scroll';
-/**
- * Add resource path to the set with MIME type encoding if needed
- * @param pathname - Resource pathname
- * @param mimeType - MIME type from response headers (optional)
- * @returns Encoded resource path
- */
-function encodeResourcePath(pathname, mimeType) {
-    // Normalize empty pathname to "/"
-    if (pathname === '') {
-        pathname = '/';
-    }
-    // Check if the last segment has an extension
-    const lastSlashIndex = pathname.lastIndexOf('/');
-    const lastSegment = lastSlashIndex === -1 ? pathname : pathname.slice(lastSlashIndex + 1);
-    const hasExtension = lastSegment.includes('.');
-    // For paths without extension, encode with MIME type if available
-    if (!hasExtension && mimeType) {
-        return `${pathname}:::${mimeType}`;
-    }
-    // For paths with extension or without MIME type, return as-is
-    return pathname;
-}
+import { encodeResourcePath } from '@d-zero/shared/encode-resource-path';
 createChildProcess((param) => {
     const { devices, timeout } = param;
     return {
@@ -31,8 +10,7 @@ createChildProcess((param) => {
             const pageHostname = new URL(url).hostname;
             // Add the page URL itself first (in case response event is missed)
             const pageUrlObj = new URL(url);
-            const pagePathname = pageUrlObj.pathname;
-            resourcePaths.add(encodeResourcePath(pagePathname, 'text/html'));
+            resourcePaths.add(encodeResourcePath(pageUrlObj, 'text/html'));
             // Listen to all network responses
             const responseHandler = (response) => {
                 const responseUrl = response.url();
@@ -53,12 +31,11 @@ createChildProcess((param) => {
                 if (resourceUrlObj.hostname !== pageHostname) {
                     return;
                 }
-                // Get pathname and MIME type
-                const resourcePath = resourceUrlObj.pathname;
+                // Get MIME type
                 const contentType = response.headers()['content-type'];
                 const mimeType = contentType?.split(';')[0]?.trim();
                 // Add resource with MIME encoding if needed
-                resourcePaths.add(encodeResourcePath(resourcePath, mimeType));
+                resourcePaths.add(encodeResourcePath(resourceUrlObj, mimeType));
             };
             page.on('response', responseHandler);
             const defaultSizes = {

package/dist/index.js

@@ -1,31 +1,10 @@
 import path from 'node:path';
 import { deal, createProcess } from '@d-zero/puppeteer-dealer';
 import { devicePresets } from '@d-zero/puppeteer-page-scan';
+import { encodeResourcePath } from '@d-zero/shared/encode-resource-path';
 import { validateSameHost } from '@d-zero/shared/validate-same-host';
 import c from 'ansi-colors';
 import { downloadResources } from './resource-downloader.js';
-/**
- * Encode resource path with MIME type if needed
- * @param pathname - Resource pathname
- * @param mimeType - MIME type (optional)
- * @returns Encoded resource path
- */
-function encodeResourcePath(pathname, mimeType) {
-    // Normalize empty pathname to "/"
-    if (pathname === '') {
-        pathname = '/';
-    }
-    // Check if the last segment has an extension
-    const lastSlashIndex = pathname.lastIndexOf('/');
-    const lastSegment = lastSlashIndex === -1 ? pathname : pathname.slice(lastSlashIndex + 1);
-    const hasExtension = lastSegment.includes('.');
-    // For paths without extension, encode with MIME type if available
-    if (!hasExtension && mimeType) {
-        return `${pathname}:::${mimeType}`;
-    }
-    // For paths with extension or without MIME type, return as-is
-    return pathname;
-}
 /**
  * Collect page URLs without resource scanning (page-only mode)
  * @param urls - Array of URLs to process
@@ -38,9 +17,8 @@ function collectPageUrlsOnly(urls, progress) {
     const encodedUrls = new Set();
     for (const url of urls) {
         const urlObj = new URL(url);
-        const pathname = urlObj.pathname || '/';
         // Encode as HTML page
-        const encodedPath = encodeResourcePath(pathname, 'text/html');
+        const encodedPath = encodeResourcePath(urlObj, 'text/html');
         encodedUrls.add(encodedPath);
     }
     progress(c.bold.green(`✅ Prepared ${encodedUrls.size} page(s) for download`));

package/dist/resource-downloader.js

@@ -1,31 +1,8 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { deal } from '@d-zero/dealer';
-import { mimeToExtension } from '@d-zero/shared/mime-to-extension';
-import { urlToLocalPath } from '@d-zero/shared/url-to-local-path';
+import { parseEncodedPath } from '@d-zero/shared/encode-resource-path';
 import c from 'ansi-colors';
-/**
- * Parse encoded pathname and return the actual URL and local path
- * @param encodedPath - pathname or "pathname:::MIME/type" format
- * @param baseUrl - Base URL to construct full URL from pathname
- */
-function parseEncodedPath(encodedPath, baseUrl) {
-    const parts = encodedPath.split(':::');
-    if (parts.length === 2) {
-        // Format: "pathname:::MIME/type"
-        const pathname = parts[0];
-        const mimeType = parts[1];
-        const url = new URL(pathname, baseUrl).href;
-        const extension = mimeToExtension(mimeType);
-        const localPath = urlToLocalPath(url, extension);
-        return { url, localPath };
-    }
-    // Regular pathname without MIME encoding
-    const pathname = encodedPath;
-    const url = new URL(pathname, baseUrl).href;
-    const localPath = urlToLocalPath(url, '');
-    return { url, localPath };
-}
 /**
  * Download and save resources to disk
  * @param encodedPaths - Array of encoded pathnames

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/replicator",
-	"version": "0.7.0",
+	"version": "0.7.2",
 	"description": "Replicate web pages with all their resources to local directories",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,12 +24,12 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/cli-core": "1.2.0",
-		"@d-zero/dealer": "1.4.0",
-		"@d-zero/puppeteer-dealer": "0.5.10",
-		"@d-zero/puppeteer-page-scan": "4.2.6",
-		"@d-zero/puppeteer-scroll": "3.1.0",
-		"@d-zero/shared": "0.13.0",
+		"@d-zero/cli-core": "1.2.2",
+		"@d-zero/dealer": "1.4.2",
+		"@d-zero/puppeteer-dealer": "0.5.12",
+		"@d-zero/puppeteer-page-scan": "4.2.8",
+		"@d-zero/puppeteer-scroll": "3.1.2",
+		"@d-zero/shared": "0.15.0",
 		"ansi-colors": "4.1.3",
 		"minimist": "1.2.8",
 		"puppeteer": "24.27.0"
@@ -37,5 +37,5 @@
 	"devDependencies": {
 		"@types/minimist": "1.2.5"
 	},
-	"gitHead": "85abd39686d2ce02c7b8db071437464d212dd982"
+	"gitHead": "858864f8e57494a5a872431e1d772e18e4cc843a"
 }