html-to-gutenberg 4.2.9 → 4.2.11

package/vendor/fetch-page-assets/README.md

@@ -0,0 +1,117 @@
+# fetch-page-assets
+
+Download page assets, rewrite the HTML to point at the fetched assets, and optionally upload those downloads directly to Cloudflare R2 instead of writing them to disk.
+
+## Installation
+
+```bash
+npm install fetch-page-assets
+```
+
+## Environment
+
+Keep real secrets in `.env` and never commit them.
+
+```bash
+cp .env.example .env
+```
+
+Required for R2 uploads:
+
+- `CLOUDFLARE_R2_ACCOUNT_ID`
+- `CLOUDFLARE_R2_BUCKET`
+- `CLOUDFLARE_R2_ACCESS_KEY_ID`
+- `CLOUDFLARE_R2_SECRET_ACCESS_KEY`
+- `CLOUDFLARE_R2_PUBLIC_BASE_URL`
+
+Optional:
+
+- `CLOUDFLARE_API_TOKEN`
+- `CLOUDFLARE_R2_ENDPOINT`
+
+## Getting and rotating Cloudflare credentials
+
+1. Open the Cloudflare dashboard.
+2. Create or rotate the R2 access keys for the bucket you want to use.
+3. Update `.env` with the new key values.
+4. If you use a Cloudflare API token for verification or other account workflows, rotate it in the API Tokens section and update `.env`.
+5. Restart the service after updating `.env`.
+6. Revoke the old token or key after the new one is confirmed working.
+
+To verify a token without exposing it in source code:
+
+```bash
+curl "https://api.cloudflare.com/client/v4/user/tokens/verify" \
+  -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN"
+```
+
+## Usage
+
+### Legacy local mode
+
+```js
+import extractAssets from 'fetch-page-assets';
+
+const html = await extractAssets('<img src="/logo.png" />', {
+  source: 'https://example.com',
+  basePath: process.cwd(),
+  saveFile: true
+});
+```
+
+### R2 upload mode
+
+```js
+import extractAssets from 'fetch-page-assets';
+
+const result = await extractAssets('<img src="/logo.png" />', {
+  source: 'https://example.com',
+  uploadToR2: true,
+  returnDetails: true,
+  jobId: 'conv_123',
+  r2Prefix: 'generated/conv_123/assets'
+});
+
+console.log(result);
+```
+
+Example response:
+
+```json
+{
+  "html": "<img src=\"https://storage.example.com/generated/conv_123/assets/logo.png\">",
+  "assets": [
+    {
+      "id": "file_1",
+      "name": "logo.png",
+      "type": "image/png",
+      "size": 48211,
+      "path": "/generated/conv_123/assets/logo.png",
+      "url": "https://storage.example.com/generated/conv_123/assets/logo.png",
+      "kind": "asset"
+    }
+  ]
+}
+```
+
+## Options
+
+| Option | Description | Type | Default |
+| --- | --- | --- | --- |
+| `source` | Base URL used to resolve relative asset paths. | `string` | `''` |
+| `basePath` | Local base path used in legacy mode. | `string` | current directory |
+| `saveFile` | Writes downloaded assets to disk in legacy mode. | `boolean` | `true` |
+| `uploadToR2` | Uploads resolved assets to Cloudflare R2. | `boolean` | `false` |
+| `returnDetails` | Returns `{ html, assets }` metadata instead of only the rewritten HTML string. | `boolean` | `false` |
+| `jobId` | Stable conversion identifier used to build remote paths. | `string` | `conv_local` |
+| `r2Prefix` | Remote storage prefix for uploaded assets. | `string` | derived from `jobId` |
+| `concurrency` | Maximum number of simultaneous downloads. | `number` | `8` |
+| `maxRetryAttempts` | Maximum attempts per asset. | `number` | `3` |
+| `retryDelay` | Delay between attempts in milliseconds. | `number` | `1000` |
+| `verbose` | Enables console logging. | `boolean` | `true` |
+
+## Notes
+
+- `returnDetails: true` is the recommended mode when using R2 because it gives you the uploaded asset metadata.
+- Keep all Cloudflare credentials in `.env`.
+- Do not hardcode tokens or access keys in code, documentation, tests, or shell history.

package/vendor/fetch-page-assets/index.js

@@ -0,0 +1,362 @@
+import https from "https";
+import http from "http";
+import fs from "fs";
+import path from "path";
+import beautify from "beautify";
+import { decode } from "html-entities";
+import mime from "mime";
+import { createFileRecord, inferContentType, uploadBufferToR2 } from "../../r2.js";
+export const h = (t, i = true) => { if (i)
+    console.error(`[Error] ${t}`); };
+export const p = (t, i = true) => { if (i)
+    console.log(`[Success] ${t}`); };
+export const d = (t) => t.startsWith("//");
+export const m = (t, r = "https") => (d(t) ? `${r}:${t}` : t);
+export const u = (t) => t.endsWith("/");
+export const w = (t) => !t.startsWith("http") && !d(t);
+export const $ = (e) => e.trim().replace(/^['"]|['"]$/g, "");
+export const E = (relativePath, a = "", r = "https") => {
+    try {
+        if (w(relativePath)) {
+            const cleanPath = $(relativePath);
+            const resolved = new URL(cleanPath, a);
+            return resolved.href;
+        }
+        return m(relativePath, r);
+    }
+    catch (err) {
+        h(`Error resolving path: ${relativePath} — ${err.message}`);
+        return relativePath;
+    }
+};
+export const g = (t) => path.join(...t);
+export const U = (t, i = true) => {
+    fs.mkdirSync(t, { recursive: !0 });
+    p(`Directory ensured: ${t}`, i);
+};
+export const v = (t, l, s, c, i = true) => {
+    if (c) {
+        const { parsedUrl: e, destinationFilePath: a } = t;
+        const { origin: r } = new URL(e);
+        const urlStr = typeof e === "string" ? e : e.toString();
+        let relativeLocalPath = path.relative(s, a).split(path.sep).join("/");
+        l = l.replaceAll(urlStr, relativeLocalPath);
+        l = l.replaceAll(`${r}${urlStr}`, relativeLocalPath);
+        fs.writeFileSync(path.join(s, "index.html"), beautify(l, { format: "html" }), "utf8");
+        p(`Updated HTML with local asset path for ${urlStr} -> ${relativeLocalPath}`, i);
+    }
+};
+export const A = (t) => t.split("?")[0].split("#")[0];
+export const F = (t, e) => A(path.join(t, e));
+export const x = (t) => t.split(".");
+export const P = (t, e) => t[e] || t[e.toLowerCase()];
+export const R = (headers, fallback) => {
+    let filename = P(headers, "Content-Disposition")?.match(/filename="(.+?)"/)?.[1] || fallback;
+    filename = filename?.split("?")[0].split("#")[0];
+    filename = filename.replace(/[^a-zA-Z0-9.\-_]/g, "_");
+    const contentType = P(headers, "Content-Type");
+    const hasExt = filename.includes(".");
+    if (!hasExt && contentType) {
+        const ext = mime.getExtension(contentType);
+        if (ext) {
+            filename = `${filename}.${ext}`;
+        }
+    }
+    return filename;
+};
+export const D = (t) => {
+    const { loaded: e, total: s } = t;
+    const a = e && s ? Math.round((e / s) * 100) : 0;
+    if (!isNaN(a))
+        console.log(`Download progress: ${a}%`);
+};
+const extractAssets = async (t, e = {}) => {
+    let { basePath: s = process.cwd(), source: a = "", protocol: r = "https", maxRetryAttempts: o = 3, retryDelay: n = 1000, verbose: i = true, saveFile: c = true, concurrency: y = 8, uploadToR2: k = false, returnDetails: q = false, jobId: z = "conv_local", r2Prefix: G, _assetTaskCache: H, _ensuredDirs: B } = e;
+    a = a || "";
+    r = r || "https";
+    n = n || 1000;
+    s = s || process.cwd();
+    o = Math.max(1, o || 3);
+    y = Math.max(1, Number.isFinite(y) ? Math.floor(y) : 8);
+    let l = "";
+    const uploadedAssets = [];
+    const assetTaskCache = H instanceof Map ? H : new Map();
+    const ensuredDirs = B instanceof Set ? B : new Set();
+    const h = (message) => {
+        if (i) {
+            console.error(`[Error] ${message}`);
+        }
+    };
+    const p = (message) => {
+        if (i) {
+            console.log(`[Success] ${message}`);
+        }
+    };
+    const sleep = (delay) => new Promise((resolve) => setTimeout(resolve, Math.max(0, delay)));
+    const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const isProtocolRelative = (value) => value.startsWith("//");
+    const applyProtocol = (value) => (isProtocolRelative(value) ? `${r}:${value}` : value);
+    const isRelativePath = (value) => !value.startsWith("http") && !isProtocolRelative(value);
+    const stripQuotes = (value) => value.trim().replace(/^['"]|['"]$/g, "");
+    const ensureDir = (dirPath) => {
+        if (!dirPath || ensuredDirs.has(dirPath)) {
+            return;
+        }
+        fs.mkdirSync(dirPath, { recursive: true });
+        ensuredDirs.add(dirPath);
+        p(`Directory ensured: ${dirPath}`);
+    };
+    const resolveAssetUrl = (relativePath) => {
+        try {
+            if (isRelativePath(relativePath)) {
+                const cleanPath = stripQuotes(relativePath);
+                if (!a) {
+                    throw new Error("A source URL is required to resolve relative asset paths.");
+                }
+                return new URL(cleanPath, a).href;
+            }
+            return applyProtocol(relativePath);
+        }
+        catch (err) {
+            h(`Error resolving path: ${relativePath} — ${err.message}`);
+            return relativePath;
+        }
+    };
+    const pickHeader = (headers, headerName) => headers[headerName] || headers[headerName.toLowerCase()];
+    const getFileNameFromHeaders = (headers, fallback) => {
+        let filename = pickHeader(headers, "Content-Disposition")?.match(/filename="(.+?)"/)?.[1] || fallback;
+        filename = filename?.split("?")[0].split("#")[0];
+        filename = filename.replace(/[^a-zA-Z0-9._-]/g, "_");
+        const contentType = pickHeader(headers, "Content-Type");
+        if (!filename.includes(".") && contentType) {
+            const ext = mime.getExtension(contentType);
+            if (ext) {
+                filename = `${filename}.${ext}`;
+            }
+        }
+        return filename;
+    };
+    const fetchBuffer = async (url, fallbackName) => {
+        const decodedUrl = decode(url);
+        if (decodedUrl.startsWith("file://")) {
+            const localPath = decodedUrl.replace("file://", "");
+            p(`Reading local file: ${localPath}`);
+            const data = fs.readFileSync(localPath);
+            return { data, fileName: path.basename(localPath) };
+        }
+        p(`Starting download for: ${decodedUrl}`);
+        return new Promise((resolve, reject) => {
+            const client = decodedUrl.startsWith("https") ? https : http;
+            client.get(decodedUrl, {
+                headers: {
+                    "User-Agent": "Mozilla/5.0",
+                    "Accept": "*/*"
+                }
+            }, (res) => {
+                const statusCode = res.statusCode || 0;
+                const location = res.headers.location;
+                if (statusCode >= 300 && statusCode < 400 && location) {
+                    res.resume();
+                    resolve(fetchBuffer(new URL(location, decodedUrl).href, fallbackName));
+                    return;
+                }
+                if (statusCode >= 400) {
+                    res.resume();
+                    reject(new Error(`HTTP error! Status: ${statusCode}`));
+                    return;
+                }
+                const chunks = [];
+                res.on("data", (chunk) => {
+                    chunks.push(chunk);
+                });
+                res.on("end", () => {
+                    resolve({
+                        data: Buffer.concat(chunks),
+                        fileName: getFileNameFromHeaders(res.headers, fallbackName)
+                    });
+                });
+                res.on("error", reject);
+            }).on("error", reject);
+        });
+    };
+    const fetchText = async (url) => {
+        const decodedUrl = decode(url);
+        p(`Fetching content: ${decodedUrl}`);
+        return new Promise((resolve, reject) => {
+            const client = decodedUrl.startsWith("https") ? https : http;
+            client.get(decodedUrl, {
+                headers: {
+                    "User-Agent": "Mozilla/5.0",
+                    "Accept": "*/*"
+                }
+            }, (res) => {
+                const statusCode = res.statusCode || 0;
+                const location = res.headers.location;
+                if (statusCode >= 300 && statusCode < 400 && location) {
+                    res.resume();
+                    resolve(fetchText(new URL(location, decodedUrl).href));
+                    return;
+                }
+                if (statusCode >= 400) {
+                    res.resume();
+                    reject(new Error(`HTTP error! Status: ${statusCode}`));
+                    return;
+                }
+                const chunks = [];
+                res.on("data", (chunk) => {
+                    chunks.push(chunk);
+                });
+                res.on("end", () => {
+                    resolve(Buffer.concat(chunks).toString("utf-8"));
+                });
+                res.on("error", reject);
+            }).on("error", reject);
+        });
+    };
+    const isValidUrl = (value) => {
+        try {
+            return !!new URL(applyProtocol(value));
+        }
+        catch {
+            return false;
+        }
+    };
+    const hasValidHttpProtocol = (value) => {
+        const { protocol, hostname, href } = new URL(A(value));
+        if (!protocol || !["http:", "https:"].includes(protocol)) {
+            throw new Error("Invalid baseUrl. Only http and https are supported.");
+        }
+        if (!hostname) {
+            throw new Error("Invalid baseUrl. Provide a valid URL with a hostname.");
+        }
+        return !!href;
+    };
+    const loadInputHtml = async () => {
+        if (typeof t !== "string" || typeof s !== "string") {
+            h("Invalid user input: source and basePath must be strings.");
+            return;
+        }
+        if (isValidUrl(t)) {
+            try {
+                hasValidHttpProtocol(t);
+                l = await fetchText(t);
+                if (!a) {
+                    a = t;
+                }
+            }
+            catch (err) {
+                h(err.message || err);
+            }
+            return;
+        }
+        l = t;
+    };
+    const saveResolvedAsset = async (asset) => {
+        const absoluteAssetUrl = resolveAssetUrl(asset);
+        if (assetTaskCache.has(absoluteAssetUrl)) {
+            return assetTaskCache.get(absoluteAssetUrl);
+        }
+        const task = (async () => {
+            try {
+                const urlObj = new URL(absoluteAssetUrl);
+                const urlPath = urlObj.pathname.replace(/^\//, "");
+                const destinationPath = path.join(s, path.dirname(urlPath));
+                const fileNameGuess = path.basename(urlPath).split("?")[0].split("#")[0] || "asset";
+                ensureDir(destinationPath);
+                for (let attempt = 0; attempt < o; attempt++) {
+                    try {
+                        const { data, fileName } = await fetchBuffer(absoluteAssetUrl, fileNameGuess);
+                        const fullPath = path.join(destinationPath, fileName);
+                        let uploadedFile = null;
+                        if (k) {
+                            const storageKey = path.posix.join(G || `generated/${z}/assets`, path.dirname(urlPath).split(path.sep).join("/"), fileName);
+                            const uploadResult = await uploadBufferToR2({
+                                storageKey,
+                                body: data,
+                                contentType: inferContentType(fileName)
+                            });
+                            uploadedFile = createFileRecord({
+                                id: `asset_${uploadedAssets.length + 1}`,
+                                name: fileName,
+                                kind: "asset",
+                                storageKey: uploadResult.storageKey,
+                                size: uploadResult.size,
+                                type: uploadResult.type,
+                                url: uploadResult.url
+                            });
+                            uploadedAssets.push({ ...uploadedFile, buffer: data });
+                            p(`Asset uploaded successfully to ${uploadResult.url}`);
+                        }
+                        else if (c) {
+                            fs.writeFileSync(fullPath, data);
+                            p(`Asset saved successfully to ${fullPath}`);
+                        }
+                        return {
+                            parsedUrl: asset,
+                            absoluteAssetUrl,
+                            destinationPath,
+                            destinationFilePath: fullPath,
+                            fileName,
+                            uploadedFile
+                        };
+                    }
+                    catch (err) {
+                        const isLastAttempt = attempt === o - 1;
+                        if (isLastAttempt) {
+                            const { message, code } = err || {};
+                            if (["ECONNRESET", "ETIMEDOUT"].includes(code)) {
+                                h(`Network error occurred while downloading asset from ${absoluteAssetUrl}: ${message}.`);
+                            }
+                            else if (["EACCES", "EISDIR"].includes(code)) {
+                                h("Error saving asset. Permission denied or target path is a directory.");
+                            }
+                            else {
+                                h(`Error downloading asset from ${absoluteAssetUrl}: ${message || err}.`);
+                            }
+                            return null;
+                        }
+                        await sleep(n);
+                    }
+                }
+            }
+            catch (err) {
+                h(`Error downloading asset from ${absoluteAssetUrl}: ${err.message || err}.`);
+            }
+            return null;
+        })();
+        assetTaskCache.set(absoluteAssetUrl, task);
+        return task;
+    };
+    await loadInputHtml();
+    if (!l) {
+        return l;
+    }
+    l = l.replace(/srcset="(.*?)"/gi, "").replace(/sizes="(.*?)"/gi, "");
+    if (a) {
+        l = l.replace(new RegExp(escapeRegExp(a), "g"), "");
+    }
+    const regex = /(<link[^>]+rel=["']stylesheet["'][^>]+href=["'])([^"']+\.[^"']+)["']|<(img|script|source)[^>]+src=["']([^"']+\.(?!json)[^"']+)["']/gi;
+    const matches = [
+        ...[...l.matchAll(regex)].map((match) => match[2] || match[4] || ""),
+        ...[...l.matchAll(/url\(["']?(.*?)["']?\)/gi)]
+            .map((match) => match[1])
+            .filter((url) => !/^#/.test(url))
+    ].filter((match) => !!match && !match.startsWith("data:"));
+    const uniqueMatches = [...new Set(matches)];
+    const queue = [...uniqueMatches];
+    const workers = Array.from({ length: Math.min(y, queue.length || 1) }, async () => {
+        while (queue.length > 0) {
+            const asset = queue.shift();
+            if (!asset) {
+                return;
+            }
+            await saveResolvedAsset(asset);
+        }
+    });
+    await Promise.all(workers);
+    if (q) {
+        return { html: l, assets: uploadedAssets };
+    }
+    return l;
+};
+export default extractAssets;

package/vendor/fetch-page-assets/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "fetch-page-assets",
+  "version": "1.2.7",
+  "type": "module",
+  "description": "A versatile Node.js module for extracting assets (such as CSS files, JavaScript files, fonts, and images) from HTML content or URLs.",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "README.md",
+    "LICENSE.MD"
+  ],
+  "scripts": {
+    "test": "node -e \"console.log('No vendored tests configured')\""
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/DiogoAngelim/fetch-page-assets.git"
+  },
+  "keywords": [
+    "backend",
+    "asset",
+    "extractor",
+    "downloader",
+    "html",
+    "css",
+    "javascript",
+    "image",
+    "fetch",
+    "save"
+  ],
+  "author": "Diogo Angelim",
+  "license": "MIT",
+  "funding": "https://www.paypal.com/donate/?hosted_button_id=XA5LN4XR39PMQ",
+  "bugs": {
+    "url": "https://github.com/DiogoAngelim/fetch-page-assets/issues"
+  },
+  "homepage": "https://github.com/DiogoAngelim/fetch-page-assets#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "beautify": "^0.0.8",
+    "fs": "^0.0.1-security",
+    "html-entities": "^2.5.2",
+    "mime": "^4.0.3",
+    "path": "^0.12.7"
+  }
+}

package/.env

@@ -1 +0,0 @@
-SNAPAPI_KEY=sk_live_5b420427f2ec2509a0971c267b322300efec77a498c44ada