@pagepocket/lib 0.7.1 → 0.8.0

package/dist/resource-proxy/multimap.js

@@ -0,0 +1,18 @@
+export const addMulti = (map, key, value) => {
+    const existing = map.get(key);
+    if (!existing) {
+        map.set(key, value);
+        return;
+    }
+    if (Array.isArray(existing)) {
+        existing.push(value);
+        return;
+    }
+    map.set(key, [existing, value]);
+};
+export const toArray = (value) => {
+    if (!value) {
+        return [];
+    }
+    return Array.isArray(value) ? value : [value];
+};

package/dist/resource-proxy/pathname-variants.d.ts

@@ -0,0 +1,3 @@
+export declare const makePathnameVariants: (pathname: string) => string[];
+export declare const getBasename: (pathname: string) => string;
+export declare const stripHashFromUrlString: (url: string) => string;

package/dist/resource-proxy/pathname-variants.js

@@ -0,0 +1,54 @@
+import { stripHash, stripTrailingSlash } from "@pagepocket/shared";
+const isLikelyHostname = (value) => {
+    if (!value)
+        return false;
+    if (value === "localhost")
+        return true;
+    return value.includes(".");
+};
+const encodeEmbeddedUrlTailIfPresent = (pathname) => {
+    const raw = String(pathname || "");
+    if (!raw.includes("/http")) {
+        return null;
+    }
+    const parts = raw.split("/");
+    for (let i = 0; i < parts.length; i += 1) {
+        const scheme = parts[i];
+        if (scheme !== "http:" && scheme !== "https:") {
+            continue;
+        }
+        const hasDoubleSlash = parts[i + 1] === "";
+        const host = parts[i + 2] || "";
+        if (!hasDoubleSlash || !isLikelyHostname(host)) {
+            continue;
+        }
+        const embedded = scheme + "//" + parts.slice(i + 2).join("/");
+        const encoded = encodeURIComponent(embedded);
+        const nextParts = parts.slice(0, i).concat(encoded);
+        const rebuilt = nextParts.join("/") || "/";
+        return rebuilt.startsWith("/") ? rebuilt : "/" + rebuilt;
+    }
+    return null;
+};
+export const makePathnameVariants = (pathname) => {
+    const variants = new Set();
+    const push = (value) => {
+        if (!value)
+            return;
+        variants.add(value);
+    };
+    push(pathname);
+    push(stripTrailingSlash(pathname));
+    const encodedTail = encodeEmbeddedUrlTailIfPresent(pathname);
+    if (encodedTail && encodedTail !== pathname) {
+        push(encodedTail);
+        push(stripTrailingSlash(encodedTail));
+    }
+    return Array.from(variants);
+};
+export const getBasename = (pathname) => {
+    const clean = pathname.split("?")[0] || "";
+    const parts = clean.split("/").filter(Boolean);
+    return parts[parts.length - 1] || "";
+};
+export const stripHashFromUrlString = (url) => stripHash(url);

package/dist/resource-proxy.d.ts

@@ -1,4 +1,5 @@
-import type { ResourceType } from "./types";
+import { type MultiMap } from "./resource-proxy/multimap.js";
+import type { ResourceType } from "./types.js";
 export type ResourcesPathSnapshotItem = {
     url: string;
     path: string;
@@ -16,10 +17,11 @@ type IndexedItem = ResourcesPathSnapshotItem & {
     pathname: string;
     pathnameWithSearch: string;
     basename: string;
+    canonicalUrl: string;
 };
-type MultiMap<K, V> = Map<K, V | V[]>;
 export type ResourceProxyIndex = {
     byExactUrl: Map<string, IndexedItem>;
+    byCanonicalUrl: Map<string, IndexedItem>;
     byPathnameWithSearch: MultiMap<string, IndexedItem>;
     byPathname: MultiMap<string, IndexedItem>;
     byBasename: MultiMap<string, IndexedItem>;

package/dist/resource-proxy.js

@@ -1,113 +1,15 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.resolveToLocalPath = exports.buildResourceProxyIndex = void 0;
-const addMulti = (map, key, value) => {
-    const existing = map.get(key);
-    if (!existing) {
-        map.set(key, value);
-        return;
-    }
-    if (Array.isArray(existing)) {
-        existing.push(value);
-        return;
-    }
-    map.set(key, [existing, value]);
-};
-const toArray = (value) => {
-    if (!value) {
-        return [];
-    }
-    return Array.isArray(value) ? value : [value];
-};
-const stripHash = (value) => {
-    const index = value.indexOf("#");
-    return index === -1 ? value : value.slice(0, index);
-};
-const stripTrailingSlash = (value) => {
-    if (!value || value === "/") {
-        return value;
-    }
-    return value.endsWith("/") ? value.slice(0, -1) : value;
-};
-const looksAlreadyEscapedForStaticServers = (value) => {
-    // Heuristic: if the path contains "%25XX" patterns, it was likely already
-    // escaped once ("%" -> "%25") to survive static-server decoding.
-    //
-    // This is intentionally conservative; double-escaping breaks lookups.
-    return /%25[0-9a-fA-F]{2}/.test(value);
-};
-const escapePercentForStaticServersOnce = (value) => {
-    if (!value) {
-        return value;
-    }
-    if (looksAlreadyEscapedForStaticServers(value)) {
-        return value;
-    }
-    return value.split("%").join("%25");
-};
-const isLikelyHostname = (value) => {
-    // Keep this loose; we only use it as a guard for embedded-URL detection.
-    if (!value)
-        return false;
-    if (value === "localhost")
-        return true;
-    return value.includes(".");
-};
-const encodeEmbeddedUrlTailIfPresent = (pathname) => {
-    // Some CDNs embed a full absolute URL into a single path segment using
-    // encodeURIComponent (e.g. ".../https%3A%2F%2Fexample.com%2Fa.png").
-    //
-    // Other runtimes may request the *decoded* form in-path (e.g.
-    // ".../https://example.com/a.png"), which changes path segments.
-    //
-    // To be resilient, detect an embedded absolute URL tail (http(s)://...) and
-    // produce an alternate pathname with that tail collapsed into one encoded
-    // segment.
-    const raw = String(pathname || "");
-    if (!raw.includes("/http")) {
-        return null;
-    }
-    const parts = raw.split("/");
-    for (let i = 0; i < parts.length; i += 1) {
-        const scheme = parts[i];
-        if (scheme !== "http:" && scheme !== "https:") {
-            continue;
-        }
-        // A real absolute URL in-path is typically split like:
-        //   ["...", "https:", "", "example.com", "a", "b.png"]
-        const hasDoubleSlash = parts[i + 1] === "";
-        const host = parts[i + 2] || "";
-        if (!hasDoubleSlash || !isLikelyHostname(host)) {
-            continue;
-        }
-        const embedded = scheme + "//" + parts.slice(i + 2).join("/");
-        const encoded = encodeURIComponent(embedded);
-        const nextParts = parts.slice(0, i).concat(encoded);
-        const rebuilt = nextParts.join("/") || "/";
-        return rebuilt.startsWith("/") ? rebuilt : "/" + rebuilt;
-    }
-    return null;
-};
-const makePathnameVariants = (pathname) => {
-    const variants = new Set();
-    const push = (value) => {
-        if (!value)
-            return;
-        variants.add(value);
-    };
-    push(pathname);
-    push(stripTrailingSlash(pathname));
-    const encodedTail = encodeEmbeddedUrlTailIfPresent(pathname);
-    if (encodedTail && encodedTail !== pathname) {
-        push(encodedTail);
-        push(stripTrailingSlash(encodedTail));
-    }
-    return Array.from(variants);
-};
-const getBasename = (pathname) => {
-    const clean = pathname.split("?")[0] || "";
-    const parts = clean.split("/").filter(Boolean);
-    return parts[parts.length - 1] || "";
+import { stripHash } from "@pagepocket/shared";
+import { escapePercentForStaticServersOnce } from "./resource-proxy/escape-percent.js";
+import { addMulti, toArray } from "./resource-proxy/multimap.js";
+import { getBasename, makePathnameVariants } from "./resource-proxy/pathname-variants.js";
+import { urlEquivalent } from "./utils.js";
+const canonicalizeHttpUrlForIndex = (url) => {
+    // Keep the host and full path/search stable, but ignore http/https scheme.
+    // This is used only as a secondary exact-match key.
+    if (url.protocol === "http:" || url.protocol === "https:") {
+        return `//${url.host}${url.pathname}${url.search}`;
+    }
+    return url.toString();
 };
 const toUrlOrNull = (value) => {
     try {
@@ -117,8 +19,9 @@ const toUrlOrNull = (value) => {
         return null;
     }
 };
-const buildResourceProxyIndex = (snapshot) => {
+export const buildResourceProxyIndex = (snapshot) => {
     const byExactUrl = new Map();
+    const byCanonicalUrl = new Map();
     const byPathnameWithSearch = new Map();
     const byPathname = new Map();
     const byBasename = new Map();
@@ -138,11 +41,18 @@ const buildResourceProxyIndex = (snapshot) => {
             parsed,
             pathname,
             pathnameWithSearch,
-            basename
+            basename,
+            canonicalUrl: canonicalizeHttpUrlForIndex(parsed)
         };
         // Prefer first-seen item for exact URL.
-        if (!byExactUrl.has(parsed.toString())) {
-            byExactUrl.set(parsed.toString(), indexed);
+        const exactKey = parsed.toString();
+        if (!byExactUrl.has(exactKey)) {
+            byExactUrl.set(exactKey, indexed);
+        }
+        // Secondary exact key: ignore http/https protocol differences.
+        const canonicalKey = indexed.canonicalUrl;
+        if (!byCanonicalUrl.has(canonicalKey)) {
+            byCanonicalUrl.set(canonicalKey, indexed);
         }
         addMulti(byPathnameWithSearch, pathnameWithSearch, indexed);
         addMulti(byPathname, pathname, indexed);
@@ -152,12 +62,12 @@ const buildResourceProxyIndex = (snapshot) => {
     }
     return {
         byExactUrl,
+        byCanonicalUrl,
         byPathnameWithSearch,
         byPathname,
         byBasename
     };
 };
-exports.buildResourceProxyIndex = buildResourceProxyIndex;
 const uniqByPath = (items) => {
     const seen = new Set();
     const out = [];
@@ -216,7 +126,7 @@ const makeSuffixes = (pathname) => {
     }
     return out;
 };
-const resolveToLocalPath = (options) => {
+export const resolveToLocalPath = (options) => {
     const { requestUrl, baseUrl, index } = options;
     if (!requestUrl) {
         return undefined;
@@ -236,12 +146,31 @@ const resolveToLocalPath = (options) => {
     if (exact) {
         return escapePercentForStaticServersOnce(exact.path);
     }
+    // Scheme-insensitive exact match for http/https.
+    // This is a safe optimization and also fixes snapshots where the runtime URL
+    // differs only by protocol from the recorded one.
+    const canonicalAbs = canonicalizeHttpUrlForIndex(abs);
+    const canonicalExact = index.byCanonicalUrl.get(canonicalAbs);
+    if (canonicalExact) {
+        return escapePercentForStaticServersOnce(canonicalExact.path);
+    }
     const withoutHash = stripHash(absString);
     if (withoutHash !== absString) {
         const found = index.byExactUrl.get(withoutHash);
         if (found) {
             return escapePercentForStaticServersOnce(found.path);
         }
+        try {
+            const withoutHashUrl = new URL(withoutHash);
+            const canonicalWithoutHash = canonicalizeHttpUrlForIndex(withoutHashUrl);
+            const canonicalFound = index.byCanonicalUrl.get(canonicalWithoutHash);
+            if (canonicalFound) {
+                return escapePercentForStaticServersOnce(canonicalFound.path);
+            }
+        }
+        catch {
+            // ignore
+        }
     }
     const pathname = abs.pathname || "/";
     const pathnameVariants = makePathnameVariants(pathname);
@@ -258,6 +187,9 @@ const resolveToLocalPath = (options) => {
         const items = toArray(index.byPathname.get(key));
         const match = tryCandidates(items, baseUrl, 99);
         if (match) {
+            if (!urlEquivalent(requestUrl, match.url, { baseUrl })) {
+                continue;
+            }
             return escapePercentForStaticServersOnce(match.path);
         }
     }
@@ -281,4 +213,3 @@ const resolveToLocalPath = (options) => {
     }
     return undefined;
 };
-exports.resolveToLocalPath = resolveToLocalPath;

package/dist/resources.js

@@ -1,41 +1,5 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractResourceUrls = exports.toAbsoluteUrl = void 0;
-const cheerio = __importStar(require("cheerio"));
-const toAbsoluteUrl = (baseUrl, resourceUrl) => {
+import * as cheerio from "cheerio";
+export const toAbsoluteUrl = (baseUrl, resourceUrl) => {
     try {
         return new URL(resourceUrl, baseUrl).toString();
     }
@@ -43,8 +7,7 @@ const toAbsoluteUrl = (baseUrl, resourceUrl) => {
         return resourceUrl;
     }
 };
-exports.toAbsoluteUrl = toAbsoluteUrl;
-const extractResourceUrls = (html, baseUrl) => {
+export const extractResourceUrls = (html, baseUrl) => {
     const $ = cheerio.load(html);
     const urls = [];
     const collect = (selector, attr) => {
@@ -74,9 +37,8 @@ const extractResourceUrls = (html, baseUrl) => {
         return {
             attr,
             element,
-            url: (0, exports.toAbsoluteUrl)(baseUrl, value)
+            url: toAbsoluteUrl(baseUrl, value)
         };
     });
     return { $, resourceUrls, srcsetItems };
 };
-exports.extractResourceUrls = extractResourceUrls;

package/dist/rewrite-links/js-imports.d.ts

@@ -0,0 +1,3 @@
+type UrlResolver = (absoluteUrl: string) => string | null;
+export declare const rewriteJsText: (source: string, resolve: UrlResolver, baseUrl: string) => Promise<string>;
+export {};

package/dist/rewrite-links/js-imports.js

@@ -0,0 +1,56 @@
+import { shouldSkipValue } from "./skip.js";
+import { resolveUrlValue } from "./url-resolve.js";
+export const rewriteJsText = async (source, resolve, baseUrl) => {
+    const replaceSpecifier = async (specifier) => {
+        const trimmed = specifier.trim();
+        if (shouldSkipValue(trimmed)) {
+            return specifier;
+        }
+        const resolved = resolveUrlValue(trimmed, baseUrl, resolve);
+        return resolved ?? specifier;
+    };
+    const importFromPattern = /(\bimport\s+[^'"]*?\sfrom\s+)(["'])([^"']+)\2/g;
+    const importSideEffectPattern = /(\bimport\s+)(["'])([^"']+)\2/g;
+    const dynamicImportPattern = /(\bimport\s*\(\s*)(["'])([^"']+)\2(\s*\))/g;
+    let replaced = "";
+    let lastIndex = 0;
+    for (const match of source.matchAll(importFromPattern)) {
+        const index = match.index ?? 0;
+        replaced += source.slice(lastIndex, index);
+        const prefix = match[1] || "";
+        const quote = match[2] || "";
+        const specifier = match[3] || "";
+        const next = await replaceSpecifier(specifier);
+        replaced += `${prefix}${quote}${next}${quote}`;
+        lastIndex = index + match[0].length;
+    }
+    replaced += source.slice(lastIndex);
+    let final = "";
+    lastIndex = 0;
+    for (const match of replaced.matchAll(importSideEffectPattern)) {
+        const index = match.index ?? 0;
+        final += replaced.slice(lastIndex, index);
+        const prefix = match[1] || "";
+        const quote = match[2] || "";
+        const specifier = match[3] || "";
+        const next = await replaceSpecifier(specifier);
+        final += `${prefix}${quote}${next}${quote}`;
+        lastIndex = index + match[0].length;
+    }
+    final += replaced.slice(lastIndex);
+    let dynamicFinal = "";
+    lastIndex = 0;
+    for (const match of final.matchAll(dynamicImportPattern)) {
+        const index = match.index ?? 0;
+        dynamicFinal += final.slice(lastIndex, index);
+        const prefix = match[1] || "";
+        const quote = match[2] || "";
+        const specifier = match[3] || "";
+        const suffix = match[4] || "";
+        const next = await replaceSpecifier(specifier);
+        dynamicFinal += `${prefix}${quote}${next}${quote}${suffix}`;
+        lastIndex = index + match[0].length;
+    }
+    dynamicFinal += final.slice(lastIndex);
+    return dynamicFinal;
+};

package/dist/rewrite-links/link-rel.d.ts

@@ -0,0 +1,2 @@
+import * as cheerio from "cheerio";
+export declare const shouldRewriteLinkHref: ($element: cheerio.Cheerio<any>) => boolean;

package/dist/rewrite-links/link-rel.js

@@ -0,0 +1,10 @@
+export const shouldRewriteLinkHref = ($element) => {
+    const rel = ($element.attr("rel") || "").trim().toLowerCase();
+    if (!rel) {
+        return true;
+    }
+    return (rel.includes("stylesheet") ||
+        rel.includes("preload") ||
+        rel.includes("prefetch") ||
+        rel.includes("icon"));
+};

package/dist/rewrite-links/meta-refresh.d.ts

@@ -0,0 +1,3 @@
+type UrlResolver = (absoluteUrl: string) => string | null;
+export declare const rewriteMetaRefresh: (content: string, baseUrl: string, resolve: UrlResolver) => string;
+export {};

package/dist/rewrite-links/meta-refresh.js

@@ -0,0 +1,22 @@
+import { resolveUrlValue } from "./url-resolve.js";
+export const rewriteMetaRefresh = (content, baseUrl, resolve) => {
+    const parts = content.split(";");
+    if (parts.length < 2)
+        return content;
+    const urlPartIndex = parts.findIndex((part) => part.trim().toLowerCase().startsWith("url="));
+    if (urlPartIndex === -1)
+        return content;
+    const urlPart = parts[urlPartIndex];
+    let rawUrl = urlPart.split("=").slice(1).join("=").trim();
+    if ((rawUrl.startsWith('"') && rawUrl.endsWith('"')) ||
+        (rawUrl.startsWith("'") && rawUrl.endsWith("'"))) {
+        rawUrl = rawUrl.slice(1, -1).trim();
+    }
+    const resolved = resolveUrlValue(rawUrl, baseUrl, resolve);
+    if (!resolved)
+        return content;
+    const next = `url=${resolved}`;
+    const nextParts = parts.slice();
+    nextParts[urlPartIndex] = next;
+    return nextParts.join(";");
+};

package/dist/rewrite-links/skip.d.ts

@@ -0,0 +1 @@
+export declare const shouldSkipValue: (value: string) => boolean;

package/dist/rewrite-links/skip.js

@@ -0,0 +1,10 @@
+export const shouldSkipValue = (value) => {
+    const trimmed = value.trim();
+    return (!trimmed ||
+        trimmed.startsWith("data:") ||
+        trimmed.startsWith("blob:") ||
+        trimmed.startsWith("mailto:") ||
+        trimmed.startsWith("tel:") ||
+        trimmed.startsWith("javascript:") ||
+        trimmed.startsWith("#"));
+};

package/dist/rewrite-links/srcset.d.ts

@@ -0,0 +1,3 @@
+type UrlResolver = (absoluteUrl: string) => string | null;
+export declare const rewriteSrcsetValue: (value: string, baseUrl: string, resolve: UrlResolver) => string;
+export {};

package/dist/rewrite-links/srcset.js

@@ -0,0 +1,63 @@
+import { resolveUrlValue } from "./url-resolve.js";
+const isUnsafeSrcsetValue = (value) => {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return false;
+    }
+    const hasFetchTransform = trimmed.includes("/image/fetch/");
+    const hasEncodedUrlTail = trimmed.includes("https%3A%2F%2F");
+    const hasCommaTokens = trimmed.includes(",w_") ||
+        trimmed.includes(", w_") ||
+        trimmed.includes(",h_") ||
+        trimmed.includes(", h_") ||
+        trimmed.includes(",c_") ||
+        trimmed.includes(", c_");
+    return hasFetchTransform && hasEncodedUrlTail && hasCommaTokens;
+};
+const isDescriptorToken = (token) => {
+    const trimmed = token.trim();
+    if (!trimmed)
+        return false;
+    return /^\d+(\.\d+)?x$/i.test(trimmed) || /^\d+w$/i.test(trimmed);
+};
+const parseSrcset = (input) => {
+    const rawCandidates = input
+        .split(",")
+        .map((c) => c.trim())
+        .filter(Boolean);
+    return rawCandidates.map((candidate) => {
+        const tokens = candidate.split(/\s+/).filter(Boolean);
+        if (tokens.length === 0) {
+            return { url: candidate };
+        }
+        const last = tokens[tokens.length - 1] ?? "";
+        if (tokens.length >= 2 && isDescriptorToken(last)) {
+            const descriptor = last;
+            const url = candidate.slice(0, candidate.lastIndexOf(descriptor)).trim();
+            return { url, descriptor };
+        }
+        return { url: candidate };
+    });
+};
+const stringifySrcset = (candidates) => {
+    return candidates
+        .map((c) => {
+        const url = c.url.trim();
+        if (!c.descriptor)
+            return url;
+        return `${url} ${c.descriptor.trim()}`;
+    })
+        .filter(Boolean)
+        .join(",");
+};
+export const rewriteSrcsetValue = (value, baseUrl, resolve) => {
+    if (isUnsafeSrcsetValue(value)) {
+        return "";
+    }
+    const candidates = parseSrcset(value);
+    const rewritten = candidates.map((c) => {
+        const resolved = resolveUrlValue(c.url, baseUrl, resolve);
+        return { url: resolved ?? c.url, descriptor: c.descriptor };
+    });
+    return stringifySrcset(rewritten);
+};

package/dist/rewrite-links/url-resolve.d.ts

@@ -0,0 +1,3 @@
+type UrlResolver = (absoluteUrl: string) => string | null;
+export declare const resolveUrlValue: (value: string, baseUrl: string, resolve: UrlResolver) => string | null;
+export {};

package/dist/rewrite-links/url-resolve.js

@@ -0,0 +1,13 @@
+import { shouldSkipValue } from "./skip.js";
+export const resolveUrlValue = (value, baseUrl, resolve) => {
+    if (shouldSkipValue(value)) {
+        return null;
+    }
+    try {
+        const absolute = new URL(value, baseUrl).toString();
+        return resolve(absolute);
+    }
+    catch {
+        return null;
+    }
+};

package/dist/rewrite-links.d.ts

@@ -1,6 +1,7 @@
-import type { ReplaceElementsConfig } from "./types";
+import { rewriteJsText } from "./rewrite-links/js-imports.js";
+import type { ReplaceElementsConfig } from "./types.js";
 type UrlResolver = (absoluteUrl: string) => string | null;
-export declare const rewriteJsText: (source: string, resolve: UrlResolver, baseUrl: string) => Promise<string>;
+export { rewriteJsText };
 export declare const rewriteEntryHtml: (input: {
     html: string;
     entryUrl: string;
@@ -18,4 +19,3 @@ export declare const rewriteEntryHtml: (input: {
     html: string;
     title?: string;
 }>;
-export {};