@pagepocket/lib 0.6.2 → 0.7.0

package/dist/resource-proxy.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveToLocalPath = exports.buildResourceProxyIndex = void 0;
+const addMulti = (map, key, value) => {
+    const existing = map.get(key);
+    if (!existing) {
+        map.set(key, value);
+        return;
+    }
+    if (Array.isArray(existing)) {
+        existing.push(value);
+        return;
+    }
+    map.set(key, [existing, value]);
+};
+const toArray = (value) => {
+    if (!value) {
+        return [];
+    }
+    return Array.isArray(value) ? value : [value];
+};
+const stripHash = (value) => {
+    const index = value.indexOf("#");
+    return index === -1 ? value : value.slice(0, index);
+};
+const stripTrailingSlash = (value) => {
+    if (!value || value === "/") {
+        return value;
+    }
+    return value.endsWith("/") ? value.slice(0, -1) : value;
+};
+const looksAlreadyEscapedForStaticServers = (value) => {
+    // Heuristic: if the path contains "%25XX" patterns, it was likely already
+    // escaped once ("%" -> "%25") to survive static-server decoding.
+    //
+    // This is intentionally conservative; double-escaping breaks lookups.
+    return /%25[0-9a-fA-F]{2}/.test(value);
+};
+const escapePercentForStaticServersOnce = (value) => {
+    if (!value) {
+        return value;
+    }
+    if (looksAlreadyEscapedForStaticServers(value)) {
+        return value;
+    }
+    return value.split("%").join("%25");
+};
+const isLikelyHostname = (value) => {
+    // Keep this loose; we only use it as a guard for embedded-URL detection.
+    if (!value)
+        return false;
+    if (value === "localhost")
+        return true;
+    return value.includes(".");
+};
+const encodeEmbeddedUrlTailIfPresent = (pathname) => {
+    // Some CDNs embed a full absolute URL into a single path segment using
+    // encodeURIComponent (e.g. ".../https%3A%2F%2Fexample.com%2Fa.png").
+    //
+    // Other runtimes may request the *decoded* form in-path (e.g.
+    // ".../https://example.com/a.png"), which changes path segments.
+    //
+    // To be resilient, detect an embedded absolute URL tail (http(s)://...) and
+    // produce an alternate pathname with that tail collapsed into one encoded
+    // segment.
+    const raw = String(pathname || "");
+    if (!raw.includes("/http")) {
+        return null;
+    }
+    const parts = raw.split("/");
+    for (let i = 0; i < parts.length; i += 1) {
+        const scheme = parts[i];
+        if (scheme !== "http:" && scheme !== "https:") {
+            continue;
+        }
+        // A real absolute URL in-path is typically split like:
+        //   ["...", "https:", "", "example.com", "a", "b.png"]
+        const hasDoubleSlash = parts[i + 1] === "";
+        const host = parts[i + 2] || "";
+        if (!hasDoubleSlash || !isLikelyHostname(host)) {
+            continue;
+        }
+        const embedded = scheme + "//" + parts.slice(i + 2).join("/");
+        const encoded = encodeURIComponent(embedded);
+        const nextParts = parts.slice(0, i).concat(encoded);
+        const rebuilt = nextParts.join("/") || "/";
+        return rebuilt.startsWith("/") ? rebuilt : "/" + rebuilt;
+    }
+    return null;
+};
+const makePathnameVariants = (pathname) => {
+    const variants = new Set();
+    const push = (value) => {
+        if (!value)
+            return;
+        variants.add(value);
+    };
+    push(pathname);
+    push(stripTrailingSlash(pathname));
+    const encodedTail = encodeEmbeddedUrlTailIfPresent(pathname);
+    if (encodedTail && encodedTail !== pathname) {
+        push(encodedTail);
+        push(stripTrailingSlash(encodedTail));
+    }
+    return Array.from(variants);
+};
+const getBasename = (pathname) => {
+    const clean = pathname.split("?")[0] || "";
+    const parts = clean.split("/").filter(Boolean);
+    return parts[parts.length - 1] || "";
+};
+const toUrlOrNull = (value) => {
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+};
+const buildResourceProxyIndex = (snapshot) => {
+    const byExactUrl = new Map();
+    const byPathnameWithSearch = new Map();
+    const byPathname = new Map();
+    const byBasename = new Map();
+    for (const item of snapshot.items || []) {
+        if (!item || !item.url || !item.path) {
+            continue;
+        }
+        const parsed = toUrlOrNull(item.url);
+        if (!parsed) {
+            continue;
+        }
+        const pathname = parsed.pathname || "/";
+        const pathnameWithSearch = pathname + (parsed.search || "");
+        const basename = getBasename(pathname);
+        const indexed = {
+            ...item,
+            parsed,
+            pathname,
+            pathnameWithSearch,
+            basename
+        };
+        // Prefer first-seen item for exact URL.
+        if (!byExactUrl.has(parsed.toString())) {
+            byExactUrl.set(parsed.toString(), indexed);
+        }
+        addMulti(byPathnameWithSearch, pathnameWithSearch, indexed);
+        addMulti(byPathname, pathname, indexed);
+        if (basename) {
+            addMulti(byBasename, basename, indexed);
+        }
+    }
+    return {
+        byExactUrl,
+        byPathnameWithSearch,
+        byPathname,
+        byBasename
+    };
+};
+exports.buildResourceProxyIndex = buildResourceProxyIndex;
+const uniqByPath = (items) => {
+    const seen = new Set();
+    const out = [];
+    for (const item of items) {
+        if (seen.has(item.path))
+            continue;
+        seen.add(item.path);
+        out.push(item);
+    }
+    return out;
+};
+const preferSingle = (items, baseUrl, suffixLength) => {
+    if (items.length <= 1) {
+        return items[0] ?? null;
+    }
+    const baseParsed = (() => {
+        try {
+            return new URL(baseUrl);
+        }
+        catch {
+            return null;
+        }
+    })();
+    if (baseParsed) {
+        const sameOrigin = items.filter((i) => i.parsed.origin === baseParsed.origin);
+        if (sameOrigin.length === 1) {
+            return sameOrigin[0];
+        }
+        if (sameOrigin.length > 1) {
+            items = sameOrigin;
+        }
+    }
+    // If still ambiguous, only accept when the match key is strong.
+    // We treat very short suffix matches (or basename-only) as too risky.
+    if (suffixLength < 2) {
+        return null;
+    }
+    return null;
+};
+const tryCandidates = (items, baseUrl, suffixLength) => {
+    const unique = uniqByPath(items);
+    if (unique.length === 0) {
+        return null;
+    }
+    if (unique.length === 1) {
+        return unique[0];
+    }
+    return preferSingle(unique, baseUrl, suffixLength);
+};
+const makeSuffixes = (pathname) => {
+    const parts = pathname.split("/").filter(Boolean);
+    const out = [];
+    for (let i = 0; i < parts.length; i += 1) {
+        const suffix = "/" + parts.slice(i).join("/");
+        out.push({ key: suffix, depth: parts.length - i });
+    }
+    return out;
+};
+const resolveToLocalPath = (options) => {
+    const { requestUrl, baseUrl, index } = options;
+    if (!requestUrl) {
+        return undefined;
+    }
+    let abs = null;
+    try {
+        abs = new URL(requestUrl, baseUrl);
+    }
+    catch {
+        abs = null;
+    }
+    if (!abs) {
+        return undefined;
+    }
+    const absString = abs.toString();
+    const exact = index.byExactUrl.get(absString);
+    if (exact) {
+        return escapePercentForStaticServersOnce(exact.path);
+    }
+    const withoutHash = stripHash(absString);
+    if (withoutHash !== absString) {
+        const found = index.byExactUrl.get(withoutHash);
+        if (found) {
+            return escapePercentForStaticServersOnce(found.path);
+        }
+    }
+    const pathname = abs.pathname || "/";
+    const pathnameVariants = makePathnameVariants(pathname);
+    const search = abs.search || "";
+    const pathnameWithSearchVariants = pathnameVariants.map((p) => p + search);
+    for (const key of pathnameWithSearchVariants) {
+        const items = toArray(index.byPathnameWithSearch.get(key));
+        const match = tryCandidates(items, baseUrl, 99);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    for (const key of pathnameVariants) {
+        const items = toArray(index.byPathname.get(key));
+        const match = tryCandidates(items, baseUrl, 99);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    // Suffix fallback: progressively remove leading segments.
+    for (const variant of pathnameVariants) {
+        for (const suffix of makeSuffixes(variant)) {
+            const items = toArray(index.byPathname.get(suffix.key));
+            const match = tryCandidates(items, baseUrl, suffix.depth);
+            if (match) {
+                return escapePercentForStaticServersOnce(match.path);
+            }
+        }
+    }
+    const basename = getBasename(pathname);
+    if (basename) {
+        const items = toArray(index.byBasename.get(basename));
+        const match = tryCandidates(items, baseUrl, 1);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    return undefined;
+};
+exports.resolveToLocalPath = resolveToLocalPath;

package/dist/rewrite-links.d.ts

@@ -1,3 +1,4 @@
+import type { ReplaceElementsConfig } from "./types";
 type UrlResolver = (absoluteUrl: string) => string | null;
 export declare const rewriteJsText: (source: string, resolve: UrlResolver, baseUrl: string) => Promise<string>;
 export declare const rewriteEntryHtml: (input: {
@@ -6,6 +7,13 @@ export declare const rewriteEntryHtml: (input: {
     apiPath: string;
     resolve: UrlResolver;
     rewriteLinks?: boolean;
+    replaceElements?: ReplaceElementsConfig;
+    isEntryDocument?: boolean;
+    /**
+     * The top-level entry URL for this snapshot (may differ from entryUrl when
+     * multiple documents/frames are captured).
+     */
+    snapshotEntryUrl?: string;
 }) => Promise<{
     html: string;
     title?: string;

package/dist/rewrite-links.js

@@ -37,6 +37,7 @@ exports.rewriteEntryHtml = exports.rewriteJsText = void 0;
 const cheerio = __importStar(require("cheerio"));
 const css_rewrite_1 = require("./css-rewrite");
 const hack_html_1 = require("./hack-html");
+const replace_elements_1 = require("./replace-elements");
 const shouldSkipValue = (value) => {
     const trimmed = value.trim();
     return (!trimmed ||
@@ -59,18 +60,83 @@ const resolveUrlValue = (value, baseUrl, resolve) => {
         return null;
     }
 };
+const isUnsafeSrcsetValue = (value) => {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return false;
+    }
+    // Some sites (notably Substack) emit image transform URLs that contain commas
+    // inside the URL itself (e.g. "/image/fetch/...,$w_40,$h_40,.../https%3A...").
+    //
+    // In the HTML srcset grammar, commas separate candidates, so unescaped commas
+    // inside a URL make the srcset invalid. Browsers will parse it into garbage
+    // URLs like "https%3A%2F%2F...png" and try to fetch them.
+    //
+    // For offline snapshots, it's better to drop srcset entirely and rely on
+    // the already-rewritten img[src].
+    const hasFetchTransform = trimmed.includes("/image/fetch/");
+    const hasEncodedUrlTail = trimmed.includes("https%3A%2F%2F");
+    const hasCommaTokens = trimmed.includes(",w_") ||
+        trimmed.includes(", w_") ||
+        trimmed.includes(",h_") ||
+        trimmed.includes(", h_") ||
+        trimmed.includes(",c_") ||
+        trimmed.includes(", c_");
+    return hasFetchTransform && hasEncodedUrlTail && hasCommaTokens;
+};
+const isDescriptorToken = (token) => {
+    const trimmed = token.trim();
+    if (!trimmed)
+        return false;
+    // Common srcset descriptors: 1x, 2x, 320w
+    return /^\d+(\.\d+)?x$/i.test(trimmed) || /^\d+w$/i.test(trimmed);
+};
+const parseSrcset = (input) => {
+    // Minimal srcset parser:
+    // - Candidates are separated by commas.
+    // - Each candidate is "<url> [descriptor]".
+    // - URLs may contain spaces/commas (e.g. CDN transform strings). To avoid
+    //   breaking those, we locate the descriptor from the *end* of the candidate.
+    const rawCandidates = input
+        .split(",")
+        .map((c) => c.trim())
+        .filter(Boolean);
+    return rawCandidates.map((candidate) => {
+        const tokens = candidate.split(/\s+/).filter(Boolean);
+        if (tokens.length === 0) {
+            return { url: candidate };
+        }
+        const last = tokens[tokens.length - 1] ?? "";
+        if (tokens.length >= 2 && isDescriptorToken(last)) {
+            const descriptor = last;
+            const url = candidate.slice(0, candidate.lastIndexOf(descriptor)).trim();
+            return { url, descriptor };
+        }
+        return { url: candidate };
+    });
+};
+const stringifySrcset = (candidates) => {
+    return (candidates
+        .map((c) => {
+        const url = c.url.trim();
+        if (!c.descriptor)
+            return url;
+        return `${url} ${c.descriptor.trim()}`;
+    })
+        .filter(Boolean)
+        // Don't introduce spaces after commas inside URL tokens.
+        .join(","));
+};
 const rewriteSrcsetValue = (value, baseUrl, resolve) => {
-    const parts = value.split(",").map((part) => part.trim());
-    const rewritten = parts.map((part) => {
-        const [rawUrl, descriptor] = part.split(/\s+/, 2);
-        if (!rawUrl)
-            return part;
-        const resolved = resolveUrlValue(rawUrl, baseUrl, resolve);
-        if (!resolved)
-            return part;
-        return descriptor ? `${resolved} ${descriptor}` : resolved;
+    if (isUnsafeSrcsetValue(value)) {
+        return "";
+    }
+    const candidates = parseSrcset(value);
+    const rewritten = candidates.map((c) => {
+        const resolved = resolveUrlValue(c.url, baseUrl, resolve);
+        return { url: resolved ?? c.url, descriptor: c.descriptor };
     });
-    return rewritten.join(", ");
+    return stringifySrcset(rewritten);
 };
 const rewriteMetaRefresh = (content, baseUrl, resolve) => {
     const parts = content.split(";");
@@ -80,7 +146,13 @@ const rewriteMetaRefresh = (content, baseUrl, resolve) => {
     if (urlPartIndex === -1)
         return content;
     const urlPart = parts[urlPartIndex];
-    const rawUrl = urlPart.split("=").slice(1).join("=").trim();
+    let rawUrl = urlPart.split("=").slice(1).join("=").trim();
+    // Some pages quote the URL value (url="/next" or url='/next').
+    // Strip a single pair of surrounding quotes to improve rewrite coverage.
+    if ((rawUrl.startsWith('"') && rawUrl.endsWith('"')) ||
+        (rawUrl.startsWith("'") && rawUrl.endsWith("'"))) {
+        rawUrl = rawUrl.slice(1, -1).trim();
+    }
     const resolved = resolveUrlValue(rawUrl, baseUrl, resolve);
     if (!resolved)
         return content;
@@ -89,6 +161,18 @@ const rewriteMetaRefresh = (content, baseUrl, resolve) => {
     nextParts[urlPartIndex] = next;
     return nextParts.join(";");
 };
+const shouldRewriteLinkHref = ($element) => {
+    const rel = ($element.attr("rel") || "").trim().toLowerCase();
+    if (!rel) {
+        return true;
+    }
+    // Only rewrite link rels that are expected to load a resource.
+    // Avoid rewriting navigational/SEO links like canonical, preconnect, etc.
+    return (rel.includes("stylesheet") ||
+        rel.includes("preload") ||
+        rel.includes("prefetch") ||
+        rel.includes("icon"));
+};
 const rewriteJsText = async (source, resolve, baseUrl) => {
     const replaceSpecifier = async (specifier) => {
         const trimmed = specifier.trim();
@@ -171,7 +255,19 @@ const rewriteEntryHtml = async (input) => {
         rewriteAttr("iframe[src]", "src");
         rewriteAttr("embed[src]", "src");
         rewriteAttr("object[data]", "data");
-        rewriteAttr("link[href]", "href");
+        $("link[href]").each((_, element) => {
+            const el = $(element);
+            if (!shouldRewriteLinkHref(el)) {
+                return;
+            }
+            const value = el.attr("href");
+            if (!value)
+                return;
+            const resolved = resolveUrlValue(value, baseUrl, resolve);
+            if (resolved) {
+                el.attr("href", resolved);
+            }
+        });
         rewriteAttr("[poster]", "poster");
         rewriteDataAttrs("[data-src]", "data-src");
         rewriteDataAttrs("[data-href]", "data-href");
@@ -184,6 +280,13 @@ const rewriteEntryHtml = async (input) => {
             const rewritten = rewriteSrcsetValue(value, baseUrl, resolve);
             $(element).attr("srcset", rewritten);
         });
+        $("link[imagesrcset]").each((_, element) => {
+            const value = $(element).attr("imagesrcset");
+            if (!value)
+                return;
+            const rewritten = rewriteSrcsetValue(value, baseUrl, resolve);
+            $(element).attr("imagesrcset", rewritten);
+        });
         $("meta[http-equiv]").each((_, element) => {
             const httpEquiv = ($(element).attr("http-equiv") || "").toLowerCase();
             if (httpEquiv !== "refresh")
@@ -245,6 +348,13 @@ const rewriteEntryHtml = async (input) => {
         baseUrl: baseUrl,
         apiPath: input.apiPath
     });
+    await (0, replace_elements_1.applyReplaceElements)({
+        $,
+        entryUrl: input.snapshotEntryUrl ?? baseUrl,
+        url: baseUrl,
+        replaceElements: input.replaceElements,
+        isEntryDocument: input.isEntryDocument ?? true
+    });
     const title = $("title").first().text() || undefined;
     return { html: $.html(), title };
 };

package/dist/snapshot-builder.d.ts

@@ -1,5 +1,5 @@
 import type { ApiEntry, StoredResource } from "./network-store";
-import type { ContentStore, PageSnapshot, PathResolver } from "./types";
+import type { ContentStore, PageSnapshot, PathResolver, ReplaceElementsConfig } from "./types";
 type BuildOptions = {
     entryUrl: string;
     createdAt: number;
@@ -9,6 +9,7 @@ type BuildOptions = {
     pathResolver?: PathResolver;
     rewriteEntry: boolean;
     rewriteCSS: boolean;
+    replaceElements?: ReplaceElementsConfig;
     warnings: string[];
 };
 export declare const buildSnapshot: (input: BuildOptions) => Promise<PageSnapshot>;

package/dist/snapshot-builder.js

@@ -6,6 +6,18 @@ const path_resolver_1 = require("./path-resolver");
 const rewrite_links_1 = require("./rewrite-links");
 const snapshot_1 = require("./snapshot");
 const utils_1 = require("./utils");
+const escapePercentForStaticServers = (value) => {
+    // Many static servers decode percent-encoding in the request path before
+    // resolving it to a filesystem path.
+    //
+    // Our snapshots can contain literal "%2F" sequences in filenames (e.g.
+    // Substack image URLs embedded into a path segment). When a server decodes
+    // "%2F" to "/", it changes the path structure and causes 404s.
+    //
+    // Escaping "%" to "%25" makes the request decode back to the original
+    // filename on disk.
+    return value.split("%").join("%25");
+};
 const streamToUint8Array = async (stream) => {
     const reader = stream.getReader();
     const chunks = [];
@@ -119,6 +131,42 @@ const buildApiSnapshot = (url, createdAt, entries) => ({
     createdAt,
     records: entries.map((entry) => entry.record)
 });
+const buildResourcesPathSnapshot = (createdAt, files) => {
+    const items = [];
+    for (const file of files) {
+        const originalUrl = file.originalUrl;
+        if (!originalUrl) {
+            continue;
+        }
+        // Exclude HTML documents. We only want static resources.
+        if (file.resourceType === "document") {
+            continue;
+        }
+        // Skip snapshot-local pseudo URLs.
+        if (originalUrl.startsWith("/")) {
+            continue;
+        }
+        // Be defensive: only include valid absolute URLs.
+        try {
+            new URL(originalUrl);
+        }
+        catch {
+            continue;
+        }
+        items.push({
+            url: originalUrl,
+            path: file.path,
+            resourceType: file.resourceType,
+            mimeType: file.mimeType,
+            size: file.size
+        });
+    }
+    return {
+        version: "1.0",
+        createdAt,
+        items
+    };
+};
 const buildSnapshot = async (input) => {
     const warnings = input.warnings;
     const groups = groupResources({
@@ -147,7 +195,18 @@ const buildSnapshot = async (input) => {
             });
             urlToPath.set(resource.request.url, path);
         }
-        const resolve = (absoluteUrl) => urlToPath.get(absoluteUrl) ?? null;
+        const resolve = (absoluteUrl) => {
+            const resolved = urlToPath.get(absoluteUrl);
+            if (!resolved) {
+                return null;
+            }
+            // Only escape snapshot-local paths. (Defensive: resolved should always be
+            // a path, but avoid breaking any unexpected absolute URLs.)
+            if (resolved.includes("://")) {
+                return resolved;
+            }
+            return escapePercentForStaticServers(resolved);
+        };
         const apiPath = (0, utils_1.ensureLeadingSlash)(multiDoc ? `${(0, utils_1.sanitizePosixPath)(docDir)}/api.json` : "/api.json");
         for (const resource of group.resources) {
             if (resource.request.resourceType === "document") {
@@ -161,7 +220,10 @@ const buildSnapshot = async (input) => {
                     entryUrl: group.url,
                     apiPath,
                     resolve,
-                    rewriteLinks: input.rewriteEntry
+                    rewriteLinks: input.rewriteEntry,
+                    replaceElements: input.replaceElements,
+                    isEntryDocument: group.url === input.entryUrl,
+                    snapshotEntryUrl: input.entryUrl
                 });
                 html = rewritten.html;
                 if (!title) {
@@ -253,6 +315,17 @@ const buildSnapshot = async (input) => {
             originalUrl: apiPath
         });
     }
+    {
+        const resourcesPath = buildResourcesPathSnapshot(input.createdAt, files);
+        const bytes = new TextEncoder().encode(JSON.stringify(resourcesPath, null, 2));
+        // Snapshot-local artifact. It intentionally has no originalUrl.
+        files.push({
+            path: "/resources_path.json",
+            mimeType: "application/json",
+            size: bytes.byteLength,
+            source: { kind: "memory", data: bytes }
+        });
+    }
     const totalBytes = files.reduce((sum, file) => sum + (file.size ?? 0), 0);
     const totalFiles = files.length;
     const snapshotUrl = input.entryUrl || groups[0]?.url || "";