@pagepocket/lib 0.6.3 → 0.7.0

package/README.md

@@ -30,17 +30,19 @@ await snapshot.toDirectory("./out");
 class PagePocket {
   static fromURL(url: string, options?: PagePocketOptions): PagePocket;
   static fromTarget(target: InterceptTarget, options?: PagePocketOptions): PagePocket;
+  interceptedRequestEvents(): NetworkEventStream;
   capture(options?: CaptureOptions): Promise<PageSnapshot>;
 }
 ```
 
 ### CaptureOptions (core)
 
-```ts
+````ts
 interface CaptureOptions {
   interceptor: NetworkInterceptorAdapter;
   completion?: CompletionStrategy | CompletionStrategy[];
   filter?: ResourceFilter;
+  blacklist?: RegExp[];
   pathResolver?: PathResolver;
   contentStore?: ContentStore;
   rewriteEntry?: boolean;
@@ -51,8 +53,33 @@ interface CaptureOptions {
     maxResources?: number;
   };
 }
+
+type NetworkEventStream = AsyncIterable<NetworkEvent>;
+
+### Built-in blacklist
+
+```ts
+import { PagePocket, gaBlacklist } from "@pagepocket/lib";
+
+const snapshot = await PagePocket.fromURL("https://example.com").capture({
+  interceptor,
+  blacklist: gaBlacklist.ga
+});
+````
+
+You can combine multiple built-in lists:
+
+```ts
+import { PagePocket, builtinBlacklist } from "@pagepocket/lib";
+
+const snapshot = await PagePocket.fromURL("https://example.com").capture({
+  interceptor,
+  blacklist: [...builtinBlacklist.ga]
+});
 ```
 
+````
+
 ### PageSnapshot output
 
 ```ts
@@ -63,18 +90,30 @@ interface PageSnapshot {
   entry: string;
   files: SnapshotFile[];
   toDirectory(outDir: string, options?: WriteFSOptions): Promise<WriteResult>;
-  toZip(options?: ZipOptions): Promise<Uint8Array | Blob>;
+  toZip(options?: ZipOptions): Promise<ZipResult>;
 }
 
 interface WriteFSOptions {
   clearCache?: boolean;
+  overwrite?: boolean;
+  suffix?: string;
 }
 
 interface ZipOptions {
   asBlob?: boolean;
   clearCache?: boolean;
+  overwrite?: boolean;
+  suffix?: string;
+  outputPath?: string;
 }
-```
+
+interface ZipWriteResult {
+  data: Uint8Array | Blob;
+  outputPath: string;
+}
+
+type ZipResult = Uint8Array | Blob | ZipWriteResult;
+````
 
 Snapshot layout:
 
@@ -92,3 +131,226 @@ directory based on the document URL path (e.g. `foo/bar/index.html`).
 
 - Uses `@pagepocket/uni-fs` for file IO so it works in Node and OPFS contexts.
 - Network data comes only from the interceptor events.
+
+## HTML element replacement (replaceElements)
+
+During capture, `@pagepocket/lib` rewrites the captured HTML (the raw Document response body) before writing the final snapshot HTML.
+
+`replaceElements` lets you declaratively or programmatically replace parts of that HTML.
+
+Typical use-cases:
+
+- Replace embedded players/widgets with a placeholder (e.g. YouTube iframe → `<div>Player</div>`)
+- Rename tags at scale (e.g. all `<div>` → `<p>`)
+- Replace a specific element (e.g. `<div id="foo">` → `<span>hello</span>`)
+
+### Where this runs
+
+`replaceElements` runs during the **HTML rewrite stage (Cheerio)**, not on a live browser DOM.
+
+- Input is the captured HTML string.
+- You can match using CSS selectors.
+- Function rules receive a Cheerio element context.
+
+If you need to modify the live DOM after scripts run, that is a different capture mode and is not supported by this option.
+
+### Configuration
+
+Add `replaceElements` to `CaptureOptions`:
+
+```ts
+import type {
+  CaptureOptions,
+  ReplaceElementsConfig,
+  ReplaceAction,
+  ReplaceElementContext,
+  ReplaceElementRule
+} from "@pagepocket/lib";
+
+const replaceElements: ReplaceElementsConfig = [
+  {
+    name: "replace-youtube-embed",
+    match: 'iframe[src*="youtube.com/embed"]',
+    replace: { type: "replaceWithHtml", html: "<div>Player</div>" }
+  }
+];
+
+const options: CaptureOptions = {
+  interceptor,
+  replaceElements
+};
+```
+
+### Data structure
+
+`replaceElements` is an array of items. Each item can be:
+
+1. A **rule object** (declarative)
+2. A **function** (imperative; convenience form)
+3. A **function rule with query** (imperative; recommended for performance)
+
+```ts
+export type ReplaceElementsConfig = Array<
+  ReplaceElementRule | ReplaceElementFn | ReplaceElementFnWithQuery
+>;
+
+export interface ReplaceElementRule {
+  name?: string;
+  match: MatchQuery;
+  replace: ReplaceAction;
+  apply?: ApplyOptions;
+}
+
+export type ReplaceElementFn = (
+  ctx: ReplaceElementContext
+) => void | ReplaceAction | ReplaceAction[] | Promise<void | ReplaceAction | ReplaceAction[]>;
+
+export interface ReplaceElementFnWithQuery {
+  name?: string;
+  query: string;
+  run: ReplaceElementFn;
+  apply?: ApplyOptions;
+}
+```
+
+#### MatchQuery
+
+First-class support is CSS selectors (Cheerio selectors).
+
+```ts
+export type MatchQuery =
+  | string
+  | {
+      selector?: string;
+      tagName?: string;
+      id?: string;
+      attrs?: Record<string, string | RegExp | true>;
+    };
+```
+
+#### ReplaceAction
+
+```ts
+export type ReplaceAction =
+  | { type: "replaceWithHtml"; html: string }
+  | {
+      type: "replaceWithElement";
+      tagName: string;
+      textContent?: string;
+      html?: string;
+      attrs?: Record<string, string | null>;
+    }
+  | {
+      type: "renameTag";
+      to: string;
+      keepAttributes?: boolean;
+      keepChildren?: boolean;
+    }
+  | { type: "remove" };
+```
+
+#### ApplyOptions
+
+```ts
+export interface ApplyOptions {
+  /** default: "document" */
+  scope?: "document" | "allFrames";
+  /** default: "all" */
+  limit?: number | "all";
+  /** default: "stop" */
+  onReplaced?: "stop" | "continue";
+}
+```
+
+Notes:
+
+- `scope: "allFrames"` applies to multiple captured documents (if your interceptor captures frames / subdocuments). Cross-origin frames may not be available depending on capture.
+- `limit` limits how many matched elements are replaced for that rule.
+
+### Semantics
+
+- Rules run **in array order**.
+- Each rule matches against the **current HTML state** (earlier replacements affect later matches).
+- Default `onReplaced: "stop"` makes results predictable: once an element is replaced by a rule, later rules won't attempt to re-process that same original element.
+
+### Examples
+
+Replace YouTube iframe embeds with a placeholder:
+
+```ts
+replaceElements: [
+  {
+    match: 'iframe[src*="youtube.com/embed"]',
+    replace: { type: "replaceWithHtml", html: "<div>Player</div>" }
+  }
+];
+```
+
+Rename all `div` tags to `p`:
+
+```ts
+replaceElements: [
+  {
+    match: "div",
+    replace: { type: "renameTag", to: "p", keepAttributes: true, keepChildren: true }
+  }
+];
+```
+
+Replace `<div id="foo">` with `<span>hello</span>`:
+
+```ts
+replaceElements: [
+  {
+    match: "div#foo",
+    replace: { type: "replaceWithElement", tagName: "span", textContent: "hello" }
+  }
+];
+```
+
+Complex matching via function (recommended form with `query`):
+
+```ts
+replaceElements: [
+  {
+    query: "iframe",
+    run: (ctx) => {
+      const src = ctx.$el.attr("src") || "";
+      if (!/youtube\.com\/embed/.test(src)) return;
+
+      return { type: "replaceWithHtml", html: "<div>Player</div>" };
+    }
+  }
+];
+```
+
+### Function context
+
+Function rules receive a Cheerio-aware context:
+
+```ts
+export interface ReplaceElementContext {
+  /** Cheerio root ($) for the current document */
+  $: CheerioAPI;
+  /** Cheerio wrapper for the matched element */
+  $el: Cheerio<any>;
+
+  /** Document URL being rewritten */
+  url: string;
+  /** Entry URL (top-level) */
+  entryUrl: string;
+
+  ruleIndex: number;
+  matchIndex: number;
+}
+```
+
+### Manual validation
+
+Run the CLI against a page containing the target elements:
+
+```bash
+pnpm -F @pagepocket/cli start -- https://example.com
+```
+
+Then inspect the output `*.html` for the expected replacements.

package/dist/builtin-blacklist.d.ts

@@ -0,0 +1,3 @@
+export declare const ga: RegExp[];
+export declare const sw_iframe: RegExp[];
+export declare const ns: RegExp[];

package/dist/builtin-blacklist.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ns = exports.sw_iframe = exports.ga = void 0;
+exports.ga = [/google-analytics/i, /analytics\.google/i];
+exports.sw_iframe = [/sw_iframe\.html/i];
+exports.ns = [/googletagmanager/i];

package/dist/debug.d.ts

@@ -0,0 +1,2 @@
+export declare const debug_log: (...args: unknown[]) => void;
+export declare const debugLog: (...args: unknown[]) => void;

package/dist/debug.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.debugLog = exports.debug_log = void 0;
+const isDebugEnabled = () => {
+    const globalProcess = globalThis
+        .process;
+    const value = globalProcess?.env?.PAGEPOCKET_DEBUG;
+    return Boolean(value);
+};
+const debug_log = (...args) => {
+    if (!isDebugEnabled()) {
+        return;
+    }
+    // eslint-disable-next-line no-console
+    console.log(...args);
+};
+exports.debug_log = debug_log;
+exports.debugLog = exports.debug_log;

package/dist/hackers/index.js

@@ -5,6 +5,7 @@ const preload_fetch_1 = require("./preload-fetch");
 const preload_xhr_1 = require("./preload-xhr");
 const replay_beacon_1 = require("./replay-beacon");
 const replay_block_text_fragment_1 = require("./replay-block-text-fragment");
+const replay_css_proxy_1 = require("./replay-css-proxy");
 const replay_dom_rewrite_1 = require("./replay-dom-rewrite");
 const replay_eventsource_1 = require("./replay-eventsource");
 const replay_fetch_1 = require("./replay-fetch");
@@ -18,6 +19,7 @@ exports.replayHackers = [
     replay_history_path_1.replayHistoryPath,
     replay_fetch_1.replayFetchResponder,
     replay_xhr_1.replayXhrResponder,
+    replay_css_proxy_1.replayCssProxy,
     replay_dom_rewrite_1.replayDomRewriter,
     replay_svg_image_1.replaySvgImageRewriter,
     replay_beacon_1.replayBeaconStub,

package/dist/hackers/replay-css-proxy.d.ts

@@ -0,0 +1,2 @@
+import type { ScriptHacker } from "./types";
+export declare const replayCssProxy: ScriptHacker;

package/dist/hackers/replay-css-proxy.js

@@ -0,0 +1,206 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayCssProxy = void 0;
+exports.replayCssProxy = {
+    id: "replay-css-proxy",
+    stage: "replay",
+    build: () => `
+  const shouldSkipCssUrl = (value) => {
+    const trimmed = String(value || "").trim();
+    return (
+      !trimmed ||
+      trimmed.startsWith("data:") ||
+      trimmed.startsWith("blob:") ||
+      trimmed.startsWith("mailto:") ||
+      trimmed.startsWith("tel:") ||
+      trimmed.startsWith("javascript:") ||
+      trimmed.startsWith("#")
+    );
+  };
+
+  const rewriteCssUrlsSync = (cssText, cssBaseUrl) => {
+    try {
+      const base = cssBaseUrl || (document && document.baseURI) || baseUrl;
+      const text = String(cssText || "");
+
+      // Rewrite url(...)
+      const urlPattern = /url\(\s*(['"]?)([^'")]+)\\1\s*\)/g;
+      const updated = text.replace(urlPattern, (full, quote, rawUrl) => {
+        try {
+          const raw = String(rawUrl || "").trim();
+          if (shouldSkipCssUrl(raw)) {
+            return full;
+          }
+          const absolute = new URL(raw, base).toString();
+          const localPath = findLocalPath(absolute);
+          if (!localPath) {
+            return full;
+          }
+          const q = quote || "";
+          return "url(" + q + localPath + q + ")";
+        } catch {
+          return full;
+        }
+      });
+
+      // Rewrite @import
+      const importPattern = /@import\s+(?:url\()?['"]?([^'")]+)['"]?\)?/g;
+      const final = updated.replace(importPattern, (full, rawUrl) => {
+        try {
+          const raw = String(rawUrl || "").trim();
+          if (shouldSkipCssUrl(raw)) {
+            return full;
+          }
+          const absolute = new URL(raw, base).toString();
+          const localPath = findLocalPath(absolute);
+          if (!localPath) {
+            return full;
+          }
+          return full.replace(rawUrl, localPath);
+        } catch {
+          return full;
+        }
+      });
+
+      return final;
+    } catch {
+      return String(cssText || "");
+    }
+  };
+
+  // <style> text mutations.
+  const rewriteStyleElement = (styleEl) => {
+    try {
+      if (!styleEl || !styleEl.tagName) return;
+      const tag = String(styleEl.tagName || "").toLowerCase();
+      if (tag !== "style") return;
+
+      const current = styleEl.textContent || "";
+      const next = rewriteCssUrlsSync(current, document && document.baseURI);
+      if (next !== current) {
+        styleEl.textContent = next;
+      }
+    } catch {}
+  };
+
+  // Inline style: element.setAttribute('style', ...)
+  const rewriteInlineStyleValue = (value) => {
+    try {
+      return rewriteCssUrlsSync(String(value || ""), document && document.baseURI);
+    } catch {
+      return String(value || "");
+    }
+  };
+
+  // Patch setAttribute to rewrite inline style values.
+  try {
+    const originalSetAttribute = Element.prototype.setAttribute;
+    Element.prototype.setAttribute = function(name, value) {
+      const attr = String(name).toLowerCase();
+      if (attr === "style") {
+        const next = rewriteInlineStyleValue(value);
+        return originalSetAttribute.call(this, name, next);
+      }
+      return originalSetAttribute.call(this, name, value);
+    };
+  } catch {}
+
+  // Patch CSSStyleDeclaration.setProperty.
+  try {
+    const originalSetProperty = CSSStyleDeclaration.prototype.setProperty;
+    CSSStyleDeclaration.prototype.setProperty = function(propertyName, value, priority) {
+      const next = rewriteInlineStyleValue(value);
+      return originalSetProperty.call(this, propertyName, next, priority);
+    };
+  } catch {}
+
+  // Patch CSSStyleDeclaration.cssText setter.
+  try {
+    const desc = Object.getOwnPropertyDescriptor(CSSStyleDeclaration.prototype, "cssText");
+    if (desc && desc.set) {
+      Object.defineProperty(CSSStyleDeclaration.prototype, "cssText", {
+        configurable: true,
+        get: desc.get,
+        set: function(value) {
+          const next = rewriteInlineStyleValue(value);
+          return desc.set.call(this, next);
+        }
+      });
+    }
+  } catch {}
+
+  // Patch CSSStyleSheet.insertRule.
+  try {
+    const originalInsertRule = CSSStyleSheet.prototype.insertRule;
+    CSSStyleSheet.prototype.insertRule = function(rule, index) {
+      const next = rewriteCssUrlsSync(String(rule || ""), document && document.baseURI);
+      return originalInsertRule.call(this, next, index);
+    };
+  } catch {}
+
+  // Patch constructable stylesheet APIs.
+  try {
+    if (CSSStyleSheet.prototype.replaceSync) {
+      const originalReplaceSync = CSSStyleSheet.prototype.replaceSync;
+      CSSStyleSheet.prototype.replaceSync = function(text) {
+        const next = rewriteCssUrlsSync(String(text || ""), document && document.baseURI);
+        return originalReplaceSync.call(this, next);
+      };
+    }
+  } catch {}
+
+  try {
+    if (CSSStyleSheet.prototype.replace) {
+      const originalReplace = CSSStyleSheet.prototype.replace;
+      CSSStyleSheet.prototype.replace = function(text) {
+        const next = rewriteCssUrlsSync(String(text || ""), document && document.baseURI);
+        return originalReplace.call(this, next);
+      };
+    }
+  } catch {}
+
+  // Observe <style> elements being added/updated.
+  try {
+    const rewriteAllStyles = () => {
+      try {
+        document.querySelectorAll("style").forEach((el) => rewriteStyleElement(el));
+      } catch {}
+    };
+
+    const observer = new MutationObserver((mutations) => {
+      for (const mutation of mutations) {
+        if (mutation.type === "childList") {
+          mutation.addedNodes.forEach((node) => {
+            try {
+              if (!node) return;
+              if (node.nodeType === 1) {
+                // Element
+                rewriteStyleElement(node);
+                if (node.querySelectorAll) {
+                  node.querySelectorAll("style").forEach((el) => rewriteStyleElement(el));
+                }
+              }
+            } catch {}
+          });
+        }
+
+        if (mutation.type === "characterData") {
+          const parent = mutation.target && mutation.target.parentElement;
+          if (parent && parent.tagName && String(parent.tagName).toLowerCase() === "style") {
+            rewriteStyleElement(parent);
+          }
+        }
+      }
+    });
+
+    observer.observe(document.documentElement, {
+      subtree: true,
+      childList: true,
+      characterData: true
+    });
+
+    // Initial pass.
+    onReady(() => rewriteAllStyles());
+  } catch {}
+  `
+};