npm - @africanpilot/next-snapshot - Versions diffs - 0.1.2 → 0.2.0 - Mend

@africanpilot/next-snapshot 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

package/README.md CHANGED Viewed

@@ -106,9 +106,40 @@ A config is an ES module; relative paths resolve against it.
 | `offline.missingLinks` | `"show"` | Links to pages the snapshot does not hold: `"show"` (they open a "not in snapshot" page), `"disable"` (dimmed, not clickable) or `"hide"`. With `include` narrowing the crawl, `"hide"` removes the nav entries for everything left out. |
 | `offline.badge` | `"bottom-right"` | The "Offline snapshot" pill; `false` to hide. |
 | `offline.switcher` | `true` | A variant `<select>` in the badge. |
+| `compress` | `"gzip"` | How pages are packed. `"gzip"` stores each page on its own, decoded by the browser itself. `"zstd"` sorts pages by route, packs them into clusters of `clusterBytes` and compresses each cluster as one stream, inlining an 8 KB decoder — far smaller for an app with many similar pages. See [Size](#size-and-compression). |
+| `clusterBytes` | 4 MB | Raw bytes of pages per cluster, with `compress: "zstd"`. Bigger is smaller, but the first page of each cluster takes longer to open. |
 | `includeStatic` | `true` | Also embed every file under `.next/static`, so lazily-loaded chunks the crawl never triggered are present. |
 | `viewport`, `locale`, `timezoneId`, `browser` | | Passed to Chrome. `browser.executablePath` if Chrome is not installed. |
+## Size and compression
+Pages of an app repeat each other: the same layout, nav and table shell, over
+and over. By default each page is gzipped on its own, which cannot exploit that
+— gzip looks only 32 KB back, and a page's near-twin is further away than that.
+For a handful of pages this costs nothing worth fixing.
+For an app with hundreds of pages, `compress: "zstd"` sorts pages by route,
+packs them into clusters and compresses each cluster as one stream, so the
+repetition is paid for once:
+```js
+export default {
+  // …
+  compress: "zstd",
+  clusterBytes: 4 * 1024 * 1024,   // the default
+};
+```
+On a 260-page report app, measured: **12.4 MB → about 2 MB**. The cost is an
+8 KB decoder inlined in the file, and tens of milliseconds to open the first
+page of a cluster; pages in an already-decoded cluster are free. Assets stay
+per-page gzip either way, since they are wanted all at once at startup and are
+mostly already-compressed formats.
+Why not brotli, which is smaller still: Chrome cannot decompress brotli from
+JavaScript (`DecompressionStream` has no brotli there, and no zstd anywhere), so
+it would mean inlining a 208 KB decoder to save about 10%.
 ## An app running in Docker
 The tool drives Chrome on your machine, so it reaches the container the same way

package/lib/bundle.mjs CHANGED Viewed

@@ -6,12 +6,25 @@
 //         real URL and a placeholder for the frame shim go first in <head>.
 //   (The transformations themselves live in rewrite.mjs.)
 //
-// Every body is gzipped and base64'd into its own inert <script type=text/plain>.
-// The runtime (runtime/shell.js) decodes assets once into blob: URLs, replaces
-// tokens with them, and decodes a page only when it is navigated to.
+// Every body is then packed into the file as base64 in an inert
+// <script type="text/plain">, one of two ways:
+//
+//   compress: "gzip"  each body gzipped on its own. The browser decodes it
+//                     natively, and the file needs no decoder of its own.
+//   compress: "zstd"  pages are sorted by route and packed into clusters of
+//                     `clusterBytes`, each compressed as one zstd stream, with
+//                     a small decoder inlined. Pages of an app repeat each
+//                     other heavily and gzip's window is only 32KB, so a page
+//                     cannot see its near-twin; a cluster can. Assets stay
+//                     per-body gzip: they are wanted all at once at startup,
+//                     and most are already-compressed formats.
+//
+// The runtime (runtime/shell.js) decodes an asset once into a blob: URL, and a
+// page only when it is navigated to — a cluster at a time, cached.
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
+import { createRequire } from "node:module";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import zlib from "node:zlib";
@@ -20,6 +33,7 @@ import { urlKey } from "./key.js";
 import { createRewriter, escHTML, serialisePost, virtualiseLocation } from "./rewrite.mjs";
 const HERE = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
 // Nothing in the file may reach the network. Anything that tries is refused by
 // the browser and shows up as a CSP violation, which `verify` reports.
@@ -41,6 +55,12 @@ export async function bundle(cfg, log) {
   const origin = M.origin;
   const readBody = (sha) => fs.readFile(path.join(cfg.captureDir, "bodies", sha));
   const t0 = Date.now();
+  const zstd = cfg.compress === "zstd";
+  if (zstd && typeof zlib.zstdCompressSync !== "function") {
+    throw new Error(
+      `compress: "zstd" needs a Node built with zstd (zlib.zstdCompressSync); this is ${process.version}. Upgrade Node, or use compress: "gzip".`,
+    );
+  }
   // --- asset table ---------------------------------------------------------
   const assets = [];
@@ -74,18 +94,17 @@ export async function bundle(cfg, log) {
   });
   // --- bodies ---------------------------------------------------------------
-  const out = []; // id -> base64(gzip)
-  const outIndex = new Map(); // content hash -> id
-  const sizes = []; // {what, raw, packed}
-  function emit(buf, what) {
+  // Collected first, compressed at the end: how they are packed depends on all
+  // of them. Identical content is stored once, whatever refers to it.
+  const bodies = []; // id -> { buf, what, page }
+  const bodyIndex = new Map(); // content hash -> id
+  function emit(buf, what, page = false) {
     const h = crypto.createHash("sha256").update(buf).digest("hex");
-    let id = outIndex.get(h);
+    let id = bodyIndex.get(h);
     if (id == null) {
-      const packed = zlib.gzipSync(buf, { level: 9 }).toString("base64");
-      id = out.length;
-      out.push(packed);
-      outIndex.set(h, id);
-      sizes.push({ what, raw: buf.length, packed: packed.length });
+      id = bodies.length;
+      bodies.push({ buf, what, page });
+      bodyIndex.set(h, id);
     }
     return id;
   }
@@ -127,12 +146,58 @@ export async function bundle(cfg, log) {
         if (i >= 0) pagesOut[v][k] = { a: i };
       } else {
         const html = (await readBody(e.body)).toString("utf8");
-        pagesOut[v][k] = { b: emit(Buffer.from(rewriteHTML(html, k, v)), `page ${v} ${k}`), s: e.status };
+        pagesOut[v][k] = { b: emit(Buffer.from(rewriteHTML(html, k, v)), `page ${v} ${k}`, true), s: e.status };
         pageCount++;
       }
     }
   }
+  // --- packing ------------------------------------------------------------------
+  const gzipB64 = (buf) => zlib.gzipSync(buf, { level: 9 }).toString("base64");
+  const blocks = new Map(); // body id -> base64, for bodies stored on their own
+  const clusters = []; // base64 of each cluster
+  const locs = {}; // body id -> [cluster, offset, length], for clustered pages
+  let clusterRawMax = 0;
+  if (!zstd) {
+    for (let i = 0; i < bodies.length; i++) blocks.set(i, gzipB64(bodies[i].buf));
+  } else {
+    for (let i = 0; i < bodies.length; i++) if (!bodies[i].page) blocks.set(i, gzipB64(bodies[i].buf));
+    // Sorted by route and query, so a cluster holds pages that resemble each
+    // other — which is the whole reason a cluster is smaller than its parts.
+    const pageIds = bodies.map((b, i) => i).filter((i) => bodies[i].page).sort((x, y) => (bodies[x].what < bodies[y].what ? -1 : 1));
+    // A window at least as large as a cluster: a page must be able to match
+    // against any earlier page in the same cluster.
+    const windowLog = Math.min(27, Math.max(20, Math.ceil(Math.log2(Math.max(cfg.clusterBytes, 1)))));
+    let cur = [];
+    let size = 0;
+    const flush = () => {
+      if (!cur.length) return;
+      let offset = 0;
+      for (const i of cur) {
+        locs[i] = [clusters.length, offset, bodies[i].buf.length];
+        offset += bodies[i].buf.length;
+      }
+      const raw = Buffer.concat(cur.map((i) => bodies[i].buf));
+      clusterRawMax = Math.max(clusterRawMax, raw.length);
+      clusters.push(
+        zlib
+          .zstdCompressSync(raw, {
+            params: { [zlib.constants.ZSTD_c_compressionLevel]: 19, [zlib.constants.ZSTD_c_windowLog]: windowLog },
+          })
+          .toString("base64"),
+      );
+      cur = [];
+      size = 0;
+    };
+    for (const i of pageIds) {
+      if (cur.length && size + bodies[i].buf.length > cfg.clusterBytes) flush();
+      cur.push(i);
+      size += bodies[i].buf.length;
+    }
+    flush();
+  }
   // --- runtime + output ---------------------------------------------------------
   const frameSrc = await fs.readFile(path.join(HERE, "runtime", "frame.js"), "utf8");
   const shellSrc = (await fs.readFile(path.join(HERE, "runtime", "shell.js"), "utf8"))
@@ -152,6 +217,8 @@ export async function bundle(cfg, log) {
     badge: cfg.offline.badge,
     switcher: cfg.offline.switcher,
     missingLinks: cfg.offline.missingLinks,
+    codec: zstd ? "zstd" : "gzip",
+    locs: zstd ? locs : undefined,
     assets: assets.map((a) => [a.k, a.v, a.b, a.type, a.status ?? 200]),
     pages: pagesOut,
   };
@@ -168,7 +235,9 @@ export async function bundle(cfg, log) {
     `<div id="no-loading">Opening ${title}…</div>`,
     `<script type="application/json" id="no-manifest">${JSON.stringify(manifestOut).replace(/</g, "\\u003c")}</script>`,
   ];
-  for (let i = 0; i < out.length; i++) parts.push(`<script type="text/plain" id="no-b${i}">${out[i]}</script>`);
+  for (const [id, b64] of blocks) parts.push(`<script type="text/plain" id="no-b${id}">${b64}</script>`);
+  for (let i = 0; i < clusters.length; i++) parts.push(`<script type="text/plain" id="no-c${i}">${clusters[i]}</script>`);
+  if (zstd) parts.push(`<script>${await inlineZstdDecoder()}</script>`);
   parts.push(`<script>${shellSrc.replace(/<\/script/gi, "<\\/script").replace(/<!--/g, "<\\!--")}</script>`);
   parts.push(`</body></html>\n`);
@@ -178,25 +247,57 @@ export async function bundle(cfg, log) {
   // --- report ----------------------------------------------------------------------
   const mb = (n) => (n / 1024 / 1024).toFixed(2) + " MB";
-  const pageBytes = sizes.filter((s) => s.what.startsWith("page")).reduce((n, s) => n + s.packed, 0);
-  const assetBytes = sizes.filter((s) => s.what.startsWith("asset")).reduce((n, s) => n + s.packed, 0);
+  const assetBytes = assets.reduce((n, a) => n + (blocks.get(a.b)?.length ?? 0), 0);
+  const pageBytes = zstd
+    ? clusters.reduce((n, c) => n + c.length, 0)
+    : bodies.reduce((n, b, i) => n + (b.page ? blocks.get(i).length : 0), 0);
   log("");
   log(`bundle finished in ${((Date.now() - t0) / 1000).toFixed(1)}s -> ${path.relative(process.cwd(), cfg.out)}`);
   log(`  file size        ${mb(html.length)}   (pages ${mb(pageBytes)}, assets ${mb(assetBytes)})`);
-  log(`  pages            ${pageCount} captured, ${out.length} unique bodies after dedupe`);
+  log(`  pages            ${pageCount} captured, ${bodies.filter((b) => b.page).length} unique bodies after dedupe`);
   log(`  assets           ${assets.length} (${jsRewritten} scripts with location virtualised)`);
-  const biggest = [...sizes].sort((a, b) => b.packed - a.packed).slice(0, 6);
-  log(`  largest bodies   ${biggest.map((s) => `${s.what.slice(0, 70)} ${mb(s.packed)}`).join("\n                   ")}`);
+  if (zstd) {
+    log(`  clusters         ${clusters.length} zstd, up to ${mb(clusterRawMax)} of pages each, decoded on demand`);
+  } else {
+    log(`  compression      gzip per page — try compress: "zstd" if this file is large`);
+  }
   if (missing.size) {
     const top = [...missing.entries()].sort((a, b) => b[1] - a[1]).slice(0, 10);
     log(`  NOT CAPTURED     ${missing.size} referenced URL(s) will fail offline:`);
     for (const [k, n] of top) log(`                   ${k}  (${n}x)`);
   }
-  const report = { file: cfg.out, bytes: html.length, pageCount, bodies: out.length, missing: Object.fromEntries(missing), sizes };
+  const report = {
+    file: cfg.out,
+    bytes: html.length,
+    pageCount,
+    codec: manifestOut.codec,
+    clusters: clusters.length,
+    pageBytes,
+    assetBytes,
+    missing: Object.fromEntries(missing),
+  };
   await fs.writeFile(cfg.out.replace(/\.html?$/, "") + ".bundle.json", JSON.stringify(report, null, 1));
   return report;
 }
+/**
+ * fzstd's UMD build, which assigns a `fzstd` global when run as a plain script.
+ * Read by path rather than resolved: the package's exports map does not expose
+ * this build, and it is the only one that works inside the snapshot.
+ */
+async function inlineZstdDecoder() {
+  // Resolved through the package's main entry, then up to its directory: the
+  // exports map does not expose "./package.json" or "./umd/index.js", so
+  // neither can be resolved by subpath.
+  const pkg = path.dirname(path.dirname(require.resolve("fzstd")));
+  const umd = path.join(pkg, "umd", "index.js");
+  try {
+    return await fs.readFile(umd, "utf8");
+  } catch (e) {
+    throw new Error(`compress: "zstd" needs fzstd's UMD build at ${umd}, which is missing (${e.code}). Reinstall dependencies, or use compress: "gzip".`);
+  }
+}
 const SHELL_CSS = `
 html,body{margin:0;height:100%;overflow:hidden;background:#fff}
 .no-frame{position:fixed;inset:0;width:100%;height:100%;border:0;display:block;background:#fff}

package/lib/config.mjs CHANGED Viewed

@@ -38,6 +38,9 @@ export async function loadConfig(file) {
     label: v.id,
     ...v,
   }));
+  const compress = raw.compress ?? "gzip";
+  if (compress !== "gzip" && compress !== "zstd") throw new Error(`compress must be "gzip" or "zstd", not ${JSON.stringify(compress)}`);
   const ids = new Set();
   for (const v of variants) {
     if (!/^[A-Za-z0-9_.-]+$/.test(v.id)) throw new Error(`Variant id "${v.id}" must be [A-Za-z0-9_.-]+`);
@@ -93,6 +96,17 @@ export async function loadConfig(file) {
     // `docker cp` it out before bundling. `staticPath` is where the build lives
     // inside the image (Next's own Dockerfile puts it under /app).
     docker: raw.docker ? { staticPath: "/app/.next/static", ...raw.docker } : null,
+    // How page bodies are packed into the file.
+    //
+    //   "gzip"  each page gzipped on its own, decoded by the browser itself.
+    //   "zstd"  pages of a route packed together into clusters and compressed
+    //           as one stream, with a small decoder inlined. Pages of an app
+    //           mostly repeat each other, and gzip's 32KB window cannot see
+    //           past one page — clustering is what makes a big app small.
+    compress,
+    // Raw bytes of pages per cluster, when clustering. Bigger is smaller but
+    // slower to open the first page of each cluster.
+    clusterBytes: raw.clusterBytes ?? 4 * 1024 * 1024,
     includeStatic: raw.includeStatic ?? true,
     staticDir: r(raw.staticDir) ?? defaultStatic,
     staticPrefix: raw.staticPrefix ?? "/_next/static/",

package/lib/runtime/shell.js CHANGED Viewed

@@ -41,15 +41,39 @@
     for (var i = 0; i < n; i++) a[i] = bin.charCodeAt(i);
     return a;
   }
+  // Pages may be packed in clusters: many of them compressed as one stream,
+  // because pages of an app repeat each other and gzip cannot see past one
+  // page. A cluster is decoded on first use and the last few are kept.
+  var LOCS = M.locs || null;
+  var clusterCache = new Map();
+  function cluster(i) {
+    var hit = clusterCache.get(i);
+    if (hit) return hit;
+    var p = (async function () {
+      var el = document.getElementById("no-c" + i);
+      if (!el) throw new Error("snapshot cluster " + i + " is missing");
+      return fzstd.decompress(b64(el.textContent.trim()));
+    })();
+    clusterCache.set(i, p);
+    // Three decoded clusters is a few MB; older ones cost nothing to redo.
+    if (clusterCache.size > 3) clusterCache.delete(clusterCache.keys().next().value);
+    return p;
+  }
   var kept = {};
   function body(id, keep) {
     if (kept[id]) return kept[id];
-    var p = (async function () {
-      var el = document.getElementById("no-b" + id);
-      if (!el) throw new Error("snapshot body " + id + " is missing");
-      var stream = new Blob([b64(el.textContent.trim())]).stream().pipeThrough(new DecompressionStream("gzip"));
-      return new Uint8Array(await new Response(stream).arrayBuffer());
-    })();
+    var loc = LOCS && LOCS[id];
+    var p = loc
+      ? cluster(loc[0]).then(function (bytes) {
+          return bytes.subarray(loc[1], loc[1] + loc[2]);
+        })
+      : (async function () {
+          var el = document.getElementById("no-b" + id);
+          if (!el) throw new Error("snapshot body " + id + " is missing");
+          var stream = new Blob([b64(el.textContent.trim())]).stream().pipeThrough(new DecompressionStream("gzip"));
+          return new Uint8Array(await new Response(stream).arrayBuffer());
+        })();
     if (keep) kept[id] = p;
     return p;
   }

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@africanpilot/next-snapshot",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Capture a running Next.js app and bundle it into one self-contained, offline HTML file.",
   "keywords": [
     "nextjs",
@@ -45,6 +45,7 @@
   },
   "dependencies": {
     "esbuild": "^0.28.2",
+    "fzstd": "^0.1.1",
     "playwright-core": "~1.63.0"
   },
   "publishConfig": {