@africanpilot/next-snapshot 0.2.0 → 0.2.1

package/README.md

@@ -195,14 +195,18 @@ other host it redirects to, add it to `aliases`.
 ## Security
 
 A config is code: `app.build`, `app.start`, `login` hooks and `explore.custom`
-run with your privileges. A snapshot contains every page it captured, for every
-variant — treat it like access to the app. See [SECURITY.md](SECURITY.md),
-including how to report a vulnerability.
+run with your privileges. A snapshot contains every page it captured, and the
+API responses those pages fetched, for every variant — treat it like access to
+the app, and check what is in it before sharing. Replayed pages run in a
+sandboxed frame so that content the app never trusted cannot carry the snapshot
+anywhere. See [SECURITY.md](SECURITY.md), including how to report a
+vulnerability.
 
 ## Development
 
 `npm test` runs the unit and end-to-end suites; `npm run test:next` snapshots a
-real Next.js app. See [CONTRIBUTING.md](CONTRIBUTING.md).
+real Next.js app. See [CONTRIBUTING.md](CONTRIBUTING.md). What might come next,
+and what was deliberately ruled out, is in [docs/ROADMAP.md](docs/ROADMAP.md).
 
 ## License
 

package/lib/bundle.mjs

@@ -51,7 +51,17 @@ export const CSP = [
 ].join("; ");
 
 export async function bundle(cfg, log) {
-  const M = JSON.parse(await fs.readFile(path.join(cfg.captureDir, "manifest.json"), "utf8"));
+  let M;
+  try {
+    M = JSON.parse(await fs.readFile(path.join(cfg.captureDir, "manifest.json"), "utf8"));
+  } catch (e) {
+    throw new Error(
+      `No capture to bundle at ${cfg.captureDir} (${e.code ?? e.message}). Run \`capture\` first, or \`all\` to do both.`,
+    );
+  }
+  if (!M.pages || !M.assets || !M.variantAssets) {
+    throw new Error(`The capture at ${cfg.captureDir} is incomplete or was written by an older version. Run \`capture\` again.`);
+  }
   const origin = M.origin;
   const readBody = (sha) => fs.readFile(path.join(cfg.captureDir, "bodies", sha));
   const t0 = Date.now();
@@ -217,6 +227,7 @@ export async function bundle(cfg, log) {
     badge: cfg.offline.badge,
     switcher: cfg.offline.switcher,
     missingLinks: cfg.offline.missingLinks,
+    staticPrefix: cfg.staticPrefix,
     codec: zstd ? "zstd" : "gzip",
     locs: zstd ? locs : undefined,
     assets: assets.map((a) => [a.k, a.v, a.b, a.type, a.status ?? 200]),
@@ -247,7 +258,8 @@ export async function bundle(cfg, log) {
 
   // --- report ----------------------------------------------------------------------
   const mb = (n) => (n / 1024 / 1024).toFixed(2) + " MB";
-  const assetBytes = assets.reduce((n, a) => n + (blocks.get(a.b)?.length ?? 0), 0);
+  // By body, not by asset row: one block shared by several keys is one cost.
+  const assetBytes = [...new Set(assets.map((a) => a.b))].reduce((n, b) => n + (blocks.get(b)?.length ?? 0), 0);
   const pageBytes = zstd
     ? clusters.reduce((n, c) => n + c.length, 0)
     : bodies.reduce((n, b, i) => n + (b.page ? blocks.get(i).length : 0), 0);

package/lib/capture.mjs

@@ -116,8 +116,11 @@ export async function capture(cfg, log) {
 
   const pending = new Set();
   const track = (p) => {
-    pending.add(p);
-    p.finally(() => pending.delete(p));
+    // Caught here: an unhandled rejection from a background body read (a full
+    // disk, too many open files) would otherwise end the whole crawl silently.
+    const q = p.catch((e) => log(`  warn: recording a response failed: ${e.message}`));
+    pending.add(q);
+    q.finally(() => pending.delete(q));
   };
 
   const browser = await launch(cfg);
@@ -567,6 +570,12 @@ function summarise(M, log, ms) {
     log(`  ${v.id.padEnd(22)} ${String(html).padStart(4)} pages  ${String(red).padStart(4)} redirects  ${Object.keys(M.variantAssets[v.id]).length} data responses`);
   }
   log(`  shared assets: ${Object.keys(M.assets).length}   RSC payloads skipped: ${M.rscSkipped}`);
+  // An app that is answering with error pages captures perfectly happily; say
+  // so, or the snapshot looks complete and is a book of 500s.
+  const errorPages = M.variants.flatMap((v) => Object.entries(M.pages[v.id]).filter(([, e]) => e.body && e.status >= 400));
+  if (errorPages.length) {
+    log(`  WARNING: ${errorPages.length} captured page(s) are error responses — e.g. ${errorPages[0][1].status} ${errorPages[0][0]}`);
+  }
   if (M.blocked.length) log(`  blocked ${M.blocked.length} non-GET request(s) — the crawl never writes`);
   if (M.failures.length) log(`  ${M.failures.length} navigation failure(s): ${M.failures.slice(0, 3).map((f) => f.key).join(", ")}`);
   if (M.liveErrors.length) log(`  the LIVE app threw ${M.liveErrors.length} error(s) during capture (first: ${M.liveErrors[0].message})`);

package/lib/runtime/frame.js

@@ -291,7 +291,8 @@
       NO.openInNewTab(u.href);
       return null;
     }
-    return realOpen.call(W, u.href, target || "_blank", features);
+    NO.external(u.href);
+    return null;
   };
   var jar = NO.cookies;
   try {
@@ -322,7 +323,7 @@
     if (!/^https?:$/.test(u.protocol)) return;
     e.preventDefault();
     if (a.hasAttribute("download") || NO.isAsset(u.href)) { NO.openAsset(u.href, a.getAttribute("download")); return; }
-    if (u.origin !== cur.origin) { realOpen.call(W, u.href, "_blank", "noopener"); return; }
+    if (u.origin !== cur.origin) { NO.external(u.href); return; }
     var t = a.getAttribute("target");
     if (newTab || e.metaKey || e.ctrlKey || e.shiftKey || (t && !/^_(self|top|parent)$/i.test(t))) { NO.openInNewTab(u.href); return; }
     go(u.href);

package/lib/runtime/shell.js

@@ -87,11 +87,15 @@
     idx[(a.v || "") + "\u0000" + a.k] = i;
   });
   var variant = M.defaultVariant;
+  // Where the build output lives, which basePath moves: the bundler and the
+  // runtime must agree, or a chunk with a query is found at bundle time and
+  // lost at read time.
+  var STATIC_PREFIX = M.staticPrefix || "/_next/static/";
   function lookup(key, v) {
     if (key == null) return -1;
     var i = idx[(v || "") + "\u0000" + key];
     if (i == null) i = idx["\u0000" + key];
-    if (i == null && key.indexOf("?") > 0 && key.indexOf("/_next/static/") === 0) i = idx["\u0000" + key.split("?")[0]];
+    if (i == null && key.indexOf("?") > 0 && key.indexOf(STATIC_PREFIX) === 0) i = idx["\u0000" + key.split("?")[0]];
     return i == null ? -1 : i;
   }
   var bytes = await Promise.all(A.map(function (a) { return body(a.b, true); }));
@@ -111,7 +115,12 @@
     return u;
   }
   function tokens(text) {
-    return text.replace(/__NOA(\d+)__/g, function (_, i) { return assetURL(+i); });
+    // A captured page may contain the token's own shape as ordinary text. An
+    // index that names no asset is left as it stands: one odd string in a page
+    // must not take down the file.
+    return text.replace(/__NOA(\d+)__/g, function (whole, i) {
+      return +i >= 0 && +i < A.length ? assetURL(+i) : whole;
+    });
   }
   for (var i = 0; i < A.length; i++) assetURL(i);
 
@@ -191,8 +200,20 @@
   }
 
   async function load(v, rawKey, hash, mode) {
+    try {
+      await loadPage(v, rawKey, hash, mode);
+    } catch (err) {
+      // A body that will not decode — a truncated file, a half-finished copy.
+      // Saying so beats leaving the reader on "Opening…" for ever.
+      NO.report("error", "could not open " + rawKey + ": " + ((err && err.message) || err));
+      current = { variant: variant, key: rawKey, hash: hash || "", base: rawKey };
+      mount(brokenHTML(rawKey, (err && err.message) || String(err)), NO.seq);
+    }
+  }
+
+  async function loadPage(v, rawKey, hash, mode) {
     var seq = ++NO.seq;
-    if (!M.pages[v]) v = M.defaultVariant;
+    if (!Object.prototype.hasOwnProperty.call(M.pages, v)) v = M.defaultVariant;
     setVariant(v);
     var r = resolvePage(rawKey, v), e = r.entry, html, base = r.key;
     if (!e && soft[v + "\u0000" + r.key]) {
@@ -216,6 +237,12 @@
     var f = document.createElement("iframe");
     f.className = "no-frame loading";
     f.setAttribute("title", M.title || "Application");
+    // A captured page may contain content its app never trusted — a comment
+    // field, a hostile API value — and its inline scripts run here. Same-origin
+    // is kept because the shim needs `parent.__NO`, but without top navigation
+    // or popups that content cannot carry the snapshot off to a server. Links
+    // that genuinely lead outside go through the shell instead (NO.external).
+    f.setAttribute("sandbox", "allow-scripts allow-same-origin allow-forms allow-modals");
     f.addEventListener("load", function () {
       NO.frameReady(f.contentWindow, seq);
       if (seq === NO.seq) {
@@ -276,6 +303,17 @@
     var u = new URL(href, ORIGIN);
     window.open(location.href.split("#")[0] + hashFor(variant, NO.key(u.href), u.hash), "_blank");
   };
+  // A link out of the app. The frame cannot open one itself — that is the point
+  // of its sandbox — so the shell asks, and the reader decides. Anything the
+  // page does without a click never gets here.
+  NO.external = function (href) {
+    var u;
+    try { u = new URL(href); } catch (e) { return; }
+    if (!/^https?:$/.test(u.protocol)) return;
+    if (window.confirm("This link leaves the offline snapshot and connects to the internet:\n\n" + u.href + "\n\nOpen it?")) {
+      window.open(u.href, "_blank", "noopener");
+    }
+  };
   NO.openAsset = function (href, name) {
     var i = NO.assetIndex(href);
     if (i >= 0) openAssetIndex(i, name);
@@ -418,6 +456,17 @@
   function esc(s) {
     return String(s).replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/"/g, "&quot;");
   }
+  function brokenHTML(key, why) {
+    return (
+      '<!doctype html><html><head><base href="' + esc(ORIGIN + key) + '"><script data-no-shim></script>' +
+      '<meta charset="utf-8"><title>This page could not be opened</title><style>body{font:14px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif;margin:48px auto;padding:0 24px;color:#222;max-width:720px}' +
+      "code{background:#f2f2f2;padding:1px 5px;border-radius:4px}</style></head><body>" +
+      '<h1 style="font-size:20px">This page could not be opened</h1><p><code>' + esc(key) + "</code> is in this snapshot, but its content would not decode:</p>" +
+      "<p><code>" + esc(why) + "</code></p>" +
+      "<p>The file is probably incomplete — a copy or download that did not finish. Try the original file again.</p></body></html>"
+    );
+  }
+
   function missingHTML(v, key) {
     var P = M.pages[v] || {}, path = key.split("?")[0];
     var pages = Object.keys(P).filter(function (k) { return P[k].b != null; }).sort();
@@ -438,7 +487,9 @@
 
   // --- boot ------------------------------------------------------------------------------------
   var start = parseHash();
-  if (start && M.pages[start.variant]) variant = start.variant;
+  // hasOwnProperty: a hash of "#__proto__:/" would otherwise name a variant
+  // that exists only on Object.prototype.
+  if (start && Object.prototype.hasOwnProperty.call(M.pages, start.variant)) variant = start.variant;
   renderBadge();
   NO.ready = true;
   if (start) load(variant, start.key, start.hash, "replace");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@africanpilot/next-snapshot",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Capture a running Next.js app and bundle it into one self-contained, offline HTML file.",
   "keywords": [
     "nextjs",