@instafy/cli 0.2.3 → 0.2.4

package/dist/bind/src/starters/timeline/data.json

@@ -0,0 +1,41 @@
+{
+  "title": "A three-day plan",
+  "lede": "Set up on day one, run the sessions on day two, wrap up on day three.",
+  "days": [
+    {
+      "n": 1,
+      "date": "2026-10-24",
+      "place": "Main hall",
+      "stops": [
+        { "ref": "stop:setup", "name": "Set up the room", "note": "Chairs for 40, one screen" },
+        { "ref": "stop:sound-check", "name": "Sound check", "time": { "v": "14:00", "src": "Venue booking", "url": "https://example.com/booking", "date": "2026-10-17" } },
+        { "ref": "stop:walkthrough", "name": "Walk-through with the team" }
+      ],
+      "legs": []
+    },
+    {
+      "n": 2,
+      "date": "2026-10-25",
+      "place": "Main hall and Room B",
+      "stops": [
+        { "ref": "stop:doors", "name": "Doors open", "time": { "v": "09:00", "src": "Venue booking", "url": "https://example.com/booking", "date": "2026-10-17" } },
+        { "ref": "stop:sessions", "name": "Morning sessions", "note": "Three talks, 30 minutes each" },
+        { "ref": "stop:workshop", "name": "Workshop in Room B", "facts": [{ "note": "step-free access to Room B" }] }
+      ],
+      "legs": [
+        { "from": "stop:sessions", "to": "stop:workshop", "mode": "walk", "duration": { "v": "5 min", "est": true } }
+      ]
+    },
+    {
+      "n": 3,
+      "date": "2026-10-26",
+      "place": "Online",
+      "stops": [
+        { "ref": "stop:review", "name": "Review call", "time": { "v": "10:00", "src": "Team calendar", "url": "https://example.com/calendar", "date": "2026-10-17" } },
+        { "ref": "stop:send-notes", "name": "Send the notes to everyone" }
+      ],
+      "legs": []
+    }
+  ],
+  "unchecked": ["Ask the venue whether Room B is step-free", "Confirm day 2 lunch with the caterer"]
+}

package/dist/bind/src/starters/timeline/index.html

@@ -0,0 +1,22 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
+<meta name="theme-color" content="#FFFFFF" media="(prefers-color-scheme: light)">
+<meta name="theme-color" content="#141414" media="(prefers-color-scheme: dark)">
+<title>Day plan</title>
+<link rel="stylesheet" href="theme.css">
+<!-- Restyle only when asked: override --if-* tokens in a <style> here, after theme.css. -->
+</head>
+<body>
+<main class="page">
+  <header class="head"><h1 class="title" id="title">Day plan</h1><p class="lede" id="lede"></p></header>
+  <section id="list"></section>
+  <footer class="notes" id="notes" hidden></footer>
+</main>
+<script src="__bind/client.js"></script>
+<script src="site.js"></script>
+<script src="app.js"></script>
+</body>
+</html>

package/dist/bind/src/static.mjs

@@ -0,0 +1,141 @@
+// Static file serving: GET/HEAD, Range, MIME types, index.html, no dotfiles, no symlink escapes.
+import fs from "node:fs";
+import path from "node:path";
+
+export const MIME = {
+  ".html": "text/html; charset=utf-8", ".htm": "text/html; charset=utf-8", ".js": "text/javascript; charset=utf-8",
+  ".mjs": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".json": "application/json; charset=utf-8",
+  ".geojson": "application/geo+json; charset=utf-8", ".map": "application/json; charset=utf-8",
+  ".txt": "text/plain; charset=utf-8", ".md": "text/markdown; charset=utf-8", ".csv": "text/csv; charset=utf-8",
+  ".xml": "application/xml; charset=utf-8", ".svg": "image/svg+xml", ".png": "image/png", ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp", ".avif": "image/avif", ".ico": "image/x-icon",
+  ".woff": "font/woff", ".woff2": "font/woff2", ".ttf": "font/ttf", ".otf": "font/otf", ".mp4": "video/mp4",
+  ".webm": "video/webm", ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", ".pdf": "application/pdf",
+  ".wasm": "application/wasm", ".pmtiles": "application/octet-stream", ".pbf": "application/x-protobuf",
+  ".webmanifest": "application/manifest+json",
+};
+
+export function mimeOf(file) {
+  return MIME[path.extname(file).toLowerCase()] || "application/octet-stream";
+}
+
+/** Parse a single "bytes=a-b" range. Returns {start, end}, "unsatisfiable", or null (serve whole file). */
+export function parseRange(header, size) {
+  if (!header) return null;
+  const match = /^bytes=(\d*)-(\d*)$/.exec(String(header).trim());
+  if (!match) return null;
+  const [, a, b] = match;
+  if (a === "" && b === "") return null;
+  let start;
+  let end;
+  if (a === "") {
+    const suffix = Number(b);
+    if (suffix === 0) return "unsatisfiable";
+    start = Math.max(0, size - suffix);
+    end = size - 1;
+  } else {
+    start = Number(a);
+    end = b === "" ? size - 1 : Math.min(Number(b), size - 1);
+  }
+  if (start >= size || start > end) return "unsatisfiable";
+  return { start, end };
+}
+
+function send(res, status, text, headers = {}) {
+  res.writeHead(status, { "content-type": "text/plain; charset=utf-8", "x-content-type-options": "nosniff", ...headers });
+  res.end(text);
+}
+
+/**
+ * Resolve a URL path inside root. Returns {file, stat} | {redirect} | {status}.
+ * Dot segments (".bind", ".git", ".env") are never served.
+ */
+export function resolveStatic(root, urlPath) {
+  let decoded;
+  try {
+    decoded = decodeURIComponent(urlPath);
+  } catch {
+    return { status: 400 };
+  }
+  if (decoded.includes("\0") || decoded.includes("\\")) return { status: 400 };
+  const segments = decoded.split("/").filter(Boolean);
+  if (segments.some((segment) => segment.startsWith(".") )) return { status: 404 };
+  let file = path.join(root, ...segments);
+  let stat;
+  try {
+    const real = fs.realpathSync(file);
+    const realRoot = fs.realpathSync(root);
+    if (real !== realRoot && !real.startsWith(realRoot + path.sep)) return { status: 404 };
+    stat = fs.statSync(real);
+    file = real;
+  } catch {
+    return { status: 404 };
+  }
+  if (stat.isDirectory()) {
+    if (!urlPath.endsWith("/")) return { redirect: `${urlPath}/` };
+    const index = path.join(file, "index.html");
+    try {
+      stat = fs.statSync(index);
+      file = index;
+    } catch {
+      return { status: 404, directory: file };
+    }
+  }
+  if (!stat.isFile()) return { status: 404 };
+  return { file, stat };
+}
+
+/**
+ * Serve a static file for GET/HEAD. `notFound(res)` may render something custom. `redirectBase` prefixes directory
+ * redirects (urlPath is relative to root); `htmlHeaders` are added to HTML responses only.
+ */
+export function serveStatic(req, res, root, urlPath, { notFound = null, query = "", headers: extra = {}, htmlHeaders = {}, redirectBase = "", htmlTransform = null } = {}) {
+  if (req.method !== "GET" && req.method !== "HEAD") {
+    send(res, 405, "Method not allowed", { allow: "GET, HEAD" });
+    return;
+  }
+  const resolved = resolveStatic(root, urlPath);
+  if (resolved.redirect) {
+    res.writeHead(301, { location: redirectBase + resolved.redirect + (query ? `?${query}` : "") });
+    res.end();
+    return;
+  }
+  if (resolved.status) {
+    if (resolved.status === 404 && notFound && notFound(res, resolved)) return;
+    send(res, resolved.status, resolved.status === 404 ? "Not found" : "Bad request");
+    return;
+  }
+  const { file, stat } = resolved;
+  const headers = {
+    "content-type": mimeOf(file),
+    "accept-ranges": "bytes",
+    "cache-control": "no-cache",
+    "last-modified": stat.mtime.toUTCString(),
+    "x-content-type-options": "nosniff",
+    "referrer-policy": "no-referrer",
+    ...extra,
+    ...(mimeOf(file).startsWith("text/html") ? htmlHeaders : {}),
+  };
+  if (htmlTransform && mimeOf(file).startsWith("text/html")) {
+    // a page the binder adds to (the bind client for a plain site): the whole page, never a byte range of the file
+    let body;
+    try { body = Buffer.from(htmlTransform(fs.readFileSync(file, "utf8")), "utf8"); } catch { send(res, 500, "Could not read the page"); return; }
+    res.writeHead(200, { ...headers, "accept-ranges": "none", "content-length": body.length });
+    res.end(req.method === "HEAD" ? undefined : body);
+    return;
+  }
+  const range = parseRange(req.headers.range, stat.size);
+  if (range === "unsatisfiable") {
+    send(res, 416, "Range not satisfiable", { "content-range": `bytes */${stat.size}` });
+    return;
+  }
+  if (range) {
+    res.writeHead(206, { ...headers, "content-range": `bytes ${range.start}-${range.end}/${stat.size}`, "content-length": range.end - range.start + 1 });
+    if (req.method === "HEAD") return res.end();
+    fs.createReadStream(file, { start: range.start, end: range.end }).pipe(res);
+    return;
+  }
+  res.writeHead(200, { ...headers, "content-length": stat.size });
+  if (req.method === "HEAD") return res.end();
+  fs.createReadStream(file).on("error", () => res.destroy()).pipe(res);
+}

package/dist/bind/src/tokens.mjs

@@ -0,0 +1,102 @@
+// Viewer and admin tokens. Both live under .bind/ (mode 0700, files 0600); the static server never serves dotfiles.
+import fs from "node:fs";
+import path from "node:path";
+import { randomBytes, timingSafeEqual } from "node:crypto";
+
+export const BIND_DIR = ".bind";
+const TOKEN_RE = /^[A-Za-z0-9_-]{16,128}$/;
+
+export function randomToken(bytes = 24) {
+  return randomBytes(bytes).toString("base64url");
+}
+
+export function bindDir(dir) {
+  const full = path.join(dir, BIND_DIR);
+  fs.mkdirSync(full, { recursive: true, mode: 0o700 });
+  try {
+    fs.chmodSync(full, 0o700);
+  } catch {
+    // best effort on unusual file systems
+  }
+  // Tokens and the binder's log stay on this machine: a space's files are committed and synced, and the admin
+  // token must not end up in its history.
+  const ignore = path.join(full, ".gitignore");
+  try {
+    if (!fs.existsSync(ignore)) fs.writeFileSync(ignore, "# instafy-bind: tokens and logs stay on this machine\n*\n", { mode: 0o600 });
+  } catch {
+    // best effort
+  }
+  return full;
+}
+
+/** Read a token file, or create it with a fresh token. `rotate` always writes a new one. */
+export function ensureTokenFile(file, { rotate = false } = {}) {
+  if (!rotate) {
+    try {
+      const value = fs.readFileSync(file, "utf8").trim();
+      if (TOKEN_RE.test(value)) return value;
+    } catch {
+      // create below
+    }
+  }
+  const token = randomToken();
+  const tmp = `${file}.${process.pid}.tmp`;
+  fs.writeFileSync(tmp, `${token}\n`, { mode: 0o600 });
+  fs.renameSync(tmp, file);
+  fs.chmodSync(file, 0o600);
+  return token;
+}
+
+/** Per-site viewer token, shared in the URL as #k=<token>. */
+export function viewerToken(siteDir, options) {
+  return ensureTokenFile(path.join(bindDir(siteDir), "viewer-token"), options);
+}
+
+/** Admin token for commands and confirmations, in <root>/.bind/admin-token. The site never sees it. */
+export function adminToken(rootDir, options) {
+  return ensureTokenFile(path.join(bindDir(rootDir), "admin-token"), options);
+}
+
+export function readAdminToken(rootDir) {
+  const value = fs.readFileSync(path.join(rootDir, BIND_DIR, "admin-token"), "utf8").trim();
+  if (!TOKEN_RE.test(value)) throw new Error("malformed .bind/admin-token");
+  return value;
+}
+
+/** Constant-time string comparison. */
+export function tokensEqual(a, b) {
+  if (typeof a !== "string" || typeof b !== "string") return false;
+  const left = Buffer.from(a);
+  const right = Buffer.from(b);
+  return left.length === right.length && timingSafeEqual(left, right);
+}
+
+/** Walk up from dir to the folder holding .bind/server.json (written by `serve`). */
+export function findServerState(dir) {
+  let current = path.resolve(dir);
+  for (;;) {
+    const file = path.join(current, BIND_DIR, "server.json");
+    if (fs.existsSync(file)) {
+      return { root: current, state: JSON.parse(fs.readFileSync(file, "utf8")) };
+    }
+    const parent = path.dirname(current);
+    if (parent === current) return null;
+    current = parent;
+  }
+}
+
+export function writeServerState(rootDir, state) {
+  const file = path.join(bindDir(rootDir), "server.json");
+  fs.writeFileSync(file, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 });
+  return file;
+}
+
+export function removeServerState(rootDir, pid = process.pid) {
+  const file = path.join(rootDir, BIND_DIR, "server.json");
+  try {
+    const state = JSON.parse(fs.readFileSync(file, "utf8"));
+    if (state.pid === pid) fs.rmSync(file, { force: true });
+  } catch {
+    // already gone
+  }
+}

package/dist/bind/src/try.mjs

@@ -0,0 +1,139 @@
+// `instafy-bind try <dir> <intent> [params] [--save]`: run one script intent exactly as a viewer's click would (same
+// input, same handler environment, same record checks), without serving the site; --save stores what it accepted. A model building a
+// site uses it before replying, so a button that crashes or whose records are rejected never ships silently.
+// SPDX-License-Identifier: AGPL-3.0-only
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { executorOf, loadBound, resolveIntent, targetOf } from "./bound.mjs";
+import { applyDefaults, formatErrors, validate } from "./schema-lite.mjs";
+import { checkRecords, writeRecords } from "./apply.mjs";
+import { runScript } from "./executors/script.mjs";
+import { bindDir } from "./tokens.mjs";
+
+const TRIED_FILE = "tried.json";
+
+function readJsonMaybe(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/** sha256 of a handler file (the doctor compares it with the last successful try). */
+export function handlerHash(file) {
+  try {
+    return createHash("sha256").update(fs.readFileSync(file)).digest("hex").slice(0, 16);
+  } catch {
+    return null;
+  }
+}
+
+/** The last tries of a site: { [intent]: { hash, ok, at } }. */
+export function readTried(siteDir) {
+  return readJsonMaybe(path.join(siteDir, ".bind", TRIED_FILE)) || {};
+}
+
+function writeTried(siteDir, name, entry) {
+  try {
+    const dir = bindDir(siteDir);
+    const tried = readTried(siteDir);
+    tried[name] = entry;
+    fs.writeFileSync(path.join(dir, TRIED_FILE), `${JSON.stringify(tried, null, 2)}\n`, { mode: 0o600 });
+  } catch {
+    // the result is still printed
+  }
+}
+
+function describe(record) {
+  const what = record.type === "claim" ? `${record.attr}: ${record.status === "uncertain" ? `uncertain (${record.note || "no note"})` : JSON.stringify(record.value)}`
+    : record.type === "offer" ? `offer ${record.price?.amount ?? "?"} ${record.price?.currency ?? ""} at ${record.seller?.name ?? record.prov?.source ?? "?"}`
+      : `${record.type}`;
+  return `${record.ref ?? ""} ${what} · ${record.prov?.source ?? "?"}`.trim();
+}
+
+/**
+ * Run one script intent. Resolves { exit, lines, accepted, rejected, error }.
+ * exit 0: the handler ended with at least one accepted record (a value or an uncertain claim).
+ * exit 1: it crashed, reported an error, sent no records, or every record was rejected.
+ * exit 64: the request itself was wrong (unknown intent, bad params, not a script intent).
+ */
+export async function tryIntent(siteDir, name, rawParams = "{}", { allowLocal = false, allowEnv = [], timeoutMs = null, save = false } = {}) {
+  const dir = path.resolve(siteDir);
+  const lines = [];
+  let bound;
+  try {
+    bound = loadBound(dir);
+  } catch (error) {
+    return { exit: 64, lines: [`error ${error.message}`] };
+  }
+  const def = bound.intents?.[name];
+  if (!def) return { exit: 64, lines: [`error unknown intent "${name}" (intents: ${Object.keys(bound.intents || {}).join(", ")})`] };
+  if (executorOf(def) !== "script") return { exit: 64, lines: [`error ${name} is an agent intent; check its prompt with serve --dry-run instead`] };
+  const resolved = resolveIntent(def, { siteDir: dir });
+  if (!resolved.ok) return { exit: 64, lines: [`error ${name}: ${resolved.reason}`] };
+  let params;
+  try {
+    params = applyDefaults(def.params, JSON.parse(rawParams || "{}"));
+  } catch (error) {
+    return { exit: 64, lines: [`error params are not JSON: ${error.message}`] };
+  }
+  const problems = validate(def.params, params);
+  if (problems.length) return { exit: 64, lines: [`error params: ${formatErrors(problems)}`] };
+
+  const intent = { id: `try_${Date.now().toString(36)}`, name, params, ref: typeof params.ref === "string" ? params.ref : null, at: new Date().toISOString() };
+  const records = [];
+  const controller = new AbortController();
+  const limitS = timeoutMs ? timeoutMs / 1000 : def.timeout_s || 90;
+  const timer = setTimeout(() => controller.abort({ code: "timeout", message: `no answer within ${limitS} s` }), limitS * 1000);
+  timer.unref?.();
+  let summary = null;
+  let failure = null;
+  try {
+    const done = await runScript({
+      handler: resolved.handler,
+      siteDir: dir,
+      input: { intent, bound: def, site: { dir, title: bound.title || path.basename(dir) } },
+      signal: controller.signal,
+      onProgress: (p) => lines.push(`…     ${p.step ?? ""}/${p.of ?? ""} ${p.message ?? ""}`.trimEnd()),
+      onRecords: (batch) => records.push(...batch),
+      log: (level, text) => {
+        if (level !== "debug") lines.push(`${level.padEnd(5)} ${text}`);
+      },
+      allowLocal,
+      envNames: def.env || [],
+      allowEnv,
+    });
+    summary = done.summary;
+  } catch (error) {
+    failure = error;
+  } finally {
+    clearTimeout(timer);
+  }
+  const { accepted, rejected } = checkRecords(structuredClone(records), { name, def, intent, job: intent.id });
+  for (const record of accepted) lines.push(`ok    ${describe(record)}`);
+  for (const r of rejected) lines.push(`error record #${r.index} rejected: ${r.reason}`);
+  let exit = 0;
+  if (failure) {
+    lines.push(`error ${failure.code || "upstream"}: ${failure.message}`);
+    exit = 1;
+  } else if (!records.length) {
+    lines.push("error the handler sent no records: a click would leave the cell empty (send a value or kit.uncertain with a note)");
+    exit = 1;
+  } else if (!accepted.length) {
+    lines.push("error every record was rejected: a click would show an error");
+    exit = 1;
+  }
+  if (summary && !failure) lines.push(`done  ${summary}`);
+  // --save: the accepted records go into the site the way a click's do (same merge, version bump), so a new page
+  // opens with its values; a failed try saves nothing
+  let saved = null;
+  if (save && exit === 0 && accepted.length) {
+    const file = writeRecords(dir, targetOf(name, def), accepted, { attribution: bound.attribution || [] });
+    saved = { file: file.rel, records: file.applied.length, version: file.version };
+    lines.push(`saved ${file.applied.length} record(s) to ${file.rel} (version ${file.version}); a page that is already open shows them after a reload command`);
+  } else if (save) lines.push("saved nothing: the try failed");
+  writeTried(dir, name, { hash: handlerHash(resolved.handler), ok: exit === 0, at: new Date().toISOString() });
+  return { exit, lines, accepted, rejected, saved, error: failure ? { code: failure.code, message: failure.message } : null };
+}

package/dist/bind/src/web.mjs

@@ -0,0 +1,191 @@
+// Polite page reads for script handlers (kit.fetch): honour robots.txt for "*" and the major AI user agents (checked
+// again on every redirect hop) and stop at bot checks, CAPTCHAs and login walls. None is ever worked around. What a
+// page returns is data to quote with its source, never instructions.
+// SPDX-License-Identifier: AGPL-3.0-only
+import { isUnavailable, pageFetch } from "./page-fetch.mjs";
+
+export const USER_AGENT = "Instafy-Agent/1.0 (+https://instafy.dev; reads one public page on a person's request)";
+export const AI_AGENTS = ["gptbot", "chatgpt-user", "oai-searchbot", "claudebot", "claude-user", "anthropic-ai", "perplexitybot", "ccbot", "google-extended"];
+
+/** robots.txt → [{ agents: [lowercase], rules: [{ allow, path }] }] */
+export function parseRobots(txt) {
+  const groups = [];
+  let current = null;
+  let lastWasAgent = false;
+  for (const raw of String(txt || "").split(/\r?\n/)) {
+    const line = raw.replace(/#.*/, "").trim();
+    const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
+    if (!m) continue;
+    const key = m[1].toLowerCase();
+    const value = m[2].trim();
+    if (key === "user-agent") {
+      if (!current || !lastWasAgent) {
+        current = { agents: [], rules: [] };
+        groups.push(current);
+      }
+      current.agents.push(value.toLowerCase());
+      lastWasAgent = true;
+    } else if ((key === "allow" || key === "disallow") && current) {
+      lastWasAgent = false;
+      if (value) current.rules.push({ allow: key === "allow", path: value });
+    } else lastWasAgent = false;
+  }
+  return groups;
+}
+
+const UNRESERVED = /[A-Za-z0-9\-._~]/;
+/**
+ * The form robots.txt paths are compared in (RFC 9309 §2.2.2): %XX escapes of unreserved characters decoded, other
+ * escapes upper-cased, and (request paths) runs of "/" folded, so `/%70rivate` or `//private` cannot slip past
+ * `Disallow: /private` while the server still serves /private.
+ */
+export function normalizeRobotsPath(value, { foldSlashes = true } = {}) {
+  let text = String(value).replace(/%([0-9A-Fa-f]{2})/g, (m, hex) => {
+    const ch = String.fromCharCode(parseInt(hex, 16));
+    return UNRESERVED.test(ch) ? ch : `%${hex.toUpperCase()}`;
+  });
+  if (foldSlashes) {
+    const q = text.indexOf("?");
+    const pathPart = q < 0 ? text : text.slice(0, q);
+    text = pathPart.replace(/\/{2,}/g, "/") + (q < 0 ? "" : text.slice(q));
+  }
+  return text;
+}
+
+function ruleMatches(rule, p) {
+  const normalized = normalizeRobotsPath(rule, { foldSlashes: false });
+  const re = new RegExp("^" + normalized.replace(/[.+?^{}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\\\$$|\$$/, "$"));
+  return re.test(normalizeRobotsPath(p));
+}
+
+/** Is `pathWithQuery` allowed for one agent? Longest matching rule wins; Allow wins a tie. */
+export function robotsAllows(groups, agent, pathWithQuery) {
+  const a = agent.toLowerCase();
+  const named = (g) => g.agents.some((x) => x !== "*" && (a === x || a.startsWith(x)));
+  let rules = groups.filter(named).flatMap((g) => g.rules);
+  if (!groups.some(named)) rules = groups.filter((g) => g.agents.includes("*")).flatMap((g) => g.rules);
+  let best = null;
+  for (const r of rules) {
+    if (!ruleMatches(r.path, pathWithQuery)) continue;
+    if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow)) best = r;
+  }
+  return !best || best.allow;
+}
+
+/** The first agent (of "*" and the AI agents) that robots.txt keeps out of this path, or null. */
+export function robotsBlocker(groups, pathWithQuery) {
+  for (const agent of ["*", ...AI_AGENTS]) if (!robotsAllows(groups, agent, pathWithQuery)) return agent;
+  return null;
+}
+
+/** A bot check, CAPTCHA or login wall instead of the page? Returns the reason or null. */
+export function challengeReason(status, html) {
+  if (status === 401) return "login required (HTTP 401)";
+  if (status === 403 || status === 429) return `blocked (HTTP ${status})`;
+  const head = String(html).slice(0, 20000);
+  if (/g-recaptcha|h-captcha|hcaptcha\.com|cf-challenge|challenge-platform|cf_chl_|captcha-delivery|px-captcha|perimeterx|datadome|Just a moment\.\.\.|Attention Required!|verify you are (a )?human|are you a robot/i.test(head)) return "bot check / CAPTCHA page";
+  if (status === 202 && head.length < 4000) return "interstitial (HTTP 202)";
+  return null;
+}
+
+/** The bodies of <script type="application/ld+json"> elements, found with a linear scan (no backtracking regex). */
+export function jsonLdBlocks(html, max = 50) {
+  const text = String(html);
+  const lower = text.toLowerCase();
+  const out = [];
+  let at = 0;
+  while (out.length < max) {
+    const open = lower.indexOf("<script", at);
+    if (open < 0) break;
+    const tagEnd = lower.indexOf(">", open + 7);
+    if (tagEnd < 0) break;
+    const close = lower.indexOf("</script", tagEnd + 1);
+    if (close < 0) break;
+    if (/\btype\s*=\s*["']?application\/ld\+json/.test(lower.slice(open + 7, tagEnd))) out.push(text.slice(tagEnd + 1, close));
+    at = close + 8;
+  }
+  return out;
+}
+
+/**
+ * The schema.org offers a page publishes (JSON-LD Product / ProductGroup / Offer / AggregateOffer): what the page
+ * itself states as its price, instead of a number read off the layout. At most 5 distinct [amount, currency] pairs; nothing from the page is executed.
+ */
+export function pageOffers(html, max = 5) {
+  const out = [];
+  const seen = new Set();
+  const typesOf = (n) => (Array.isArray(n?.["@type"]) ? n["@type"] : [n?.["@type"]]).filter(Boolean).map(String);
+  const num = (v) => {
+    const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^\d.]/g, "")) : NaN;
+    return Number.isFinite(n) && n > 0 && n < 1e7 ? n : null;
+  };
+  const push = (offer, name) => {
+    if (out.length >= max) return;
+    const currency = typeof offer.priceCurrency === "string" && /^[A-Z]{3}$/.test(offer.priceCurrency.trim()) ? offer.priceCurrency.trim() : null;
+    const spec = Array.isArray(offer.priceSpecification) ? offer.priceSpecification[0] : offer.priceSpecification;
+    const amount = num(offer.price) ?? num(offer.lowPrice) ?? num(spec?.price);
+    if (amount == null) return;
+    const high = num(offer.highPrice);
+    const key = `${amount}|${high ?? ""}|${currency ?? spec?.priceCurrency ?? ""}`;
+    if (seen.has(key)) return;
+    seen.add(key);
+    const availability = typeof offer.availability === "string" ? offer.availability.replace(/^.*[/#]/, "").slice(0, 30) : null;
+    out.push({
+      amount, ...(high && high !== amount ? { high } : {}),
+      currency: currency || (typeof spec?.priceCurrency === "string" ? spec.priceCurrency.slice(0, 3) : null),
+      ...(availability ? { availability } : {}),
+      ...(typeof name === "string" && name.trim() ? { name: name.replace(/\s+/g, " ").trim().slice(0, 80) } : {}),
+    });
+  };
+  const visit = (node, name, depth = 0) => {
+    if (depth > 6 || !node || typeof node !== "object" || out.length >= max) return;
+    if (Array.isArray(node)) return node.forEach((x) => visit(x, name, depth + 1));
+    const t = typesOf(node);
+    const ownName = typeof node.name === "string" ? node.name : name;
+    if (t.some((x) => /^(Offer|AggregateOffer)$/.test(x))) push(node, name);
+    for (const key of ["@graph", "offers", "hasVariant", "itemListElement", "item"]) if (node[key]) visit(node[key], ownName, depth + 1);
+  };
+  for (const text of jsonLdBlocks(html)) {
+    let block;
+    try {
+      block = JSON.parse(text.trim());
+    } catch {
+      continue;
+    }
+    visit(block, null);
+  }
+  return out;
+}
+
+const robotsCache = new Map();
+
+async function robotsGroups(origin, { fetchImpl, allowLocal, timeoutMs }) {
+  if (robotsCache.has(origin)) return robotsCache.get(origin);
+  let result;
+  try {
+    const r = await fetchImpl(`${origin}/robots.txt`, {
+      allowLocal, timeoutMs: Math.min(timeoutMs, 10_000), maxBytes: 500_000, followRedirects: true, answerAnyStatus: true,
+      headers: { "user-agent": USER_AGENT, accept: "text/plain" },
+    });
+    // 4xx = no rules; 5xx = treat as "do not crawl" (the robots.txt standard).
+    if (r.status >= 500) result = { refused: `robots.txt answered HTTP ${r.status} (treated as disallow)` };
+    else result = { groups: r.status < 400 ? parseRobots(await r.text()) : [] };
+  } catch (error) {
+    // a site that does not answer at all is unavailable, not "no rules": the page read stops here (not cached)
+    if (error?.code === "policy" || isUnavailable(error)) throw error;
+    result = { groups: [] };
+  }
+  robotsCache.set(origin, result);
+  return result;
+}
+
+/** The policy check for one URL (robots.txt): { refused: reason | null }. */
+export async function checkPolicy(href, { fetchImpl = pageFetch, allowLocal = false, timeoutMs = 20_000 } = {}) {
+  const u = new URL(href);
+  if (u.protocol !== "http:" && u.protocol !== "https:") return { refused: `not a web URL (${u.protocol})` };
+  const robots = await robotsGroups(u.origin, { fetchImpl, allowLocal, timeoutMs });
+  if (robots.refused) return { refused: robots.refused };
+  const blocker = robotsBlocker(robots.groups, u.pathname + u.search);
+  if (blocker) return { refused: `robots.txt disallows ${u.pathname} for ${blocker === "*" ? "all bots" : blocker}` };
+  return { refused: null };
+}