@opencraw/studio 0.0.30 → 0.0.32

package/dist/index.esm.js

@@ -3,7 +3,7 @@ import { createServer } from 'node:http';
 import { promisify, stripVTControlCharacters } from 'node:util';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { z } from 'zod';
-import { BODY_KINDS, HttpClient, BrowserClient, NODE_ID_ATTRIBUTE, sessionStateOf, rowsOfCells, parseXml, findDeckTables, findGridTables, analyzeTables, parseRegion, regionText, deckRegionText, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, calloutLabel, bindRecipeSet, memorySink, createCrawler, traceLine, UnknownHookError, resolveMissingPolicy, isWorkbookDocument, RecipeBindingError, RecipeValidationError, countMatches, matchesOf, tryParseJson, isPdfDocument, BrowserProfiles } from '@opencraw/core';
+import { BODY_KINDS, formatOfResponse, HttpClient, BrowserClient, NODE_ID_ATTRIBUTE, sessionStateOf, rowsOfCells, parseXml, marginRows, findDeckTables, findGridTables, analyzeTables, parseRegion, regionText, deckRegionText, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, calloutLabel, bindRecipeSet, memorySink, createCrawler, traceLine, UnknownHookError, resolveMissingPolicy, isWorkbookDocument, RecipeBindingError, RecipeValidationError, countMatches, matchesOf, tryParseJson, isPdfDocument, BrowserProfiles } from '@opencraw/core';
 import { randomBytes, createHash } from 'node:crypto';
 import { execFile } from 'node:child_process';
 import { createRequire } from 'node:module';
@@ -52,6 +52,11 @@ const openWorkspaceCommandSchema = z.object({
   type: z.literal('open-workspace'),
   folder: z.string().min(1)
 });
+/** Says what a page address returns, so a recipe is started the right way: a JSON API is read with a request, a page with a browser. */
+const detectUrlCommandSchema = z.object({
+  type: z.literal('detect-url'),
+  url: z.string().min(1)
+});
 /** Says what a path is (folder, recipe or document) and where to open it: how a link or an IDE entry opens a path (issue #235). `workspace` is the folder a document should belong to, when the caller has one. */
 const openPathCommandSchema = z.object({
   type: z.literal('open-path'),
@@ -134,7 +139,9 @@ const inferSelectorCommandSchema = z.object({
   type: z.literal('infer-selector'),
   recipeId: z.string().min(1),
   path: z.string().min(1),
-  nodeIds: z.union([onePickedNodeSchema, twoPickedNodesSchema])
+  nodeIds: z.union([onePickedNodeSchema, twoPickedNodesSchema]),
+  /** The selector of the items of a list the recipe already loops over: a single pick inside one of them answers as a field of the item, to add to the loop. */
+  within: z.string().min(1).optional()
 });
 /**
  * The Inspect panel's DOM tree and data-in-the-page findings, off the same
@@ -354,7 +361,7 @@ const importDocumentCommandSchema = z.object({
   name: z.string().min(1),
   bytes: z.string()
 });
-const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, chooseFolderCommandSchema, openPathCommandSchema, importDocumentCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema, explainWhyCommandSchema, inspectPageCommandSchema, responsesSeenCommandSchema, documentTreeCommandSchema, pdfViewCommandSchema, tablePreviewCommandSchema, regionPreviewCommandSchema, gridViewCommandSchema, gridPreviewCommandSchema, deckViewCommandSchema, deckPreviewCommandSchema, startRecordingCommandSchema, stopRecordingCommandSchema]);
+const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, chooseFolderCommandSchema, openPathCommandSchema, importDocumentCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, detectUrlCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema, explainWhyCommandSchema, inspectPageCommandSchema, responsesSeenCommandSchema, documentTreeCommandSchema, pdfViewCommandSchema, tablePreviewCommandSchema, regionPreviewCommandSchema, gridViewCommandSchema, gridPreviewCommandSchema, deckViewCommandSchema, deckPreviewCommandSchema, startRecordingCommandSchema, stopRecordingCommandSchema]);
 
 /**
  * What `take-snapshot` answers with: the rewritten document ready for the
@@ -368,7 +375,9 @@ const snapshotViewSchema = z.object({
   html: z.string(),
   nodeCount: z.number(),
   baseUrl: z.string(),
-  format: z.enum(BODY_KINDS).optional()
+  format: z.enum(BODY_KINDS).optional(),
+  /** Something to tell the person about the page: it is a bot check or an access-denied page, not the content. */
+  notice: z.string().optional()
 });
 
 /**
@@ -417,6 +426,9 @@ const inferSelectorViewSchema = z.discriminatedUnion('kind', [z.object({
   kind: z.literal('list'),
   item: itemPickSchema,
   field: fieldPickSchema
+}), z.object({
+  kind: z.literal('item-field'),
+  field: fieldPickSchema
 }), z.object({
   kind: z.literal('unsupported'),
   reason: z.string()
@@ -585,6 +597,14 @@ const sampleRunViewSchema = z.object({
   trace: z.array(z.string())
 });
 
+/**
+ * `detect-url`'s answer: what the address returns, so "Load ▾ Web page" starts the right recipe: `html` is a page (opened in a
+ * browser); `json`, `pdf`, `csv`, `xlsx`, `markdown`, `xml`... are documents (read with a request); `unknown` when it could not be fetched.
+ */
+const detectUrlViewSchema = z.object({
+  kind: z.enum(['html', 'json', 'jsonl', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown', 'xml', 'docx', 'unknown'])
+});
+
 /** What `fetch-start-page` answers with. */
 const startPageViewSchema = z.object({
   html: z.string()
@@ -675,7 +695,9 @@ const pdfCellViewSchema = z.object({
 const pdfRowViewSchema = z.object({
   top: z.number(),
   bottom: z.number(),
-  cells: z.array(pdfCellViewSchema)
+  cells: z.array(pdfCellViewSchema),
+  /** A running header or footer: the same text at the same height on several pages (page numbers aside). */
+  running: z.boolean().optional()
 });
 const pdfPageViewSchema = z.object({
   number: z.number().int(),
@@ -1039,6 +1061,72 @@ async function handleChooseFolder(state) {
   };
 }
 
+/** How much of the start of a response is looked at: enough for the engine to tell a PDF or an Office package from its bytes, and for JSON to start. */
+const LOOKED_AT_BYTES = 65_536;
+/** How long to wait for an address to start answering. */
+const TIMEOUT_MS = 20_000;
+/**
+ * What an address returns: the format the engine reads its response as (content type, then what the first bytes
+ * show, then the file extension in the address), and, for an API that serves JSON as `text/plain` (common), JSON when
+ * the text starts like an object or an array. Only the start of the response is read: a large file is not downloaded.
+ * An address that cannot be fetched is `unknown`; the recipe started from it then reports why on its first snapshot.
+ *
+ * @param url - The address.
+ * @returns The kind: `html` is a page to open in a browser, anything else a document to read.
+ */
+async function detectUrlKind(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => {
+    controller.abort();
+  }, TIMEOUT_MS);
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        'user-agent': 'Mozilla/5.0 (compatible; OpenCrawStudio)',
+        'accept': '*/*'
+      }
+    });
+    const bytes = await firstBytes(response);
+    const named = downloadName(response.headers.get('content-disposition'));
+    const where = named === undefined ? response.url === '' ? url : response.url : `https://download.invalid/${named}`;
+    const format = formatOfResponse(response.headers.get('content-type') ?? '', where, bytes, response.ok);
+    if (format === 'text' && /^[[{]/.test(new TextDecoder().decode(bytes).trimStart())) return 'json';
+    return format;
+  } catch {
+    return 'unknown';
+  } finally {
+    clearTimeout(timer);
+    controller.abort();
+  }
+}
+/** The first bytes of a response body; the rest is never downloaded. */
+async function firstBytes(response) {
+  const reader = response.body?.getReader();
+  if (reader === undefined) return new Uint8Array();
+  const chunks = [];
+  let length = 0;
+  while (length < LOOKED_AT_BYTES) {
+    const {
+      done,
+      value
+    } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    length += value.length;
+  }
+  try {
+    await reader.cancel();
+  } catch {
+    // The rest of the body was not wanted anyway.
+  }
+  return Buffer.concat(chunks);
+}
+/** The file name a download is offered under (`Content-Disposition: attachment; filename=series.csv`): an address like `/generator?format=csv` says nothing about it. */
+function downloadName(disposition) {
+  return disposition === null ? undefined : /filename=(?:UTF-8'')?"?([^";]+)"?/i.exec(disposition)?.[1]?.trim();
+}
+
 const execFileAsync = promisify(execFile);
 /** `playwright`'s own `exports` map doesn't whitelist a `./cli` subpath, so its CLI is resolved off `playwright/package.json`'s real location — the same technique `@opencraw/captcha-tesseract`'s `bundledLangPath()` uses for a bundled data path. */
 function playwrightCliPath() {
@@ -1198,6 +1286,102 @@ function textOf$2(body) {
   return `[${body.kind} document: the content pane shows HTML and JSON responses for now]`;
 }
 
+/** What a bot-protection or access-denied page says about itself, in the words the common services use. */
+const BLOCKED_PATTERNS = [/performing security verification/i, /verify(?:ing)? (?:that )?you are (?:a )?human/i, /are you a robot/i, /checking your browser/i, /enable javascript and cookies to continue/i, /just a moment\.\.\./i, /attention required/i, /access denied/i, /request (?:was )?blocked/i, /unusual traffic/i, /pardon our interruption/i];
+/**
+ * Whether a page is the site's bot check or an access-denied page instead of its content, and, if so, what to tell the person: the
+ * Studio's browser was turned away, so everything else on the screen is not the page they asked for.
+ *
+ * @param text - The page's title and the start of its visible text.
+ * @param status - The HTTP status the page came with, when known.
+ * @returns A sentence for the person, or `undefined` when the page looks like a page.
+ */
+function blockedPageNotice(text, status) {
+  const turnedAway = status === 403 || status === 429;
+  if (!turnedAway && BLOCKED_PATTERNS.every(pattern => !pattern.test(text))) return undefined;
+  if (turnedAway && text.trim().length > 4000) return undefined;
+  return 'This site is showing a security check or an access-denied page instead of its content: it has turned the Studio\'s browser away. Try again later, or look for the data the site itself offers (a download, a feed or an API).';
+}
+
+/** The attribute a pop-up that covers the page is marked with in the snapshot; the picture leaves it out unless "Show hidden" is on. */
+/**
+ * A script that, in a page, finds a cookie or consent banner's button and presses it: "Reject all", "Only necessary cookies" and the
+ * like first, else "Accept all", "I agree"... in English, Portuguese, Spanish, French, Italian, German and Dutch, and by the ids the common
+ * consent tools give their buttons (OneTrust, Didomi, Cookiebot, Usercentrics, Osano, iubenda). A banner that is left in place covers the
+ * page in the Studio's view of it, and the Studio's own clicks (to pick a value) would land on it. Only buttons whose whole text is one
+ * of these are pressed, so a "Cookie settings" or "Gerir os cookies" button, a link to a policy and the page's own buttons are left alone.
+ * The answer is the text of the button pressed, or an empty string when there was none.
+ *
+ * It only ever runs in the browser the Studio captures the page with, never in a crawl.
+ */
+const DISMISS_CONSENT_SCRIPT = String.raw`(() => {
+  const rejectIds = ['#onetrust-reject-all-handler', '#didomi-notice-disagree-button', '#CybotCookiebotDialogBodyButtonDecline', '[data-testid=uc-deny-all-button]', '.osano-cm-denyAll', '.iubenda-cs-reject-btn', '.cmp-button_button--reject', '#truste-consent-required', 'button[id*=reject i]', 'button[id*=decline i]']
+  const acceptIds = ['#onetrust-accept-btn-handler', '#didomi-notice-agree-button', '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll', '[data-testid=uc-accept-all-button]', '.osano-cm-accept-all', '.iubenda-cs-accept-btn', 'button[id*=accept i]']
+  const rejectWords = ['reject', 'decline', 'deny', 'refuse', 'rejeitar', 'recusar', 'nao aceitar', 'rechazar', 'refuser', 'rifiuta', 'rifiutare', 'ablehnen', 'weigeren', 'only necessary', 'only essential', 'necessary only', 'essential only', 'apenas necessarios', 'apenas essenciais', 'somente necessarios', 'solo necesarias', 'solo necessari', 'uniquement necessaires', 'uniquement essentiels', 'nur notwendige', 'nur erforderliche', 'alleen noodzakelijke', 'continue without accepting', 'continuar sem aceitar', 'continuar sin aceptar', 'continuer sans accepter', 'continua senza accettare', 'no thanks', 'nao obrigado']
+  const acceptWords = ['accept', 'allow', 'agree', 'i agree', 'i accept', 'ok', 'okay', 'got it', 'yes i agree', 'yes these cookies are ok', 'aceitar', 'aceito', 'concordo', 'permitir', 'aceptar', 'acepto', 'permitir todas', 'accepter', 'j accepte', 'accetta', 'accetto', 'akzeptieren', 'zustimmen', 'einverstanden', 'accepteren']
+  const tail = '(?:\\s+(?:all|todos|todas|todo|tudo|tout|tous|tutti|tutte|alle|alles|os|as|las|los|les|i|le|the|die))*(?:\\s+(?:cookies?|kekse|koekjes))?'
+  const rejectText = new RegExp('^(?:' + rejectWords.join('|') + ')' + tail + '$')
+  const acceptText = new RegExp('^(?:' + acceptWords.join('|') + ')' + tail + '$')
+  const visible = (element) => {
+    const box = element.getBoundingClientRect()
+    const style = getComputedStyle(element)
+
+    return box.width > 0 && box.height > 0 && style.visibility !== 'hidden' && style.display !== 'none'
+  }
+  const plain = (text) => text.toLowerCase().normalize('NFD').replace(/[̀-ͯ]/g, '').replace(/[^a-z\s]/g, ' ').replace(/\s+/g, ' ').trim()
+  const label = (element) => (element.innerText || element.value || element.getAttribute('aria-label') || '').trim().replace(/\s+/g, ' ')
+  const byId = (selectors) => {
+    for (const selector of selectors) {
+      const found = [...document.querySelectorAll(selector)].find(visible)
+      if (found !== undefined) return found
+    }
+
+    return undefined
+  }
+  const candidates = [...document.querySelectorAll('button, a, [role=button], input[type=button], input[type=submit]')].filter(visible)
+  const byText = (pattern) => candidates.find((element) => pattern.test(plain(label(element))))
+  const target = byId(rejectIds) ?? byText(rejectText) ?? byId(acceptIds) ?? byText(acceptText)
+  if (target === undefined) return ''
+  const text = label(target)
+  target.click()
+
+  return text || target.id
+})()`;
+/**
+ * A script that marks, in the page, every pop-up that covers it: something fixed to the window that fills most of it (a dimming backdrop,
+ * a modal), a modal dialog, or a banner a consent tool put there. The picture of the page leaves them out (the page behind is what the
+ * person wants to pick from), and "Show hidden" brings them back. Run after the consent button has been tried: a banner that went away
+ * needs nothing, one that stayed (its button in a form the script does not know, an icon with no text) is covered here.
+ *
+ * Only the outermost element of an overlay is marked, since hiding it hides all it holds.
+ */
+const MARK_OVERLAYS_SCRIPT = `(() => {
+  const attribute = 'data-oc-overlay'
+  const consentName = /cookie|consent|gdpr|privacy|onetrust|ot-sdk|didomi|cookiebot|usercentrics|trustarc|truste|osano|iubenda|cmp|banner-modal/i
+  const width = window.innerWidth
+  const height = window.innerHeight
+  const marked = []
+  for (const element of document.querySelectorAll('body *')) {
+    if (marked.some((outer) => outer.contains(element))) continue
+    const style = getComputedStyle(element)
+    if (style.position !== 'fixed' || style.display === 'none' || style.visibility === 'hidden') continue
+    const box = element.getBoundingClientRect()
+    if (box.width === 0 || box.height === 0) continue
+    const covers = box.width >= width * 0.6 && box.height >= height * 0.5
+    const modal = element.getAttribute('aria-modal') === 'true' || element.getAttribute('role') === 'dialog' || element.getAttribute('role') === 'alertdialog'
+    const named = consentName.test((element.id || '') + ' ' + (typeof element.className === 'string' ? element.className : ''))
+    // A page built as one fixed, scrolling frame is the page, not a pop-up on it.
+    if (covers && !modal && !named && (element.innerText || '').length > document.body.innerText.length * 0.6) continue
+    // A bar along an edge (a site's own header or footer) is not a pop-up unless a consent tool named it.
+    if (covers || modal || named) {
+      element.setAttribute(attribute, '1')
+      marked.push(element)
+    }
+  }
+
+  return marked.length
+})()`;
+
 /** The attribute `hiddenMarksScript` stamps on an element hidden on the live page. */
 const HIDDEN_ATTRIBUTE = 'data-oc-hidden';
 /**
@@ -1233,6 +1417,79 @@ function hiddenMarksScript(attribute = HIDDEN_ATTRIBUTE) {
   })()`;
 }
 
+const FONT_TYPES = {
+  woff2: 'font/woff2',
+  woff: 'font/woff',
+  ttf: 'font/ttf',
+  otf: 'font/otf',
+  eot: 'application/vnd.ms-fontobject',
+  svg: 'image/svg+xml'
+};
+/** `url(...)` in a stylesheet, quoted or not. */
+const CSS_URL = /url\(\s*(['"]?)([^'"\s)]+)\1\s*\)/g;
+/** The address a `url()` in a stylesheet at `base` points to, or `undefined` for one that is not an address (inline data, a fragment). */
+function resolved(reference, base) {
+  if (reference.startsWith('data:') || reference.startsWith('#')) return undefined;
+  try {
+    return new URL(reference, base).href;
+  } catch {
+    return undefined;
+  }
+}
+function fontType(address) {
+  const extension = /\.([a-z0-9]+)(?:[?#]|$)/i.exec(new URL(address).pathname + new URL(address).search)?.[1]?.toLowerCase();
+  return (extension === undefined ? undefined : FONT_TYPES[extension]) ?? 'application/octet-stream';
+}
+/**
+ * A stylesheet's text with every `url(...)` made absolute, and each font the page loaded embedded as a data address. The picture of a page
+ * is shown in a frame that is not the site, so a font file is a cross-site request the site's server does not allow: its text comes
+ * out in a fallback face and its icons (a menu, a close button) as empty boxes. A font inside the page cannot be refused.
+ *
+ * @param css - The stylesheet text.
+ * @param base - Where the stylesheet came from (or the page, for a `<style>` in it).
+ * @param assets - What the page loaded, by address.
+ * @returns The stylesheet, ready to sit inside the snapshot.
+ */
+function embedFonts(css, base, assets) {
+  // eslint-disable-next-line unicorn/prefer-string-replace-all -- one replacer with a global pattern
+  return css.replace(CSS_URL, (whole, _quote, reference) => {
+    const address = resolved(reference.trim(), base);
+    if (address === undefined) return whole;
+    const asset = assets.get(address);
+    if (asset?.kind !== 'font') return `url("${address}")`;
+    return `url("data:${fontType(address)};base64,${Buffer.from(asset.body).toString('base64')}")`;
+  });
+}
+/**
+ * Turns a captured page into one that carries its own styles: every `<link rel="stylesheet">` the page loaded becomes a `<style>` with
+ * the stylesheet's text, and the fonts in it (and in the page's own `<style>`s) are embedded. A stylesheet that was not captured is left
+ * as the link it was.
+ *
+ * @param html - The page as captured.
+ * @param pageUrl - The page's address.
+ * @param assets - The stylesheets and fonts the page loaded, by address.
+ * @returns The page with its styles and fonts inside it.
+ */
+function inlineAssets(html, pageUrl, assets) {
+  const $ = load(html);
+  $('link[rel~="stylesheet"][href]').each((_index, element) => {
+    const reference = $(element).attr('href') ?? '';
+    const address = resolved(reference, pageUrl);
+    const asset = address === undefined ? undefined : assets.get(address);
+    if (address === undefined || asset?.kind !== 'css') return;
+    const css = embedFonts(Buffer.from(asset.body).toString('utf8'), address, assets);
+    const inlined = $('<style></style>').attr('data-oc-from', address);
+    inlined.text(css);
+    $(element).replaceWith(inlined);
+  });
+  $('style').each((_index, element) => {
+    if ($(element).attr('data-oc-from') !== undefined) return;
+    const css = $(element).text();
+    if (css.includes('url(')) $(element).text(embedFonts(css, pageUrl, assets));
+  });
+  return $.html();
+}
+
 /** Attributes that carry a URL and are rewritten to absolute; `srcset` is handled separately (it carries several). */
 const URL_ATTRIBUTES = ['href', 'src', 'action', 'poster'];
 /** Schemes left untouched: they are not a location `new URL(value, base)` should resolve, or already carry their own meaning. */
@@ -1312,6 +1569,11 @@ function escapeAttribute(value) {
   return value.replaceAll('&', '&amp;').replaceAll('"', '&quot;');
 }
 
+/** How long a cookie banner is given to go away once its button is pressed. */
+const CONSENT_SETTLE_MS = 1200;
+/** A banner appears a moment after the page loads (some only after its scripts have run for a few seconds): it is looked for this many times, this far apart, and a page with none costs this much time. */
+const CONSENT_ATTEMPTS = 6;
+const CONSENT_RETRY_MS = 800;
 /**
  * Captures the snapshot the content pane shows (studio plan §3.1, issue
  * #91): runs the recipe up to `stepPath`, serialises the document, marks
@@ -1341,7 +1603,7 @@ async function takeSnapshot(input, stepPath, browser, hooks) {
   const point = input.start[0];
   if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
   const storageState = await loginState(input, browser, hooks);
-  return input.mode === 'web' ? snapshotWeb(point.url, storageState, browser) : snapshotApi(point.url, storageState);
+  return input.mode === 'web' ? snapshotWeb(point.url, storageState, browser) : snapshotApi(point.url, storageState, requestedAs(input));
 }
 /** What the recipe's own login leaves behind, so the snapshot shows the page as the crawl sees it (issue #185); nothing when it declares no login. */
 async function loginState(input, browser, hooks) {
@@ -1366,15 +1628,34 @@ async function snapshotWeb(url, storageState, browser) {
       storageState
     });
     try {
-      await session.page.goto(url);
+      const assets = new Map();
+      const captures = [];
+      const budget = {
+        bytes: 0
+      };
+      session.page.on('response', loaded => {
+        captures.push(captureAsset(loaded, assets, budget));
+      });
+      const response = await session.page.goto(url);
+      // A cookie banner left in place covers the page in this view, and the clicks that pick a value would land on it.
+      await dismissConsent(session.page);
+      // What still covers the page after that (a modal whose button the script does not know) is left out of the picture: "Show hidden" brings it back.
+      await session.page.evaluate(MARK_OVERLAYS_SCRIPT);
+      const seen = String(await session.page.evaluate(String.raw`document.title + "\n" + (document.body ? document.body.innerText : "").slice(0, 3000)`));
       await session.page.evaluate(hiddenMarksScript(HIDDEN_ATTRIBUTE));
-      const html = await session.page.content();
+      await Promise.all(captures);
       const baseUrl = session.page.url();
+      // The page's own styles and fonts go inside the picture: the frame that shows it is not the site, so the site's fonts would be refused there.
+      const html = inlineAssets(await session.page.content(), baseUrl, assets);
       const rewritten = rewriteDocument(html, baseUrl);
+      const notice = blockedPageNotice(seen, response?.status());
       return {
         ...rewritten,
         baseUrl,
-        rawHtml: html
+        rawHtml: html,
+        ...(notice !== undefined && {
+          notice
+        })
       };
     } finally {
       await session.close();
@@ -1383,13 +1664,63 @@ async function snapshotWeb(url, storageState, browser) {
     await client.close();
   }
 }
-async function snapshotApi(url, storageState) {
+/** The most a stylesheet may weigh to be kept, the most a font may, and how much of both a page may bring (bytes): a picture is not worth a heavier page than this. */
+const MAX_CSS_BYTES = 800_000;
+const MAX_FONT_BYTES = 600_000;
+const MAX_ASSET_BUDGET = 6_000_000;
+/** Keeps a stylesheet or a font the page loaded, within the limits above; anything else, or anything that cannot be read, is let go. */
+async function captureAsset(loaded, assets, budget) {
+  const type = loaded.request().resourceType();
+  if (type !== 'stylesheet' && type !== 'font') return;
+  try {
+    const body = await loaded.body();
+    if (body.length > (type === 'font' ? MAX_FONT_BYTES : MAX_CSS_BYTES) || budget.bytes + body.length > MAX_ASSET_BUDGET) return;
+    budget.bytes += body.length;
+    assets.set(loaded.url(), {
+      kind: type === 'font' ? 'font' : 'css',
+      body
+    });
+  } catch {
+    // A response that went away before it could be read is not kept.
+  }
+}
+/** How the recipe's own first request reads the document (`as`), so a JSON API served as `text/plain` is shown as JSON here too. */
+function requestedAs(input) {
+  const request = input.steps.find(step => step.type === 'request');
+  return request?.type === 'request' ? request.as : undefined;
+}
+/**
+ * Presses the cookie banner's reject (or accept) button, if the page shows one. Many sites put their consent dialog in a frame of its
+ * own (a separate document, often covering the whole window), so every frame is searched, not only the page. The page is looked at
+ * a few times, since banners come late.
+ */
+async function dismissConsent(page) {
+  for (let attempt = 0; attempt < CONSENT_ATTEMPTS; attempt += 1) {
+    for (const frame of page.frames()) {
+      let pressed = '';
+      try {
+        pressed = String(await frame.evaluate(DISMISS_CONSENT_SCRIPT));
+      } catch {
+        // A frame that went away or is not ready has no banner to press.
+      }
+      if (pressed !== '') {
+        await page.waitForTimeout(CONSENT_SETTLE_MS);
+        return;
+      }
+    }
+    await page.waitForTimeout(CONSENT_RETRY_MS);
+  }
+}
+async function snapshotApi(url, storageState, as) {
   const client = await HttpClient.open({
     storageState
   });
   try {
     const response = await client.send({
-      url
+      url,
+      ...(as !== undefined && {
+        as
+      })
     });
     const text = textOf$1(response.body);
     const rewritten = rewriteDocument(text, url);
@@ -1918,6 +2249,7 @@ function textOf(element) {
  * @returns The view.
  */
 function pdfDocumentView(document) {
+  const margins = marginRows(document);
   return {
     pages: document.pages.map(page => ({
       number: page.number,
@@ -1932,7 +2264,10 @@ function pdfDocumentView(document) {
           width: cell.width,
           height: cell.height,
           text: cell.text
-        }))
+        })),
+        ...(margins.has(row) && {
+          running: true
+        })
       })),
       rowCount: page.rows.length,
       cellCount: page.rows.reduce((total, row) => total + row.cells.length, 0),
@@ -3053,7 +3388,7 @@ function nodeToStep(node) {
  */
 async function openWorkspace(folder) {
   const documents = await readRecipeSource(folder);
-  const entries = documents.map(document => parseDocument$1(document));
+  const entries = await Promise.all(documents.map(async document => parseDocument$1(document, await storedContent(document))));
   const outputs = entries.filter(isParsed('output'));
   const inputs = entries.filter(isParsed('input'));
   for (const output of outputs) bindOutput(output, inputs);
@@ -3073,16 +3408,16 @@ async function openWorkspace(folder) {
       kind,
       id,
       issues,
-      content
+      stored
     }) => ({
       file,
       kind,
       id,
       issues,
-      text: `${JSON.stringify(content, null, 2)}\n`,
+      text: `${JSON.stringify(stored, null, 2)}\n`,
       // The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
       // throws, so this covers a file that fails validation too (its outline just has "custom" cards).
-      outline: kind === 'input' ? recipeToOutline(content) : undefined
+      outline: kind === 'input' ? recipeToOutline(stored) : undefined
     }))
   };
 }
@@ -3106,14 +3441,24 @@ function bindOutput(output, inputs) {
 function isParsed(kind) {
   return entry => entry.kind === kind && entry.recipe !== undefined;
 }
-function parseDocument$1(document) {
+/** The recipe as its `.json` file holds it; the decoded content for anything else (a JSON Lines line, text, an object), or a file that cannot be read back. */
+async function storedContent(document) {
+  if (!document.source.endsWith('.json')) return document.content;
+  try {
+    return JSON.parse(await readFile(document.source, 'utf8'));
+  } catch {
+    return document.content;
+  }
+}
+function parseDocument$1(document, stored) {
   const kind = document.content?.kind;
-  if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
-  if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
+  if (kind === 'output') return parseOne(document.source, document.content, stored, 'output', parseOutputRecipe);
+  if (kind === 'input') return parseOne(document.source, document.content, stored, 'input', parseInputRecipe);
   return {
     file: document.source,
     kind: 'unknown',
     content: document.content,
+    stored,
     issues: [{
       path: 'kind',
       message: '"kind" is missing or not "input"/"output"',
@@ -3121,13 +3466,14 @@ function parseDocument$1(document) {
     }]
   };
 }
-function parseOne(file, content, kind, parse) {
+function parseOne(file, content, stored, kind, parse) {
   try {
     const recipe = parse(content, file);
     return {
       file,
       kind,
       content,
+      stored,
       id: recipe.id,
       issues: [],
       recipe
@@ -3138,6 +3484,7 @@ function parseOne(file, content, kind, parse) {
       file,
       kind,
       content,
+      stored,
       issues: error.issues.map(issue => ({
         path: issue.path === '' ? '(root)' : issue.path,
         message: issue.message,
@@ -3201,8 +3548,10 @@ const MAX_NUMBERED_COPIES = 100;
  * `file:` URL the same way a run would read any downloaded file (core's
  * `HttpClient` handles `file:` itself). The folder is created when missing,
  * so dropping a file is also a way to start a workspace. A name already in
- * the folder is never overwritten: the copy gets the next free numbered
- * name (`report-2.pdf`), and the returned `path`/`url` say which.
+ * the folder is never overwritten: a file there with the very same bytes is
+ * reused (loading one document twice leaves one copy), a different one gets
+ * the next free numbered name (`report-2.pdf`); the returned `path`/`url`
+ * say which.
  *
  * @param folder - The workspace folder; created (recursively) when it does not exist.
  * @param name - The file's own name, no directories: the copy keeps it, extension included.
@@ -3223,10 +3572,14 @@ async function importDocument(folder, name, bytes) {
       });
       return {
         path,
-        url: pathToFileURL(path).href
+        url: `file:${encodeURIComponent(candidate)}`
       };
     } catch (error) {
       if (!isAlreadyThere(error)) throw error;
+      if (await hasBytes(path, bytes)) return {
+        path,
+        url: `file:${encodeURIComponent(candidate)}`
+      };
     }
   }
   throw new Error(`"${name}" and ${MAX_NUMBERED_COPIES} numbered copies of it are already in ${folder}`);
@@ -3242,6 +3595,11 @@ function* candidateNames(name) {
   const stem = name.slice(0, name.length - extension.length);
   for (let copy = 2; copy <= MAX_NUMBERED_COPIES + 1; copy += 1) yield `${stem}-${copy}${extension}`;
 }
+/** Whether the file at `path` is, byte for byte, `bytes`. */
+async function hasBytes(path, bytes) {
+  const existing = await readFile(path);
+  return existing.length === bytes.length && existing.equals(bytes);
+}
 function isAlreadyThere(error) {
   return typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST';
 }
@@ -3495,7 +3853,10 @@ const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`,
 /^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
 /^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
 /^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
-/^_[0-9a-z]{5,}$/i];
+/^_[0-9a-z]{5,}$/i, /** A framework's runtime state, added and removed as the page runs, not written by anyone: AngularJS/Angular `ng-scope`, `ng-binding`, `ng-star-inserted`, `ng-valid`… */
+/^ng-/, /** Ember's: `ember-view`, `ember123` */
+/^ember(?:-view|\d+)$/, /** Svelte's scoping hash: `svelte-1a2b3c` */
+/^svelte-[a-z0-9]+$/i];
 /**
  * Whether a class token looks hand-authored and safe to build a selector on.
  *
@@ -3802,7 +4163,7 @@ function handleInferSelector(state, command) {
     reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
   };
   const [pathA, pathB] = paths;
-  if (pathB === undefined) return {
+  if (pathB === undefined) return itemField(html, command, pathA) ?? {
     kind: 'field',
     field: fieldPick(html, pathA)
   };
@@ -3835,6 +4196,26 @@ function handleInferSelector(state, command) {
     throw error;
   }
 }
+/**
+ * A single pick inside an item of the list the recipe already loops over (`command.within` is the items' selector): the field read from the item,
+ * for the loop's body. `undefined` when no list is given, or the node is not inside one of its items (or is the item itself).
+ */
+function itemField(html, command, clicked) {
+  if (command.within === undefined) return undefined;
+  const $ = load(html);
+  let itemId;
+  try {
+    itemId = $(`[${NODE_ID_ATTRIBUTE}="${command.nodeIds[0]}"]`).first().closest(command.within).attr(NODE_ID_ATTRIBUTE);
+  } catch {
+    return undefined;
+  }
+  const itemPath = itemId === undefined ? undefined : pathToNode(html, itemId);
+  if (itemPath === undefined || itemPath.length >= clicked.length) return undefined;
+  return {
+    kind: 'item-field',
+    field: fieldPick(html, clicked.slice(itemPath.length - 1))
+  };
+}
 /** The best selector for one clicked node, `take` decided from the concrete clicked element; `rankSource` (a generalised path) is used for candidate generation when given, so a per-item-unique attribute of the one node picked cannot end up in the selector. */
 function fieldPick(html, clicked, rankSource = clicked) {
   const leaf = clicked.at(-1);
@@ -3932,13 +4313,28 @@ function collapseRepeats(nodes) {
   }
   return result;
 }
-/** A run member's identity for collapsing: everything but its own `nodeId` (every instance has a different one by construction). */
+/** Signatures already worked out: a node is looked at once for its run and again as part of its parent's. */
+const signatures = new WeakMap();
+/**
+ * A run member's identity for collapsing: everything but its own `nodeId` (every instance has a different one by
+ * construction). Its children enter as their own short hashes, not as the text of their signatures: nesting that
+ * text in JSON escapes it again at every level, which grows with the depth of the page until the string is too
+ * long to exist (a page nested 40 deep).
+ */
 function signatureOf(node) {
-  return JSON.stringify({
-    ...node,
+  const known = signatures.get(node);
+  if (known !== undefined) return known;
+  const {
+    children,
+    ...own
+  } = node;
+  const signature = createHash('sha1').update(JSON.stringify({
+    ...own,
     nodeId: undefined,
-    children: node.children.map(child => signatureOf(child))
-  });
+    children: children.map(child => signatureOf(child))
+  })).digest('hex');
+  signatures.set(node, signature);
+  return signature;
 }
 function abbreviate(text) {
   const trimmed = text.trim().replaceAll(/\s+/g, ' ');
@@ -4176,6 +4572,19 @@ function handleInspectPage(state, command) {
   };
 }
 
+/**
+ * Handles `detect-url`: what the address returns, for "Load ▾ Web page" to start a recipe that reads a document (a JSON API, a PDF,
+ * a spreadsheet, Markdown...) instead of a browser recipe over a response that is not a page.
+ *
+ * @param command - The address.
+ * @returns What it returns.
+ */
+async function handleDetectUrl(command) {
+  return {
+    kind: await detectUrlKind(command.url)
+  };
+}
+
 /**
  * Handles `open-path`: says what the path is (folder, recipe or document) and
  * where to open it. Opens nothing itself: the UI follows with `open-workspace`
@@ -5321,6 +5730,9 @@ function snapshotView(snapshot) {
     baseUrl: snapshot.baseUrl,
     ...(snapshot.format !== undefined && {
       format: snapshot.format
+    }),
+    ...(snapshot.notice !== undefined && {
+      notice: snapshot.notice
     })
   };
 }
@@ -5638,6 +6050,10 @@ function dispatch(command, state) {
       {
         return handleFetchStartPage(state, command);
       }
+    case 'detect-url':
+      {
+        return handleDetectUrl(command);
+      }
     case 'save-outline':
       {
         return handleSaveOutline(state, command);
@@ -5773,5 +6189,5 @@ async function main(argv, options = {}) {
   process.on('SIGTERM', shutdown);
 }
 
-export { chooseFolderCommandSchema, chooseFolderViewSchema, deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, importDocument, importDocumentCommandSchema, importDocumentViewSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, loadedHooksSchema, main, missingWhyTargetSchema, observedResponseSchema, openPathCommandSchema, openPathViewSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, regionPreviewCommandSchema, regionPreviewMatchSchema, regionPreviewViewSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
+export { chooseFolderCommandSchema, chooseFolderViewSchema, deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, detectUrlCommandSchema, detectUrlViewSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, importDocument, importDocumentCommandSchema, importDocumentViewSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, loadedHooksSchema, main, missingWhyTargetSchema, observedResponseSchema, openPathCommandSchema, openPathViewSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, regionPreviewCommandSchema, regionPreviewMatchSchema, regionPreviewViewSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
 //# sourceMappingURL=index.esm.js.map