octocrawl 0.3.2 → 0.4.0

package/dist/cli.js

@@ -3829,7 +3829,7 @@ function classifyGate(res) {
     return { reason: "captcha", signals: px };
   const wall = res.status >= 200 && res.status < 300 ? shortWallPage(res.body) : null;
   if (contentful)
-    return wall === null ? null : { reason: "bot_detected_generic", signals: wall };
+    return wall;
   const captcha = matched(lower, CAPTCHA_MARKERS);
   if (captcha.length > 0) {
     return { reason: "captcha", signals: captcha };
@@ -3845,8 +3845,6 @@ function classifyGate(res) {
       signals.push("heading_sign_in");
     return { reason: "login_wall", signals };
   }
-  if (wall !== null)
-    return { reason: "bot_detected_generic", signals: wall };
   const geo = matched(lower, GEO_MARKERS);
   if (geo.length > 0) {
     return { reason: "geo_restricted", signals: geo };
@@ -3873,18 +3871,29 @@ function classifyGate(res) {
       };
     }
   }
-  return null;
+  return wall;
 }
 function shortWallPage(body) {
-  if (!/\b(?:reference|incident|unusual|suspicious)\b/i.test(body))
+  if (!WALL_TRIGGER.test(body))
     return null;
   const page = pageText(body);
   if (page === null)
     return null;
+  const bot = (signals) => ({ reason: "bot_detected_generic", signals });
   if (AKAMAI_REFERENCE.test(page.text))
-    return ["akamai_reference", "short_page"];
+    return bot(["akamai_reference", "short_page"]);
   if (page.headings.some((h) => UNUSUAL_ACTIVITY.test(h)) && AUTOMATED_TRAFFIC.test(page.text))
-    return ["heading_unusual_activity", "text_automated_traffic", "short_page"];
+    return bot(["heading_unusual_activity", "text_automated_traffic", "short_page"]);
+  const shown = page.text.toLowerCase();
+  const named = page.headings.join("\n").toLowerCase();
+  const strong = matched(named, BOT_STRONG_MARKERS);
+  if (strong.length > 0) {
+    const person = shown.includes("activate and hold the button") ? ["text_activate_and_hold"] : [];
+    return { reason: person.length > 0 ? "captcha" : "bot_detected_generic", signals: [...strong, ...person, "short_page"] };
+  }
+  const weak = matched(shown, WALL_WEAK_MARKERS);
+  if (weak.length >= 2 && matched(named, WALL_WEAK_MARKERS).length > 0)
+    return bot([...weak, "short_page"]);
   return null;
 }
 function pageText(body) {
@@ -4002,7 +4011,7 @@ function nextLane(reason2, from) {
       return from === "browser_proxy" ? null : "browser_proxy";
   }
 }
-var HEAD_BYTES, VENDOR_HEADERS, CF_STRONG_MARKERS, CF_INTERSTITIAL_PAIR, CAPTCHA_MARKERS, LOGIN_INTENT_PHRASES, PASSWORD_INPUT, TITLE_OR_H1, BARE_LOGIN_HEADING, GEO_MARKERS, BOT_STRONG_MARKERS, BOT_WEAK_MARKERS, GATE_SHAPED_STATUS, CF_JSD_PATH, PX_CAPTCHA_SCRIPT, PX_CAPTCHA_CONTAINER, WALL_PAGE_MAX_TEXT, AKAMAI_REFERENCE, UNUSUAL_ACTIVITY, AUTOMATED_TRAFFIC, HIDDEN_ELEMENTS, HEADING_ELEMENTS;
+var HEAD_BYTES, VENDOR_HEADERS, CF_STRONG_MARKERS, CF_INTERSTITIAL_PAIR, CAPTCHA_MARKERS, LOGIN_INTENT_PHRASES, PASSWORD_INPUT, TITLE_OR_H1, BARE_LOGIN_HEADING, GEO_MARKERS, BOT_STRONG_MARKERS, BOT_WEAK_MARKERS, GATE_SHAPED_STATUS, CF_JSD_PATH, PX_CAPTCHA_SCRIPT, PX_CAPTCHA_CONTAINER, WALL_PAGE_MAX_TEXT, AKAMAI_REFERENCE, UNUSUAL_ACTIVITY, WALL_TRIGGER, WALL_WEAK_MARKERS, AUTOMATED_TRAFFIC, HIDDEN_ELEMENTS, HEADING_ELEMENTS;
 var init_gate = __esm({
   "packages/http-core/dist/gate.js"() {
     "use strict";
@@ -4096,6 +4105,8 @@ var init_gate = __esm({
     WALL_PAGE_MAX_TEXT = 1500;
     AKAMAI_REFERENCE = /\b(?:reference|incident)\b\s*(?:number|id|no\.?)?\s*[:#]?\s*#?\s*\d{1,3}\.[0-9a-f]{6,8}\.\d{10}\.[0-9a-f]{6,12}\b/i;
     UNUSUAL_ACTIVITY = /\b(?:unusual|suspicious) (?:activity|traffic)\b/i;
+    WALL_TRIGGER = /\b(?:reference|incident|unusual|suspicious|blocked|interruption|automated|robot|human|humans|bot|checking|wait|moment|security|ddos|denied)\b/i;
+    WALL_WEAK_MARKERS = BOT_WEAK_MARKERS.filter(([, signal]) => signal !== "weak_enable_js_and_cookies" && signal !== "weak_enable_javascript");
     AUTOMATED_TRAFFIC = /\bautomated (?:traffic|access|requests|queries|browsing)\b/i;
     HIDDEN_ELEMENTS = /* @__PURE__ */ new Set(["script", "style", "noscript", "template", "svg"]);
     HEADING_ELEMENTS = /* @__PURE__ */ new Set(["h1", "h2", "h3", "title"]);
@@ -8738,6 +8749,26 @@ function rawSignals(doc) {
     markers.add("aria_busy");
   return { scriptChars, markers: Array.from(markers), hydrationJson };
 }
+function hasLoadingText(region) {
+  const stack = [region];
+  while (stack.length > 0) {
+    const node = stack.pop();
+    if (node.nodeType === 3) {
+      const text2 = (node.textContent ?? "").trim();
+      if (text2.length === 0 || text2.length > LOADING_TEXT_MAX_CHARS || !(LOADING_MESSAGE.test(text2) || LOADING_TRAILING.test(text2)))
+        continue;
+      if (node.parentElement?.closest("button, a") == null)
+        return true;
+    } else if (node.nodeType === 1) {
+      const el = node;
+      if (el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true" || SCREEN_READER_ONLY.test(el.getAttribute("class") ?? ""))
+        continue;
+      for (const child of Array.from(el.childNodes))
+        stack.push(child);
+    }
+  }
+  return false;
+}
 function countEmptyTables(doc) {
   let empty = 0;
   for (const table of qsa(doc, "table")) {
@@ -8778,6 +8809,8 @@ function detectRenderSignals(raw, cleaned, context = {}) {
     reason2 = "hydration_shell";
   else if (has2("aria_busy") && raw.scriptChars >= 1e3)
     reason2 = "aria_busy";
+  else if (context.loadingShown === true && textChars <= LOADING_PAGE_MAX_TEXT && raw.scriptChars >= 1e3)
+    reason2 = "loading_text";
   const listRecords = reason2 === null && context.listing === true && raw.hydrationJson.length > 0 ? partialRecordList(recordLists(raw), visible.toLowerCase()) : null;
   if (listRecords !== null)
     reason2 = "hydration_list_partial";
@@ -8791,14 +8824,14 @@ function detectRenderSignals(raw, cleaned, context = {}) {
     reason: reason2
   };
 }
-var APP_ROOT_SELECTOR, JS_FALLBACK_CLASS, HYDRATION_SCRIPT, APP_ROOT_MAX_TEXT, HYDRATION_MIN_CHARS, NOTICE_MAX_TEXT, HYDRATION_TEXT_MIN_CHARS, HYDRATION_TEXT_MIN_SPACES, HYDRATION_MAX_JSON_CHARS, HYDRATION_MAX_PASSAGES, LIST_MIN_RECORDS, LIST_NAME_MIN_CHARS, LIST_MIN_SHOWN, LIST_MAX_NAMES, LIST_MAX_LISTS;
+var APP_ROOT_SELECTOR, JS_FALLBACK_CLASS, HYDRATION_SCRIPT, APP_ROOT_MAX_TEXT, HYDRATION_MIN_CHARS, NOTICE_MAX_TEXT, HYDRATION_TEXT_MIN_CHARS, HYDRATION_TEXT_MIN_SPACES, HYDRATION_MAX_JSON_CHARS, HYDRATION_MAX_PASSAGES, LIST_MIN_RECORDS, LIST_NAME_MIN_CHARS, LIST_MIN_SHOWN, LIST_MAX_NAMES, LIST_MAX_LISTS, LOADING_MESSAGE, LOADING_TRAILING, LOADING_TEXT_MAX_CHARS, LOADING_PAGE_MAX_TEXT, SCREEN_READER_ONLY;
 var init_render = __esm({
   "packages/extract-tf/dist/render.js"() {
     "use strict";
     init_dom();
     APP_ROOT_SELECTOR = "#root, #app, #__next, #__nuxt, #___gatsby, [data-reactroot], [ng-app], [ng-version], [data-server-rendered]";
     JS_FALLBACK_CLASS = /(^|[\s_-])(hide-if-js|hide-if-js-enabled|js-disabled|js-only|requires-js|needs-js)([\s_-]|$)/i;
-    HYDRATION_SCRIPT = /window\.__[A-Za-z0-9_]+__\s*=|window\._[A-Za-z0-9_]*(?:STATE|CONFIG|DATA)\s*=|__NEXT_DATA__|__NUXT__/;
+    HYDRATION_SCRIPT = /window\.__[A-Za-z0-9_]+__\s*=|window\._[A-Za-z0-9_]*(?:STATE|CONFIG|DATA)\s*=|window\.__(?:data|state)\s*=|__NEXT_DATA__|__NUXT__/;
     APP_ROOT_MAX_TEXT = 200;
     HYDRATION_MIN_CHARS = 2e3;
     NOTICE_MAX_TEXT = 1500;
@@ -8811,6 +8844,11 @@ var init_render = __esm({
     LIST_MIN_SHOWN = 3;
     LIST_MAX_NAMES = 500;
     LIST_MAX_LISTS = 50;
+    LOADING_MESSAGE = /^(?:(?:loading|fetching|please wait|one moment|searching|retrieving)(?:\s+[\w-]+){0,3}\s*(?:\.{2,3}|\u2026)|please wait)$/i;
+    LOADING_TRAILING = /\b(?:loading|fetching)\s*(?:\.{2,3}|\u2026)$/i;
+    LOADING_TEXT_MAX_CHARS = 48;
+    LOADING_PAGE_MAX_TEXT = 4e3;
+    SCREEN_READER_ONLY = /(?:^|[\s_-])(?:sr-only|visually-?hidden|screen-?reader-(?:only|text))(?:$|[\s_-])|sronly|screenreaderonly/i;
   }
 });
 
@@ -8895,10 +8933,12 @@ function layoutParent(el) {
 function selectMain(doc, blocks) {
   if (blocks.length === 0)
     return null;
-  for (const sel of ["article", "main"]) {
+  for (const sel of ["article", "main", '[role="main"]']) {
     const el = doc.querySelector(sel);
     if (!el)
       continue;
+    if (sel === '[role="main"]' && qsa(doc, "h1").some((h1) => !el.contains(h1)))
+      continue;
     const inside = blocks.filter((b) => el.contains(b.el));
     if (inside.length >= blocks.length * 0.5)
       return el;
@@ -9655,6 +9695,22 @@ function hasBuyBoxUnderH2(doc, prices) {
   }
   return true;
 }
+function ownPriceAfterTitle(doc, names) {
+  const h1 = qsa(doc, "h1")[0];
+  if (h1 === void 0)
+    return false;
+  const prices = new Set(visiblePrices(doc));
+  const all = qsa(doc, "*");
+  const first2 = all.slice(all.indexOf(h1) + 1).find((el) => prices.has(el));
+  if (first2 === void 0)
+    return false;
+  for (let up = first2.parentElement; up !== null && !up.contains(h1); up = up.parentElement) {
+    const text2 = textOf(up).replace(/\s+/g, " ");
+    if (names.some((name) => text2.includes(name)))
+      return false;
+  }
+  return true;
+}
 function countAll(doc) {
   const textLength = (el) => (el?.textContent ?? "").replace(/\s+/g, " ").trim().length;
   const textChars = textLength(doc.body);
@@ -9695,6 +9751,39 @@ function collectPageTypes(root, out) {
       collectPageTypes(record3["@graph"], out);
   }
 }
+function ownListedProducts(root) {
+  let most = [];
+  for (const node of Array.isArray(root) ? root : [root]) {
+    if (typeof node !== "object" || node === null)
+      continue;
+    const record3 = node;
+    if (Array.isArray(record3["@graph"])) {
+      const listed = ownListedProducts(record3["@graph"]);
+      if (listed.length > most.length)
+        most = listed;
+    }
+    if (!typeNames(record3).includes("itemlist"))
+      continue;
+    if (typeof record3["name"] === "string" && isRecommendationHeading(record3["name"]))
+      continue;
+    const products = [];
+    for (const element of Array.isArray(record3["itemListElement"]) ? record3["itemListElement"] : []) {
+      if (typeof element !== "object" || element === null)
+        continue;
+      const item = element["item"];
+      const product = typeNames(element).includes("product") ? element : typeof item === "object" && item !== null && typeNames(item).includes("product") ? item : null;
+      if (product !== null)
+        products.push(typeof product["name"] === "string" ? product["name"].replace(/\s+/g, " ").trim() : "");
+    }
+    if (products.length > most.length)
+      most = products;
+  }
+  return most;
+}
+function typeNames(node) {
+  const t = node["@type"];
+  return (typeof t === "string" ? [t] : Array.isArray(t) ? t.filter((type) => typeof type === "string") : []).map(normalizeTypeName);
+}
 function collectJsonLdTypes(node, out, listed, inList = false) {
   if (Array.isArray(node)) {
     for (const item of node)
@@ -9711,7 +9800,7 @@ function collectJsonLdTypes(node, out, listed, inList = false) {
     else
       out.push(type);
   }
-  const list2 = inList || types.includes("itemlist");
+  const list2 = inList || types.includes("itemlist") || COLLECTION_PAGE_TYPES.some((type) => types.includes(type));
   for (const value of Object.values(node)) {
     if (typeof value === "object" && value !== null)
       collectJsonLdTypes(value, out, listed, list2);
@@ -9721,6 +9810,7 @@ function collectPageSignals(doc) {
   const jsonLdTypes = [];
   const listed = { products: 0 };
   const pageTypes = [];
+  let ownListed = [];
   for (const el of qsa(doc, 'script[type="application/ld+json"]')) {
     const text2 = (el.textContent ?? "").trim();
     if (text2.length === 0)
@@ -9729,6 +9819,9 @@ function collectPageSignals(doc) {
       const parsed = JSON.parse(text2);
       collectJsonLdTypes(parsed, jsonLdTypes, listed);
       collectPageTypes(parsed, pageTypes);
+      const own = ownListedProducts(parsed);
+      if (own.length > ownListed.length)
+        ownListed = own;
     } catch {
     }
   }
@@ -9747,7 +9840,9 @@ function collectPageSignals(doc) {
     postArticles: qsa(doc, "article.post").length,
     productCards: productCards(doc),
     listedProducts: listed.products,
-    pageTypes
+    pageTypes,
+    ownListedProducts: ownListed.length,
+    ownListedNames: ownListed.filter((name) => name.length > 0)
   };
 }
 function productCards(doc) {
@@ -9900,7 +9995,7 @@ function hasCardGrid(doc) {
 function routeByCounts(c, s, doc) {
   let grid;
   const cardGrid = () => grid ??= hasCardGrid(doc);
-  const listedOnly = s.listedProducts >= PRODUCT_CARDS && COLLECTION_PAGE_TYPES.some((type) => hasToken2(s.pageTypes, type)) && (s.productCards || !hasToken2(s.itemTypeTokens, "product"));
+  const listedOnly = s.listedProducts >= PRODUCT_CARDS && (COLLECTION_PAGE_TYPES.some((type) => hasToken2(s.pageTypes, type)) || s.ownListedProducts >= PRODUCT_CARDS && !ownPriceAfterTitle(doc, s.ownListedNames)) && (s.productCards || !hasToken2(s.itemTypeTokens, "product"));
   if ((s.productCards || listedOnly) && !c.buyBox && !hasToken2(s.jsonLdTypes, "product")) {
     return { type: "collection", strategy: "article" };
   }
@@ -11198,6 +11293,26 @@ function showsDeclaredProduct(main, product) {
   const price = product.price.value.replace(/[^\d.,]/g, "");
   return text2.includes(product.name.value.replace(/\s+/g, " ").trim()) && price !== "" && text2.includes(price);
 }
+function hiddenRegion(region) {
+  for (let el = region; el !== null; el = el.parentElement) {
+    const hidden = el.getAttribute("hidden");
+    if (hidden !== null && hidden.toLowerCase() !== "until-found" && !reactStreamedSegment(el))
+      return true;
+  }
+  return false;
+}
+function reactStreamedSegment(el) {
+  if (REACT_STREAMED_SEGMENT.test(el.id))
+    return true;
+  if (el.tagName.toLowerCase() !== "table" || el.id !== "")
+    return false;
+  const part = el.firstElementChild;
+  if (part === null)
+    return false;
+  if (REACT_STREAMED_SEGMENT.test(part.id))
+    return true;
+  return part.tagName.toLowerCase() === "tbody" && part.id === "" && REACT_STREAMED_SEGMENT.test(part.firstElementChild?.id ?? "");
+}
 function headingsOnly(region) {
   const isHeading = (el) => /^h[1-6]$/.test(el.tagName.toLowerCase());
   let headingChars = 0;
@@ -11231,7 +11346,7 @@ function headingsOnly(region) {
   }
   return true;
 }
-var DEFAULT_CLASSIFY, BUY_BOX_CONFIDENCE, DESCRIPTION_MIN_CHARS, ExtractTf, extractTf, HEADINGS_ONLY_MAX_CHARS, MIN_PROSE_CHARS;
+var DEFAULT_CLASSIFY, BUY_BOX_CONFIDENCE, DESCRIPTION_MIN_CHARS, ExtractTf, extractTf, REACT_STREAMED_SEGMENT, HEADINGS_ONLY_MAX_CHARS, MIN_PROSE_CHARS;
 var init_extract = __esm({
   "packages/extract-tf/dist/extract.js"() {
     "use strict";
@@ -11350,11 +11465,11 @@ var init_extract = __esm({
           baseUrl,
           metadata,
           confidence: selected ? 1 : confidenceOf(blocks.filter((b) => main?.contains(b.el)).length, main, blocks.length, mainLength, decision.type, productShown, favorPrecision, favorRecall, product),
-          // Escalate when a strategy produced nothing at all, or the article
-          // cascade a region that only names the page (headingsOnly; a region the
-          // last resort found is a list's). Routing to a non-article strategy is
-          // not by itself an escalation reason.
-          escalate: main === null || strategy === "article" && headingsOnly(main),
+          // Escalate when a strategy produced nothing at all, a region the page
+          // hides (hiddenRegion), or the article cascade a region that only names
+          // the page (headingsOnly; a region the last resort found is a list's).
+          // Routing to a non-article strategy is not by itself an escalation reason.
+          escalate: main === null || hiddenRegion(main) || strategy === "article" && headingsOnly(main),
           ...lastResort ? { lastResort: true } : {},
           pageType: decision.type,
           strategy,
@@ -11368,7 +11483,8 @@ var init_extract = __esm({
             productShown,
             hydrationShown: dataShown,
             contentShown: showsDeclaredProduct(main, product),
-            listing: decision.type === "listing" || decision.type === "collection"
+            listing: decision.type === "listing" || decision.type === "collection",
+            loadingShown: main !== null && hasLoadingText(main)
           }),
           labelledValues: main ? collectLabelledValues(main) : [],
           timings: { parseMs, extractMs: Math.max(0, performance.now() - extractionStart) }
@@ -11378,6 +11494,7 @@ var init_extract = __esm({
       }
     };
     extractTf = new ExtractTf();
+    REACT_STREAMED_SEGMENT = /S:[0-9a-f]+$/i;
     HEADINGS_ONLY_MAX_CHARS = 100;
     MIN_PROSE_CHARS = 3;
   }
@@ -11388,7 +11505,7 @@ var EXTRACTOR_VERSION;
 var init_version = __esm({
   "packages/extract-tf/dist/version.js"() {
     "use strict";
-    EXTRACTOR_VERSION = "extract-tf/16";
+    EXTRACTOR_VERSION = "extract-tf/19";
   }
 });
 
@@ -12983,6 +13100,7 @@ var init_identity = __esm({
 var provider_exports = {};
 __export(provider_exports, {
   ProviderSubject: () => ProviderSubject,
+  providerFailure: () => providerFailure,
   robotsFetcherVia: () => robotsFetcherVia
 });
 function robotsFetcherVia(dispatcherFor) {
@@ -13013,6 +13131,40 @@ function isPlainText2(contentType) {
     return true;
   return contentType.toLowerCase().trimStart().startsWith("text/plain");
 }
+function providerFailure(url, wallMs, trace, err, aborted) {
+  return {
+    requestedUrl: url,
+    status: "failed",
+    failureReason: aborted ? "timeout" : err instanceof Error && err.message.includes("net::ERR_NAME_NOT_RESOLVED") ? "dns_error" : "provider_error",
+    blockReason: null,
+    budgetExceeded: null,
+    lane: "provider",
+    escalations: [],
+    markdown: null,
+    truncated: false,
+    truncatedAt: null,
+    compliance: null,
+    evidence: {
+      finalUrl: url,
+      httpStatus: null,
+      redirectChain: [],
+      contentType: null,
+      rawBodySha256: null,
+      artifacts: []
+    },
+    usage: {
+      wallMs,
+      bytesWire: 0,
+      bytesDecompressed: 0,
+      requestCount: 1,
+      attemptCount: 1,
+      contentTokens: null,
+      browserMs: 0,
+      externalCostUsd: null
+    },
+    trace
+  };
+}
 var defaultRobotsFetcher, ProviderSubject;
 var init_provider2 = __esm({
   "packages/bench/dist/subjects/provider.js"() {
@@ -13147,41 +13299,7 @@ var init_provider2 = __esm({
             event: "provider_failed",
             detail: { error: err instanceof Error ? err.message.slice(0, 200) : String(err) }
           });
-          return {
-            requestedUrl: url,
-            status: "failed",
-            // The provider broke, not the target. Reporting this as http_error
-            // would blame the publisher for our vendor's outage. A target name
-            // the vendor's browser could not resolve is a DNS fact, not a fault.
-            failureReason: execution.signal?.aborted ? "timeout" : err instanceof Error && err.message.includes("net::ERR_NAME_NOT_RESOLVED") ? "dns_error" : "provider_error",
-            blockReason: null,
-            budgetExceeded: null,
-            lane: "provider",
-            escalations: [],
-            markdown: null,
-            truncated: false,
-            truncatedAt: null,
-            compliance: null,
-            evidence: {
-              finalUrl: url,
-              httpStatus: null,
-              redirectChain: [],
-              contentType: null,
-              rawBodySha256: null,
-              artifacts: []
-            },
-            usage: {
-              wallMs: wallMs2,
-              bytesWire: 0,
-              bytesDecompressed: 0,
-              requestCount: 1,
-              attemptCount: 1,
-              contentTokens: null,
-              browserMs: 0,
-              externalCostUsd: null
-            },
-            trace
-          };
+          return providerFailure(url, wallMs2, trace, err, execution.signal?.aborted === true);
         }
         const wallMs = Date.now() - start;
         const observedUa = res.sentUserAgent ?? null;
@@ -25202,6 +25320,7 @@ function buildChannels(mode, opts = {}) {
       }
       return await pending;
     };
+    const requestDeadlines = /* @__PURE__ */ new WeakMap();
     const raw = {
       id: "provider",
       vendorId,
@@ -25254,13 +25373,34 @@ function buildChannels(mode, opts = {}) {
           };
           return skip;
         }
-        if (sessionApplies && session.resume !== void 0 && session.resume !== null) {
-          pendingResume = session.resume;
-          persistenceAttempted = true;
-        } else {
-          await preparePersistence(execution);
+        const start = Date.now();
+        const requestDeadlineAt = execution !== void 0 && requestDeadlines.has(execution) ? requestDeadlines.get(execution) : execution?.deadlineAt;
+        const requestEnded = () => execution?.signal?.aborted === true || requestDeadlineAt !== void 0 && Date.now() >= requestDeadlineAt;
+        const sessionEnded = () => execution?.deadlineAt !== void 0 && Date.now() >= execution.deadlineAt;
+        const failed = async (err) => {
+          const { providerFailure: providerFailure2 } = await Promise.resolve().then(() => (init_provider2(), provider_exports));
+          const wallMs = Date.now() - start;
+          const message2 = ops.secrets.reduce((m, secret) => scrubSecret(m, secret), err instanceof Error ? err.message : String(err)).slice(0, 200);
+          return providerFailure2(url, wallMs, [
+            { at: 0, lane: "provider", event: "provider_selected", detail: { provider: vendorId } },
+            { at: wallMs, lane: "provider", event: "provider_failed", detail: { error: message2 } }
+          ], err, sessionEnded());
+        };
+        let vendor;
+        try {
+          if (sessionApplies && session.resume !== void 0 && session.resume !== null) {
+            pendingResume = session.resume;
+            persistenceAttempted = true;
+          } else {
+            await preparePersistence(execution);
+          }
+          vendor = tariff !== null ? await connectOwn(execution) : await ensureConnected(execution);
+        } catch (err) {
+          if (requestEnded())
+            throw err;
+          return await failed(err);
         }
-        const { declaration, transport } = tariff !== null ? await connectOwn(execution) : await ensureConnected(execution);
+        const { declaration, transport } = vendor;
         try {
           if (pendingResume !== null) {
             transport.useResumedSession(pendingResume);
@@ -25268,6 +25408,10 @@ function buildChannels(mode, opts = {}) {
           const { ProviderSubject: ProviderSubject2 } = await Promise.resolve().then(() => (init_provider2(), provider_exports));
           const subject = new ProviderSubject2(declaration, transport, mode, null, providerRobots);
           return await subject.fetch(url, execution?.deadlineAt, execution?.signal, execution?.onRetryAfter, options);
+        } catch (err) {
+          if (requestEnded() || !sessionEnded())
+            throw err;
+          return await failed(err);
         } finally {
           if (tariff !== null)
             await transport.close().catch(() => {
@@ -25293,7 +25437,9 @@ function buildChannels(mode, opts = {}) {
       fetch: async (url, session, execution, options) => {
         if (tariff === null || tariff.maxSessionMs === null)
           return raw.fetch(url, session, execution, options);
-        return raw.fetch(url, session, { ...execution, deadlineAt: Math.min(execution?.deadlineAt ?? Number.POSITIVE_INFINITY, Date.now() + tariff.maxSessionMs) }, options);
+        const held = { ...execution, deadlineAt: Math.min(execution?.deadlineAt ?? Number.POSITIVE_INFINITY, Date.now() + tariff.maxSessionMs) };
+        requestDeadlines.set(held, execution?.deadlineAt);
+        return raw.fetch(url, session, held, options);
       }
     };
   };
@@ -25708,7 +25854,7 @@ import { mkdir as mkdir7, writeFile as writeFile6 } from "fs/promises";
 init_dist();
 
 // packages/sdk/dist/version.js
-var SDK_VERSION = "0.3.2";
+var SDK_VERSION = "0.4.0";
 
 // packages/sdk/dist/watcher.js
 init_dist();
@@ -31638,10 +31784,8 @@ function createApp(engine, options = {}) {
       return fail(c, "invalid_request", "cursor is not one this API issued");
     if (err instanceof SyntaxError)
       return fail(c, "invalid_json", "body must be JSON");
-    if (options.exposeInternalErrors === true)
-      return fail(c, "internal_error", err.message);
     console.error(JSON.stringify({ component: "api", method: c.req.method, path: c.req.path, error: err.stack ?? String(err) }));
-    return fail(c, "internal_error", "internal error");
+    return fail(c, "internal_error", options.exposeInternalErrors === true ? err.message : "internal error");
   });
   return app;
 }
@@ -32275,7 +32419,7 @@ ${LOGIN_USAGE}`);
 }
 
 // packages/cli/src/run.ts
-var CLI_VERSION = "0.3.2";
+var CLI_VERSION = "0.4.0";
 async function runCli(argv, io) {
   const [name, ...rest] = argv;
   if (name === void 0 || name === "--help" || name === "help") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "octocrawl",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Octocrawl on the command line: scrape, crawl, batch and map with an Evidence Record on every page, or serve the local API.",
   "license": "AGPL-3.0-only",
   "keywords": [