@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/dist/engine/runAdapter.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/engine/runAdapter.ts"],
+  "sourcesContent": ["import type { AdapterContext } from '../contract/adapter'\nimport { errorMessage } from '../contract/errors'\nimport type { HttpClient, Logger } from '../contract/http'\nimport { failed, type SearchOutcome } from '../contract/outcomes'\nimport type { SearchRequest } from '../contract/results'\nimport type { SearchStepEvent, SearchStepMetrics } from '../contract/steps'\nimport { linkSignals } from './deadline'\nimport type { AdapterDiagnosticStatus, EngineAdapterEntry } from './types'\n\nexport type AdapterRunResult = {\n  readonly entry: EngineAdapterEntry\n  readonly outcome: SearchOutcome\n  readonly latencyMs: number\n}\n\nexport type AdapterRunDeps = {\n  readonly http: HttpClient\n  readonly logger: Logger\n  readonly now: () => number\n  readonly report: (adapterId: string, event: SearchStepEvent, detail?: string, metrics?: SearchStepMetrics) => void\n  readonly parentSignal: AbortSignal\n  readonly deadlineAt: number\n  readonly adapterTimeoutMs: number\n}\n\nexport function outcomeToStatus(outcome: SearchOutcome): AdapterDiagnosticStatus {\n  switch (outcome.status) {\n    case 'ok':\n      return 'ok'\n    case 'empty':\n      return 'empty'\n    case 'unavailable':\n      return 'unavailable'\n    case 'blocked':\n      return 'blocked'\n    case 'timeout':\n      return 'timeout'\n    case 'error':\n      return 'error'\n  }\n}\n\nexport function outcomeReason(outcome: SearchOutcome): string | undefined {\n  if (outcome.status === 'ok' || outcome.status === 'empty') return undefined\n  return outcome.reason\n}\n\n/**\n * Runs one adapter under its own timeout and abort linkage.\n *\n * The contract says adapters return outcomes rather than throwing, but a\n * third-party adapter is not trusted to honour that \u2014 a throw here is converted\n * into an `error` outcome so one misbehaving package cannot abort the whole run.\n */\nexport async function runAdapter(\n  entry: EngineAdapterEntry,\n  request: SearchRequest,\n  deps: AdapterRunDeps,\n): Promise<AdapterRunResult> {\n  const startedAt = deps.now()\n  const allowance = entry.timeoutMs ?? deps.adapterTimeoutMs\n  const budget = Math.max(0, Math.min(allowance, deps.deadlineAt - startedAt))\n  const controller = new AbortController()\n  const timer = setTimeout(() => controller.abort(), budget)\n  timer.unref?.()\n  const linked = linkSignals([deps.parentSignal, controller.signal])\n\n  const context: AdapterContext = {\n    signal: linked.signal,\n    http: deps.http,\n    report: (event, detail, metrics) => deps.report(entry.adapter.id, event, detail, metrics),\n    deadlineAt: startedAt + budget,\n    logger: deps.logger,\n  }\n\n  deps.report(entry.adapter.id, 'started')\n\n  try {\n    const outcome = await entry.adapter.search(request, context)\n    return { entry, outcome, latencyMs: deps.now() - startedAt }\n  } catch (error) {\n    const detail = `adapter threw instead of returning an outcome: ${errorMessage(error)}`\n    deps.logger.warn(`[web-research] ${entry.adapter.id} ${detail}`)\n    return { entry, outcome: failed(detail, false), latencyMs: deps.now() - startedAt }\n  } finally {\n    clearTimeout(timer)\n    linked.dispose()\n  }\n}\n"],
+  "mappings": "AACA,SAAS,oBAAoB;AAE7B,SAAS,cAAkC;AAG3C,SAAS,mBAAmB;AAmBrB,SAAS,gBAAgB,SAAiD;AAC/E,UAAQ,QAAQ,QAAQ;AAAA,IACtB,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAEO,SAAS,cAAc,SAA4C;AACxE,MAAI,QAAQ,WAAW,QAAQ,QAAQ,WAAW,QAAS,QAAO;AAClE,SAAO,QAAQ;AACjB;AASA,eAAsB,WACpB,OACA,SACA,MAC2B;AAC3B,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,YAAY,MAAM,aAAa,KAAK;AAC1C,QAAM,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,WAAW,KAAK,aAAa,SAAS,CAAC;AAC3E,QAAM,aAAa,IAAI,gBAAgB;AACvC,QAAM,QAAQ,WAAW,MAAM,WAAW,MAAM,GAAG,MAAM;AACzD,QAAM,QAAQ;AACd,QAAM,SAAS,YAAY,CAAC,KAAK,cAAc,WAAW,MAAM,CAAC;AAEjE,QAAM,UAA0B;AAAA,IAC9B,QAAQ,OAAO;AAAA,IACf,MAAM,KAAK;AAAA,IACX,QAAQ,CAAC,OAAO,QAAQ,YAAY,KAAK,OAAO,MAAM,QAAQ,IAAI,OAAO,QAAQ,OAAO;AAAA,IACxF,YAAY,YAAY;AAAA,IACxB,QAAQ,KAAK;AAAA,EACf;AAEA,OAAK,OAAO,MAAM,QAAQ,IAAI,SAAS;AAEvC,MAAI;AACF,UAAM,UAAU,MAAM,MAAM,QAAQ,OAAO,SAAS,OAAO;AAC3D,WAAO,EAAE,OAAO,SAAS,WAAW,KAAK,IAAI,IAAI,UAAU;AAAA,EAC7D,SAAS,OAAO;AACd,UAAM,SAAS,kDAAkD,aAAa,KAAK,CAAC;AACpF,SAAK,OAAO,KAAK,kBAAkB,MAAM,QAAQ,EAAE,IAAI,MAAM,EAAE;AAC/D,WAAO,EAAE,OAAO,SAAS,OAAO,QAAQ,KAAK,GAAG,WAAW,KAAK,IAAI,IAAI,UAAU;AAAA,EACpF,UAAE;AACA,iBAAa,KAAK;AAClB,WAAO,QAAQ;AAAA,EACjB;AACF;",
+  "names": []
+}

package/dist/engine/types.js

@@ -0,0 +1 @@
+//# sourceMappingURL=types.js.map

package/dist/engine/types.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/extract/classify.js

@@ -0,0 +1,51 @@
+const CHALLENGE_MARKERS = [
+  "cf-browser-verification",
+  "cf_chl_opt",
+  "just a moment",
+  "checking your browser",
+  "enable cookies to continue",
+  "ddos protection by",
+  "access denied",
+  "captcha",
+  "are you a robot"
+];
+const JS_REQUIRED_MARKERS = [
+  "enable javascript",
+  "javascript is required",
+  "javascript is disabled",
+  "doesn't work properly without javascript",
+  "you need to enable javascript to run this app"
+];
+const EMPTY_ROOT = /<div[^<>]{0,200}\bid=["']?(root|__next|app|__nuxt)["']?[^<>]{0,200}>\s*<\/div>/i;
+const MIN_MEANINGFUL_TEXT = 200;
+const SHELL_HTML_THRESHOLD = 4e3;
+function classifyPage(input) {
+  const haystack = input.html.slice(0, 2e5).toLowerCase();
+  if (input.status === 403 || input.status === 429 || input.status === 503) {
+    return { verdict: "blocked", reason: `HTTP ${input.status}` };
+  }
+  const text = input.text.trim();
+  if (text.length === 0 && input.html.trim().length === 0) {
+    return { verdict: "empty", reason: "no response body" };
+  }
+  if (text.length < MIN_MEANINGFUL_TEXT) {
+    const challenge = CHALLENGE_MARKERS.find((marker) => haystack.includes(marker));
+    if (challenge) return { verdict: "blocked", reason: `anti-bot challenge (${challenge})` };
+  }
+  const jsMarker = JS_REQUIRED_MARKERS.find((marker) => haystack.includes(marker));
+  if (jsMarker && text.length < MIN_MEANINGFUL_TEXT) {
+    return { verdict: "js-shell", reason: `page requires JavaScript (${jsMarker})` };
+  }
+  if (EMPTY_ROOT.test(input.html) && text.length < MIN_MEANINGFUL_TEXT) {
+    return { verdict: "js-shell", reason: "empty client-side render root" };
+  }
+  if (text.length < MIN_MEANINGFUL_TEXT && input.html.length > SHELL_HTML_THRESHOLD) {
+    return { verdict: "js-shell", reason: `only ${text.length} chars of text in ${input.html.length} of markup` };
+  }
+  if (text.length === 0) return { verdict: "empty", reason: "no extractable text" };
+  return { verdict: "ok" };
+}
+export {
+  classifyPage
+};
+//# sourceMappingURL=classify.js.map

package/dist/extract/classify.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extract/classify.ts"],
+  "sourcesContent": ["import type { PageVerdict } from '../contract/results'\n\nexport type PageClassification = {\n  readonly verdict: PageVerdict\n  readonly reason?: string\n}\n\nexport type ClassifyInput = {\n  readonly status: number\n  readonly contentType: string | null\n  readonly html: string\n  readonly text: string\n}\n\nconst CHALLENGE_MARKERS = [\n  'cf-browser-verification',\n  'cf_chl_opt',\n  'just a moment',\n  'checking your browser',\n  'enable cookies to continue',\n  'ddos protection by',\n  'access denied',\n  'captcha',\n  'are you a robot',\n]\n\nconst JS_REQUIRED_MARKERS = [\n  'enable javascript',\n  'javascript is required',\n  'javascript is disabled',\n  \"doesn't work properly without javascript\",\n  'you need to enable javascript to run this app',\n]\n\nconst EMPTY_ROOT = /<div[^<>]{0,200}\\bid=[\"']?(root|__next|app|__nuxt)[\"']?[^<>]{0,200}>\\s*<\\/div>/i\n\n/** Below this, a text-bearing HTML page is almost certainly a shell, not content. */\nconst MIN_MEANINGFUL_TEXT = 200\nconst SHELL_HTML_THRESHOLD = 4_000\n\n/**\n * Decides whether an HTTP read actually produced readable content. The verdict\n * drives browser escalation in code rather than leaving the judgement to the\n * model, which costs a full round-trip and answers differently each time.\n */\nexport function classifyPage(input: ClassifyInput): PageClassification {\n  const haystack = input.html.slice(0, 200_000).toLowerCase()\n\n  if (input.status === 403 || input.status === 429 || input.status === 503) {\n    return { verdict: 'blocked', reason: `HTTP ${input.status}` }\n  }\n\n  const text = input.text.trim()\n  if (text.length === 0 && input.html.trim().length === 0) {\n    return { verdict: 'empty', reason: 'no response body' }\n  }\n\n  // Marker matching runs over raw markup, including script bundles, so plenty of\n  // ordinary pages mention \"captcha\" or \"access denied\" without being one. A\n  // challenge page is content-free by definition, so the markers only count when\n  // the page also failed to yield readable text \u2014 otherwise a working page gets\n  // flagged and triggers a pointless browser escalation.\n  if (text.length < MIN_MEANINGFUL_TEXT) {\n    const challenge = CHALLENGE_MARKERS.find((marker) => haystack.includes(marker))\n    if (challenge) return { verdict: 'blocked', reason: `anti-bot challenge (${challenge})` }\n  }\n\n  const jsMarker = JS_REQUIRED_MARKERS.find((marker) => haystack.includes(marker))\n  if (jsMarker && text.length < MIN_MEANINGFUL_TEXT) {\n    return { verdict: 'js-shell', reason: `page requires JavaScript (${jsMarker})` }\n  }\n  if (EMPTY_ROOT.test(input.html) && text.length < MIN_MEANINGFUL_TEXT) {\n    return { verdict: 'js-shell', reason: 'empty client-side render root' }\n  }\n  if (text.length < MIN_MEANINGFUL_TEXT && input.html.length > SHELL_HTML_THRESHOLD) {\n    return { verdict: 'js-shell', reason: `only ${text.length} chars of text in ${input.html.length} of markup` }\n  }\n  if (text.length === 0) return { verdict: 'empty', reason: 'no extractable text' }\n\n  return { verdict: 'ok' }\n}\n"],
+  "mappings": "AAcA,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AAGnB,MAAM,sBAAsB;AAC5B,MAAM,uBAAuB;AAOtB,SAAS,aAAa,OAA0C;AACrE,QAAM,WAAW,MAAM,KAAK,MAAM,GAAG,GAAO,EAAE,YAAY;AAE1D,MAAI,MAAM,WAAW,OAAO,MAAM,WAAW,OAAO,MAAM,WAAW,KAAK;AACxE,WAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,MAAM,MAAM,GAAG;AAAA,EAC9D;AAEA,QAAM,OAAO,MAAM,KAAK,KAAK;AAC7B,MAAI,KAAK,WAAW,KAAK,MAAM,KAAK,KAAK,EAAE,WAAW,GAAG;AACvD,WAAO,EAAE,SAAS,SAAS,QAAQ,mBAAmB;AAAA,EACxD;AAOA,MAAI,KAAK,SAAS,qBAAqB;AACrC,UAAM,YAAY,kBAAkB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC;AAC9E,QAAI,UAAW,QAAO,EAAE,SAAS,WAAW,QAAQ,uBAAuB,SAAS,IAAI;AAAA,EAC1F;AAEA,QAAM,WAAW,oBAAoB,KAAK,CAAC,WAAW,SAAS,SAAS,MAAM,CAAC;AAC/E,MAAI,YAAY,KAAK,SAAS,qBAAqB;AACjD,WAAO,EAAE,SAAS,YAAY,QAAQ,6BAA6B,QAAQ,IAAI;AAAA,EACjF;AACA,MAAI,WAAW,KAAK,MAAM,IAAI,KAAK,KAAK,SAAS,qBAAqB;AACpE,WAAO,EAAE,SAAS,YAAY,QAAQ,gCAAgC;AAAA,EACxE;AACA,MAAI,KAAK,SAAS,uBAAuB,MAAM,KAAK,SAAS,sBAAsB;AACjF,WAAO,EAAE,SAAS,YAAY,QAAQ,QAAQ,KAAK,MAAM,qBAAqB,MAAM,KAAK,MAAM,aAAa;AAAA,EAC9G;AACA,MAAI,KAAK,WAAW,EAAG,QAAO,EAAE,SAAS,SAAS,QAAQ,sBAAsB;AAEhF,SAAO,EAAE,SAAS,KAAK;AACzB;",
+  "names": []
+}

package/dist/extract/content.js

@@ -0,0 +1,104 @@
+import { BLOCK_ELEMENTS, NON_CONTENT_ELEMENTS, normalizeWhitespace } from "./text.js";
+import { tokenizeHtml } from "./tokenizer.js";
+const CANDIDATE_ELEMENTS = /* @__PURE__ */ new Set(["article", "main", "section", "div", "td", "body"]);
+const SEMANTIC_BONUS = /* @__PURE__ */ new Set(["article", "main"]);
+const NEGATIVE_HINT = /(^|[\s_-])(nav|navigation|menu|sidebar|side-bar|footer|masthead|comment|share|social|advert|promo|banner|cookie|consent|related|breadcrumb|pagination|widget|newsletter|subscribe)([\s_-]|$)/i;
+const POSITIVE_HINT = /(^|[\s_-])(article|content|post|story|entry|body-?text|main)([\s_-]|$)/i;
+const MIN_CONTENT_CHARS = 200;
+function hintScoreOf(attributes) {
+  const descriptor = `${attributes.get("class") ?? ""} ${attributes.get("id") ?? ""} ${attributes.get("role") ?? ""}`;
+  if (descriptor.trim().length === 0) return 0;
+  let score = 0;
+  if (NEGATIVE_HINT.test(descriptor)) score -= 600;
+  if (POSITIVE_HINT.test(descriptor)) score += 300;
+  return score;
+}
+function scoreOf(frame) {
+  const semantic = SEMANTIC_BONUS.has(frame.name) ? 400 : 0;
+  return frame.textLength - 2 * frame.linkTextLength + 25 * frame.paragraphs + frame.hintScore + semantic;
+}
+function extractMainContent(html) {
+  const parts = [];
+  const stack = [];
+  const best = { current: null };
+  let linkDepth = 0;
+  let skipName = null;
+  let skipDepth = 0;
+  const closeFrame = () => {
+    const frame = stack.pop();
+    if (!frame) return;
+    const score = scoreOf(frame);
+    if (frame.textLength >= MIN_CONTENT_CHARS && (best.current === null || score > best.current.score)) {
+      best.current = { startPart: frame.startPart, endPart: parts.length, score };
+    }
+    const parent = stack[stack.length - 1];
+    if (parent) {
+      parent.textLength += frame.textLength;
+      parent.linkTextLength += frame.linkTextLength;
+      parent.paragraphs += frame.paragraphs;
+    }
+  };
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === "open") {
+      if (skipName !== null) {
+        if (token.name === skipName) skipDepth += 1;
+        continue;
+      }
+      if (NON_CONTENT_ELEMENTS.has(token.name)) {
+        skipName = token.name;
+        skipDepth = 1;
+        continue;
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push("\n");
+      if (token.name === "a") linkDepth += 1;
+      if (token.name === "p") {
+        const frame2 = stack[stack.length - 1];
+        if (frame2) frame2.paragraphs += 1;
+      }
+      if (CANDIDATE_ELEMENTS.has(token.name) && !token.selfClosing) {
+        stack.push({
+          name: token.name,
+          startPart: parts.length,
+          hintScore: hintScoreOf(token.attributes),
+          textLength: 0,
+          linkTextLength: 0,
+          paragraphs: 0
+        });
+      }
+      continue;
+    }
+    if (token.kind === "close") {
+      if (skipName !== null) {
+        if (token.name === skipName) {
+          skipDepth -= 1;
+          if (skipDepth <= 0) skipName = null;
+        }
+        continue;
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push("\n");
+      if (token.name === "a" && linkDepth > 0) linkDepth -= 1;
+      if (CANDIDATE_ELEMENTS.has(token.name)) {
+        const index = stack.findLastIndex((frame2) => frame2.name === token.name);
+        if (index !== -1) {
+          while (stack.length > index) closeFrame();
+        }
+      }
+      continue;
+    }
+    if (skipName !== null) continue;
+    parts.push(token.value);
+    const frame = stack[stack.length - 1];
+    if (!frame) continue;
+    const length = token.value.trim().length;
+    frame.textLength += length;
+    if (linkDepth > 0) frame.linkTextLength += length;
+  }
+  while (stack.length > 0) closeFrame();
+  const winner = best.current;
+  const selected = winner === null ? parts : parts.slice(winner.startPart, winner.endPart);
+  return normalizeWhitespace(selected.join(""));
+}
+export {
+  extractMainContent
+};
+//# sourceMappingURL=content.js.map

package/dist/extract/content.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extract/content.ts"],
+  "sourcesContent": ["import { BLOCK_ELEMENTS, NON_CONTENT_ELEMENTS, normalizeWhitespace } from './text'\nimport { tokenizeHtml } from './tokenizer'\n\nconst CANDIDATE_ELEMENTS = new Set(['article', 'main', 'section', 'div', 'td', 'body'])\nconst SEMANTIC_BONUS = new Set(['article', 'main'])\n\nconst NEGATIVE_HINT =\n  /(^|[\\s_-])(nav|navigation|menu|sidebar|side-bar|footer|masthead|comment|share|social|advert|promo|banner|cookie|consent|related|breadcrumb|pagination|widget|newsletter|subscribe)([\\s_-]|$)/i\nconst POSITIVE_HINT = /(^|[\\s_-])(article|content|post|story|entry|body-?text|main)([\\s_-]|$)/i\n\n/** Below this a candidate is treated as chrome rather than an article. */\nconst MIN_CONTENT_CHARS = 200\n\ntype BestRange = {\n  readonly startPart: number\n  readonly endPart: number\n  readonly score: number\n}\n\ntype Frame = {\n  readonly name: string\n  readonly startPart: number\n  readonly hintScore: number\n  textLength: number\n  linkTextLength: number\n  paragraphs: number\n}\n\nfunction hintScoreOf(attributes: ReadonlyMap<string, string>): number {\n  const descriptor = `${attributes.get('class') ?? ''} ${attributes.get('id') ?? ''} ${attributes.get('role') ?? ''}`\n  if (descriptor.trim().length === 0) return 0\n  let score = 0\n  if (NEGATIVE_HINT.test(descriptor)) score -= 600\n  if (POSITIVE_HINT.test(descriptor)) score += 300\n  return score\n}\n\nfunction scoreOf(frame: Frame): number {\n  const semantic = SEMANTIC_BONUS.has(frame.name) ? 400 : 0\n  return frame.textLength - 2 * frame.linkTextLength + 25 * frame.paragraphs + frame.hintScore + semantic\n}\n\n/**\n * Picks the subtree that reads like the article, using text-to-link density plus\n * paragraph count and class/id hints. Counters are accumulated during the single\n * tokenizer pass and only the winning range is materialized, so the scan stays\n * linear in time and the output allocates once.\n */\nexport function extractMainContent(html: string): string {\n  const parts: string[] = []\n  const stack: Frame[] = []\n  const best: { current: BestRange | null } = { current: null }\n  let linkDepth = 0\n  let skipName: string | null = null\n  let skipDepth = 0\n\n  const closeFrame = (): void => {\n    const frame = stack.pop()\n    if (!frame) return\n    const score = scoreOf(frame)\n    if (frame.textLength >= MIN_CONTENT_CHARS && (best.current === null || score > best.current.score)) {\n      best.current = { startPart: frame.startPart, endPart: parts.length, score }\n    }\n    const parent = stack[stack.length - 1]\n    if (parent) {\n      parent.textLength += frame.textLength\n      parent.linkTextLength += frame.linkTextLength\n      parent.paragraphs += frame.paragraphs\n    }\n  }\n\n  for (const token of tokenizeHtml(html)) {\n    if (token.kind === 'open') {\n      if (skipName !== null) {\n        if (token.name === skipName) skipDepth += 1\n        continue\n      }\n      if (NON_CONTENT_ELEMENTS.has(token.name)) {\n        skipName = token.name\n        skipDepth = 1\n        continue\n      }\n      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\\n')\n      if (token.name === 'a') linkDepth += 1\n      if (token.name === 'p') {\n        const frame = stack[stack.length - 1]\n        if (frame) frame.paragraphs += 1\n      }\n      if (CANDIDATE_ELEMENTS.has(token.name) && !token.selfClosing) {\n        stack.push({\n          name: token.name,\n          startPart: parts.length,\n          hintScore: hintScoreOf(token.attributes),\n          textLength: 0,\n          linkTextLength: 0,\n          paragraphs: 0,\n        })\n      }\n      continue\n    }\n\n    if (token.kind === 'close') {\n      if (skipName !== null) {\n        if (token.name === skipName) {\n          skipDepth -= 1\n          if (skipDepth <= 0) skipName = null\n        }\n        continue\n      }\n      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\\n')\n      if (token.name === 'a' && linkDepth > 0) linkDepth -= 1\n      if (CANDIDATE_ELEMENTS.has(token.name)) {\n        // Unbalanced markup is common; only unwind to the nearest matching frame.\n        const index = stack.findLastIndex((frame) => frame.name === token.name)\n        if (index !== -1) {\n          while (stack.length > index) closeFrame()\n        }\n      }\n      continue\n    }\n\n    if (skipName !== null) continue\n    parts.push(token.value)\n    const frame = stack[stack.length - 1]\n    if (!frame) continue\n    const length = token.value.trim().length\n    frame.textLength += length\n    if (linkDepth > 0) frame.linkTextLength += length\n  }\n\n  while (stack.length > 0) closeFrame()\n\n  const winner = best.current\n  const selected = winner === null ? parts : parts.slice(winner.startPart, winner.endPart)\n  return normalizeWhitespace(selected.join(''))\n}\n"],
+  "mappings": "AAAA,SAAS,gBAAgB,sBAAsB,2BAA2B;AAC1E,SAAS,oBAAoB;AAE7B,MAAM,qBAAqB,oBAAI,IAAI,CAAC,WAAW,QAAQ,WAAW,OAAO,MAAM,MAAM,CAAC;AACtF,MAAM,iBAAiB,oBAAI,IAAI,CAAC,WAAW,MAAM,CAAC;AAElD,MAAM,gBACJ;AACF,MAAM,gBAAgB;AAGtB,MAAM,oBAAoB;AAiB1B,SAAS,YAAY,YAAiD;AACpE,QAAM,aAAa,GAAG,WAAW,IAAI,OAAO,KAAK,EAAE,IAAI,WAAW,IAAI,IAAI,KAAK,EAAE,IAAI,WAAW,IAAI,MAAM,KAAK,EAAE;AACjH,MAAI,WAAW,KAAK,EAAE,WAAW,EAAG,QAAO;AAC3C,MAAI,QAAQ;AACZ,MAAI,cAAc,KAAK,UAAU,EAAG,UAAS;AAC7C,MAAI,cAAc,KAAK,UAAU,EAAG,UAAS;AAC7C,SAAO;AACT;AAEA,SAAS,QAAQ,OAAsB;AACrC,QAAM,WAAW,eAAe,IAAI,MAAM,IAAI,IAAI,MAAM;AACxD,SAAO,MAAM,aAAa,IAAI,MAAM,iBAAiB,KAAK,MAAM,aAAa,MAAM,YAAY;AACjG;AAQO,SAAS,mBAAmB,MAAsB;AACvD,QAAM,QAAkB,CAAC;AACzB,QAAM,QAAiB,CAAC;AACxB,QAAM,OAAsC,EAAE,SAAS,KAAK;AAC5D,MAAI,YAAY;AAChB,MAAI,WAA0B;AAC9B,MAAI,YAAY;AAEhB,QAAM,aAAa,MAAY;AAC7B,UAAM,QAAQ,MAAM,IAAI;AACxB,QAAI,CAAC,MAAO;AACZ,UAAM,QAAQ,QAAQ,KAAK;AAC3B,QAAI,MAAM,cAAc,sBAAsB,KAAK,YAAY,QAAQ,QAAQ,KAAK,QAAQ,QAAQ;AAClG,WAAK,UAAU,EAAE,WAAW,MAAM,WAAW,SAAS,MAAM,QAAQ,MAAM;AAAA,IAC5E;AACA,UAAM,SAAS,MAAM,MAAM,SAAS,CAAC;AACrC,QAAI,QAAQ;AACV,aAAO,cAAc,MAAM;AAC3B,aAAO,kBAAkB,MAAM;AAC/B,aAAO,cAAc,MAAM;AAAA,IAC7B;AAAA,EACF;AAEA,aAAW,SAAS,aAAa,IAAI,GAAG;AACtC,QAAI,MAAM,SAAS,QAAQ;AACzB,UAAI,aAAa,MAAM;AACrB,YAAI,MAAM,SAAS,SAAU,cAAa;AAC1C;AAAA,MACF;AACA,UAAI,qBAAqB,IAAI,MAAM,IAAI,GAAG;AACxC,mBAAW,MAAM;AACjB,oBAAY;AACZ;AAAA,MACF;AACA,UAAI,eAAe,IAAI,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI;AACnD,UAAI,MAAM,SAAS,IAAK,cAAa;AACrC,UAAI,MAAM,SAAS,KAAK;AACtB,cAAMA,SAAQ,MAAM,MAAM,SAAS,CAAC;AACpC,YAAIA,OAAO,CAAAA,OAAM,cAAc;AAAA,MACjC;AACA,UAAI,mBAAmB,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,aAAa;AAC5D,cAAM,KAAK;AAAA,UACT,MAAM,MAAM;AAAA,UACZ,WAAW,MAAM;AAAA,UACjB,WAAW,YAAY,MAAM,UAAU;AAAA,UACvC,YAAY;AAAA,UACZ,gBAAgB;AAAA,UAChB,YAAY;AAAA,QACd,CAAC;AAAA,MACH;AACA;AAAA,IACF;AAEA,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,aAAa,MAAM;AACrB,YAAI,MAAM,SAAS,UAAU;AAC3B,uBAAa;AACb,cAAI,aAAa,EAAG,YAAW;AAAA,QACjC;AACA;AAAA,MACF;AACA,UAAI,eAAe,IAAI,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI;AACnD,UAAI,MAAM,SAAS,OAAO,YAAY,EAAG,cAAa;AACtD,UAAI,mBAAmB,IAAI,MAAM,IAAI,GAAG;AAEtC,cAAM,QAAQ,MAAM,cAAc,CAACA,WAAUA,OAAM,SAAS,MAAM,IAAI;AACtE,YAAI,UAAU,IAAI;AAChB,iBAAO,MAAM,SAAS,MAAO,YAAW;AAAA,QAC1C;AAAA,MACF;AACA;AAAA,IACF;AAEA,QAAI,aAAa,KAAM;AACvB,UAAM,KAAK,MAAM,KAAK;AACtB,UAAM,QAAQ,MAAM,MAAM,SAAS,CAAC;AACpC,QAAI,CAAC,MAAO;AACZ,UAAM,SAAS,MAAM,MAAM,KAAK,EAAE;AAClC,UAAM,cAAc;AACpB,QAAI,YAAY,EAAG,OAAM,kBAAkB;AAAA,EAC7C;AAEA,SAAO,MAAM,SAAS,EAAG,YAAW;AAEpC,QAAM,SAAS,KAAK;AACpB,QAAM,WAAW,WAAW,OAAO,QAAQ,MAAM,MAAM,OAAO,WAAW,OAAO,OAAO;AACvF,SAAO,oBAAoB,SAAS,KAAK,EAAE,CAAC;AAC9C;",
+  "names": ["frame"]
+}

package/dist/extract/entities.js

@@ -0,0 +1,95 @@
+const NAMED = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+  ensp: " ",
+  emsp: " ",
+  thinsp: " ",
+  shy: "",
+  hellip: "\u2026",
+  mdash: "\u2014",
+  ndash: "\u2013",
+  lsquo: "\u2018",
+  rsquo: "\u2019",
+  ldquo: "\u201C",
+  rdquo: "\u201D",
+  laquo: "\xAB",
+  raquo: "\xBB",
+  bull: "\u2022",
+  middot: "\xB7",
+  copy: "\xA9",
+  reg: "\xAE",
+  trade: "\u2122",
+  deg: "\xB0",
+  euro: "\u20AC",
+  pound: "\xA3",
+  yen: "\xA5",
+  cent: "\xA2",
+  sect: "\xA7",
+  para: "\xB6",
+  dagger: "\u2020",
+  permil: "\u2030",
+  prime: "\u2032",
+  Prime: "\u2033",
+  times: "\xD7",
+  divide: "\xF7",
+  minus: "\u2212",
+  plusmn: "\xB1",
+  frac12: "\xBD",
+  frac14: "\xBC",
+  frac34: "\xBE",
+  larr: "\u2190",
+  rarr: "\u2192",
+  harr: "\u2194",
+  darr: "\u2193",
+  uarr: "\u2191"
+};
+function codePointToString(codePoint) {
+  if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 1114111) return "";
+  if (codePoint >= 55296 && codePoint <= 57343) return "\uFFFD";
+  return String.fromCodePoint(codePoint);
+}
+function decodeEntities(input) {
+  if (!input.includes("&")) return input;
+  let output = "";
+  let position = 0;
+  while (position < input.length) {
+    const start = input.indexOf("&", position);
+    if (start === -1) {
+      output += input.slice(position);
+      break;
+    }
+    output += input.slice(position, start);
+    const end = input.indexOf(";", start + 1);
+    if (end === -1 || end - start > 32) {
+      output += "&";
+      position = start + 1;
+      continue;
+    }
+    const reference = input.slice(start + 1, end);
+    if (reference.startsWith("#")) {
+      const isHex = reference[1] === "x" || reference[1] === "X";
+      const digits = isHex ? reference.slice(2) : reference.slice(1);
+      const valid = isHex ? /^[0-9a-fA-F]+$/.test(digits) : /^[0-9]+$/.test(digits);
+      if (valid) {
+        output += codePointToString(Number.parseInt(digits, isHex ? 16 : 10));
+        position = end + 1;
+        continue;
+      }
+    } else if (Object.hasOwn(NAMED, reference)) {
+      output += NAMED[reference];
+      position = end + 1;
+      continue;
+    }
+    output += "&";
+    position = start + 1;
+  }
+  return output;
+}
+export {
+  decodeEntities
+};
+//# sourceMappingURL=entities.js.map

package/dist/extract/entities.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extract/entities.ts"],
+  "sourcesContent": ["const NAMED: Readonly<Record<string, string>> = {\n  amp: '&',\n  lt: '<',\n  gt: '>',\n  quot: '\"',\n  apos: \"'\",\n  nbsp: ' ',\n  ensp: ' ',\n  emsp: ' ',\n  thinsp: ' ',\n  shy: '',\n  hellip: '\u2026',\n  mdash: '\u2014',\n  ndash: '\u2013',\n  lsquo: '\u2018',\n  rsquo: '\u2019',\n  ldquo: '\u201C',\n  rdquo: '\u201D',\n  laquo: '\u00AB',\n  raquo: '\u00BB',\n  bull: '\u2022',\n  middot: '\u00B7',\n  copy: '\u00A9',\n  reg: '\u00AE',\n  trade: '\u2122',\n  deg: '\u00B0',\n  euro: '\u20AC',\n  pound: '\u00A3',\n  yen: '\u00A5',\n  cent: '\u00A2',\n  sect: '\u00A7',\n  para: '\u00B6',\n  dagger: '\u2020',\n  permil: '\u2030',\n  prime: '\u2032',\n  Prime: '\u2033',\n  times: '\u00D7',\n  divide: '\u00F7',\n  minus: '\u2212',\n  plusmn: '\u00B1',\n  frac12: '\u00BD',\n  frac14: '\u00BC',\n  frac34: '\u00BE',\n  larr: '\u2190',\n  rarr: '\u2192',\n  harr: '\u2194',\n  darr: '\u2193',\n  uarr: '\u2191',\n}\n\nfunction codePointToString(codePoint: number): string {\n  if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff) return ''\n  // Surrogate halves are not valid standalone scalar values.\n  if (codePoint >= 0xd800 && codePoint <= 0xdfff) return '\uFFFD'\n  return String.fromCodePoint(codePoint)\n}\n\n/**\n * Decodes HTML character references in a single forward pass. Unterminated or\n * unknown references are emitted verbatim, matching how browsers treat them.\n */\nexport function decodeEntities(input: string): string {\n  if (!input.includes('&')) return input\n  let output = ''\n  let position = 0\n  while (position < input.length) {\n    const start = input.indexOf('&', position)\n    if (start === -1) {\n      output += input.slice(position)\n      break\n    }\n    output += input.slice(position, start)\n    const end = input.indexOf(';', start + 1)\n    if (end === -1 || end - start > 32) {\n      output += '&'\n      position = start + 1\n      continue\n    }\n    const reference = input.slice(start + 1, end)\n    if (reference.startsWith('#')) {\n      const isHex = reference[1] === 'x' || reference[1] === 'X'\n      const digits = isHex ? reference.slice(2) : reference.slice(1)\n      const valid = isHex ? /^[0-9a-fA-F]+$/.test(digits) : /^[0-9]+$/.test(digits)\n      if (valid) {\n        output += codePointToString(Number.parseInt(digits, isHex ? 16 : 10))\n        position = end + 1\n        continue\n      }\n    } else if (Object.hasOwn(NAMED, reference)) {\n      output += NAMED[reference]\n      position = end + 1\n      continue\n    }\n    output += '&'\n    position = start + 1\n  }\n  return output\n}\n"],
+  "mappings": "AAAA,MAAM,QAA0C;AAAA,EAC9C,KAAK;AAAA,EACL,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,KAAK;AAAA,EACL,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,KAAK;AAAA,EACL,OAAO;AAAA,EACP,KAAK;AAAA,EACL,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,OAAO;AAAA,EACP,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AAAA,EACN,MAAM;AACR;AAEA,SAAS,kBAAkB,WAA2B;AACpD,MAAI,CAAC,OAAO,SAAS,SAAS,KAAK,YAAY,KAAK,YAAY,QAAU,QAAO;AAEjF,MAAI,aAAa,SAAU,aAAa,MAAQ,QAAO;AACvD,SAAO,OAAO,cAAc,SAAS;AACvC;AAMO,SAAS,eAAe,OAAuB;AACpD,MAAI,CAAC,MAAM,SAAS,GAAG,EAAG,QAAO;AACjC,MAAI,SAAS;AACb,MAAI,WAAW;AACf,SAAO,WAAW,MAAM,QAAQ;AAC9B,UAAM,QAAQ,MAAM,QAAQ,KAAK,QAAQ;AACzC,QAAI,UAAU,IAAI;AAChB,gBAAU,MAAM,MAAM,QAAQ;AAC9B;AAAA,IACF;AACA,cAAU,MAAM,MAAM,UAAU,KAAK;AACrC,UAAM,MAAM,MAAM,QAAQ,KAAK,QAAQ,CAAC;AACxC,QAAI,QAAQ,MAAM,MAAM,QAAQ,IAAI;AAClC,gBAAU;AACV,iBAAW,QAAQ;AACnB;AAAA,IACF;AACA,UAAM,YAAY,MAAM,MAAM,QAAQ,GAAG,GAAG;AAC5C,QAAI,UAAU,WAAW,GAAG,GAAG;AAC7B,YAAM,QAAQ,UAAU,CAAC,MAAM,OAAO,UAAU,CAAC,MAAM;AACvD,YAAM,SAAS,QAAQ,UAAU,MAAM,CAAC,IAAI,UAAU,MAAM,CAAC;AAC7D,YAAM,QAAQ,QAAQ,iBAAiB,KAAK,MAAM,IAAI,WAAW,KAAK,MAAM;AAC5E,UAAI,OAAO;AACT,kBAAU,kBAAkB,OAAO,SAAS,QAAQ,QAAQ,KAAK,EAAE,CAAC;AACpE,mBAAW,MAAM;AACjB;AAAA,MACF;AAAA,IACF,WAAW,OAAO,OAAO,OAAO,SAAS,GAAG;AAC1C,gBAAU,MAAM,SAAS;AACzB,iBAAW,MAAM;AACjB;AAAA,IACF;AACA,cAAU;AACV,eAAW,QAAQ;AAAA,EACrB;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/extract/index.js

@@ -0,0 +1,24 @@
+import { decodeEntities } from "./entities.js";
+import { classifyPage } from "./classify.js";
+import { extractMainContent } from "./content.js";
+import {
+  BLOCK_ELEMENTS,
+  NON_CONTENT_ELEMENTS,
+  extractTitle,
+  htmlToText,
+  normalizeWhitespace
+} from "./text.js";
+import { MAX_HTML_INPUT, tokenizeHtml } from "./tokenizer.js";
+export {
+  BLOCK_ELEMENTS,
+  MAX_HTML_INPUT,
+  NON_CONTENT_ELEMENTS,
+  classifyPage,
+  decodeEntities,
+  extractMainContent,
+  extractTitle,
+  htmlToText,
+  normalizeWhitespace,
+  tokenizeHtml
+};
+//# sourceMappingURL=index.js.map

package/dist/extract/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extract/index.ts"],
+  "sourcesContent": ["export { decodeEntities } from './entities'\nexport { classifyPage, type ClassifyInput, type PageClassification } from './classify'\nexport { extractMainContent } from './content'\nexport {\n  BLOCK_ELEMENTS,\n  NON_CONTENT_ELEMENTS,\n  extractTitle,\n  htmlToText,\n  normalizeWhitespace,\n} from './text'\nexport { MAX_HTML_INPUT, tokenizeHtml, type HtmlToken } from './tokenizer'\n"],
+  "mappings": "AAAA,SAAS,sBAAsB;AAC/B,SAAS,oBAAiE;AAC1E,SAAS,0BAA0B;AACnC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,gBAAgB,oBAAoC;",
+  "names": []
+}

package/dist/extract/text.js

@@ -0,0 +1,114 @@
+import { tokenizeHtml } from "./tokenizer.js";
+const NON_CONTENT_ELEMENTS = /* @__PURE__ */ new Set([
+  "script",
+  "style",
+  "noscript",
+  "template",
+  "svg",
+  "math",
+  "title",
+  "iframe",
+  "object"
+]);
+const BLOCK_ELEMENTS = /* @__PURE__ */ new Set([
+  "address",
+  "article",
+  "aside",
+  "blockquote",
+  "br",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hr",
+  "li",
+  "main",
+  "nav",
+  "ol",
+  "p",
+  "pre",
+  "section",
+  "table",
+  "tbody",
+  "td",
+  "tfoot",
+  "th",
+  "thead",
+  "tr",
+  "ul"
+]);
+const UNICODE_SPACES = new RegExp("[\\u00A0\\u1680\\u2000-\\u200A\\u202F\\u205F\\u3000]", "g");
+function normalizeWhitespace(text) {
+  return text.replace(UNICODE_SPACES, " ").replace(/[ \t\f\r\v]+/g, " ").replace(/ ?\n ?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+function htmlToText(html) {
+  const parts = [];
+  let skipName = null;
+  let skipDepth = 0;
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === "open") {
+      if (skipName !== null) {
+        if (token.name === skipName) skipDepth += 1;
+        continue;
+      }
+      if (NON_CONTENT_ELEMENTS.has(token.name)) {
+        skipName = token.name;
+        skipDepth = 1;
+        continue;
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push("\n");
+      continue;
+    }
+    if (token.kind === "close") {
+      if (skipName !== null) {
+        if (token.name === skipName) {
+          skipDepth -= 1;
+          if (skipDepth <= 0) skipName = null;
+        }
+        continue;
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push("\n");
+      continue;
+    }
+    if (skipName !== null) continue;
+    parts.push(token.value);
+  }
+  return normalizeWhitespace(parts.join(""));
+}
+function extractTitle(html) {
+  let inTitle = false;
+  let title = "";
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === "open") {
+      if (token.name === "title") inTitle = true;
+      else if (token.name === "body") break;
+      continue;
+    }
+    if (token.kind === "close") {
+      if (token.name === "title") break;
+      continue;
+    }
+    if (inTitle) title += token.value;
+  }
+  const normalized = title.replace(/\s+/g, " ").trim();
+  return normalized.length > 0 ? normalized : null;
+}
+export {
+  BLOCK_ELEMENTS,
+  NON_CONTENT_ELEMENTS,
+  extractTitle,
+  htmlToText,
+  normalizeWhitespace
+};
+//# sourceMappingURL=text.js.map

package/dist/extract/text.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/extract/text.ts"],
+  "sourcesContent": ["import { tokenizeHtml } from './tokenizer'\n\n/** Elements whose text content is never part of the readable document. */\nexport const NON_CONTENT_ELEMENTS = new Set([\n  'script',\n  'style',\n  'noscript',\n  'template',\n  'svg',\n  'math',\n  'title',\n  'iframe',\n  'object',\n])\n\nexport const BLOCK_ELEMENTS = new Set([\n  'address',\n  'article',\n  'aside',\n  'blockquote',\n  'br',\n  'dd',\n  'div',\n  'dl',\n  'dt',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'header',\n  'hr',\n  'li',\n  'main',\n  'nav',\n  'ol',\n  'p',\n  'pre',\n  'section',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'tr',\n  'ul',\n])\n\n/** Non-breaking and typographic spaces that must fold into an ordinary space. */\nconst UNICODE_SPACES = new RegExp('[\\\\u00A0\\\\u1680\\\\u2000-\\\\u200A\\\\u202F\\\\u205F\\\\u3000]', 'g')\n\nexport function normalizeWhitespace(text: string): string {\n  return text\n    .replace(UNICODE_SPACES, ' ')\n    .replace(/[ \\t\\f\\r\\v]+/g, ' ')\n    // The collapse above leaves at most one space on either side of a newline,\n    // so ` ?` is equivalent to ` *` here \u2014 and, unlike ` *`, does not backtrack\n    // quadratically over long runs of spaces (CodeQL polynomial ReDoS).\n    .replace(/ ?\\n ?/g, '\\n')\n    .replace(/\\n{3,}/g, '\\n\\n')\n    .trim()\n}\n\nexport function htmlToText(html: string): string {\n  const parts: string[] = []\n  let skipName: string | null = null\n  let skipDepth = 0\n\n  for (const token of tokenizeHtml(html)) {\n    if (token.kind === 'open') {\n      if (skipName !== null) {\n        if (token.name === skipName) skipDepth += 1\n        continue\n      }\n      if (NON_CONTENT_ELEMENTS.has(token.name)) {\n        skipName = token.name\n        skipDepth = 1\n        continue\n      }\n      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\\n')\n      continue\n    }\n    if (token.kind === 'close') {\n      if (skipName !== null) {\n        if (token.name === skipName) {\n          skipDepth -= 1\n          if (skipDepth <= 0) skipName = null\n        }\n        continue\n      }\n      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\\n')\n      continue\n    }\n    if (skipName !== null) continue\n    parts.push(token.value)\n  }\n\n  return normalizeWhitespace(parts.join(''))\n}\n\nexport function extractTitle(html: string): string | null {\n  let inTitle = false\n  let title = ''\n  for (const token of tokenizeHtml(html)) {\n    if (token.kind === 'open') {\n      if (token.name === 'title') inTitle = true\n      else if (token.name === 'body') break\n      continue\n    }\n    if (token.kind === 'close') {\n      if (token.name === 'title') break\n      continue\n    }\n    if (inTitle) title += token.value\n  }\n  const normalized = title.replace(/\\s+/g, ' ').trim()\n  return normalized.length > 0 ? normalized : null\n}\n"],
+  "mappings": "AAAA,SAAS,oBAAoB;AAGtB,MAAM,uBAAuB,oBAAI,IAAI;AAAA,EAC1C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,MAAM,iBAAiB,oBAAI,IAAI;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAGD,MAAM,iBAAiB,IAAI,OAAO,wDAAwD,GAAG;AAEtF,SAAS,oBAAoB,MAAsB;AACxD,SAAO,KACJ,QAAQ,gBAAgB,GAAG,EAC3B,QAAQ,iBAAiB,GAAG,EAI5B,QAAQ,WAAW,IAAI,EACvB,QAAQ,WAAW,MAAM,EACzB,KAAK;AACV;AAEO,SAAS,WAAW,MAAsB;AAC/C,QAAM,QAAkB,CAAC;AACzB,MAAI,WAA0B;AAC9B,MAAI,YAAY;AAEhB,aAAW,SAAS,aAAa,IAAI,GAAG;AACtC,QAAI,MAAM,SAAS,QAAQ;AACzB,UAAI,aAAa,MAAM;AACrB,YAAI,MAAM,SAAS,SAAU,cAAa;AAC1C;AAAA,MACF;AACA,UAAI,qBAAqB,IAAI,MAAM,IAAI,GAAG;AACxC,mBAAW,MAAM;AACjB,oBAAY;AACZ;AAAA,MACF;AACA,UAAI,eAAe,IAAI,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI;AACnD;AAAA,IACF;AACA,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,aAAa,MAAM;AACrB,YAAI,MAAM,SAAS,UAAU;AAC3B,uBAAa;AACb,cAAI,aAAa,EAAG,YAAW;AAAA,QACjC;AACA;AAAA,MACF;AACA,UAAI,eAAe,IAAI,MAAM,IAAI,EAAG,OAAM,KAAK,IAAI;AACnD;AAAA,IACF;AACA,QAAI,aAAa,KAAM;AACvB,UAAM,KAAK,MAAM,KAAK;AAAA,EACxB;AAEA,SAAO,oBAAoB,MAAM,KAAK,EAAE,CAAC;AAC3C;AAEO,SAAS,aAAa,MAA6B;AACxD,MAAI,UAAU;AACd,MAAI,QAAQ;AACZ,aAAW,SAAS,aAAa,IAAI,GAAG;AACtC,QAAI,MAAM,SAAS,QAAQ;AACzB,UAAI,MAAM,SAAS,QAAS,WAAU;AAAA,eAC7B,MAAM,SAAS,OAAQ;AAChC;AAAA,IACF;AACA,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,MAAM,SAAS,QAAS;AAC5B;AAAA,IACF;AACA,QAAI,QAAS,UAAS,MAAM;AAAA,EAC9B;AACA,QAAM,aAAa,MAAM,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACnD,SAAO,WAAW,SAAS,IAAI,aAAa;AAC9C;",
+  "names": []
+}

package/dist/extract/tokenizer.js

@@ -0,0 +1,141 @@
+import { decodeEntities } from "./entities.js";
+const MAX_HTML_INPUT = 8 * 1024 * 1024;
+const VOID_ELEMENTS = /* @__PURE__ */ new Set([
+  "area",
+  "base",
+  "br",
+  "col",
+  "embed",
+  "hr",
+  "img",
+  "input",
+  "link",
+  "meta",
+  "param",
+  "source",
+  "track",
+  "wbr"
+]);
+const RAW_TEXT_ELEMENTS = /* @__PURE__ */ new Set(["script", "style"]);
+function isNameStart(character) {
+  return character >= "a" && character <= "z" || character >= "A" && character <= "Z";
+}
+function isWhitespace(character) {
+  return character === " " || character === "	" || character === "\n" || character === "\r" || character === "\f";
+}
+function readStartTag(html, start) {
+  let position = start;
+  while (position < html.length && !isWhitespace(html[position]) && html[position] !== ">" && html[position] !== "/") {
+    position += 1;
+  }
+  const name = html.slice(start, position).toLowerCase();
+  if (name.length === 0) return null;
+  const attributes = /* @__PURE__ */ new Map();
+  let selfClosing = false;
+  while (position < html.length) {
+    while (position < html.length && isWhitespace(html[position])) position += 1;
+    if (position >= html.length) break;
+    if (html[position] === ">") {
+      position += 1;
+      break;
+    }
+    if (html[position] === "/") {
+      selfClosing = true;
+      position += 1;
+      continue;
+    }
+    const nameStart = position;
+    while (position < html.length && !isWhitespace(html[position]) && html[position] !== "=" && html[position] !== ">" && html[position] !== "/") {
+      position += 1;
+    }
+    const attributeName = html.slice(nameStart, position).toLowerCase();
+    if (attributeName.length === 0) {
+      position += 1;
+      continue;
+    }
+    while (position < html.length && isWhitespace(html[position])) position += 1;
+    if (html[position] !== "=") {
+      attributes.set(attributeName, "");
+      continue;
+    }
+    position += 1;
+    while (position < html.length && isWhitespace(html[position])) position += 1;
+    const quote = html[position];
+    if (quote === '"' || quote === "'") {
+      const valueStart2 = position + 1;
+      const valueEnd = html.indexOf(quote, valueStart2);
+      if (valueEnd === -1) {
+        attributes.set(attributeName, decodeEntities(html.slice(valueStart2)));
+        position = html.length;
+        break;
+      }
+      attributes.set(attributeName, decodeEntities(html.slice(valueStart2, valueEnd)));
+      position = valueEnd + 1;
+      continue;
+    }
+    const valueStart = position;
+    while (position < html.length && !isWhitespace(html[position]) && html[position] !== ">") position += 1;
+    attributes.set(attributeName, decodeEntities(html.slice(valueStart, position)));
+  }
+  return { name, attributes, selfClosing: selfClosing || VOID_ELEMENTS.has(name), end: position };
+}
+function* tokenizeHtml(rawHtml) {
+  const html = rawHtml.length > MAX_HTML_INPUT ? rawHtml.slice(0, MAX_HTML_INPUT) : rawHtml;
+  let lowerHtml = null;
+  let position = 0;
+  while (position < html.length) {
+    const angle = html.indexOf("<", position);
+    if (angle === -1) {
+      const text = html.slice(position);
+      if (text.length > 0) yield { kind: "text", value: decodeEntities(text) };
+      return;
+    }
+    if (angle > position) {
+      yield { kind: "text", value: decodeEntities(html.slice(position, angle)) };
+    }
+    const next = html[angle + 1];
+    if (next === "!") {
+      if (html.startsWith("<!--", angle)) {
+        const close2 = html.indexOf("-->", angle + 4);
+        position = close2 === -1 ? html.length : close2 + 3;
+        continue;
+      }
+      const close = html.indexOf(">", angle + 2);
+      position = close === -1 ? html.length : close + 1;
+      continue;
+    }
+    if (next === "/") {
+      const close = html.indexOf(">", angle + 2);
+      const end = close === -1 ? html.length : close;
+      const name = html.slice(angle + 2, end).trim().toLowerCase();
+      if (name.length > 0) yield { kind: "close", name };
+      position = close === -1 ? html.length : close + 1;
+      continue;
+    }
+    if (next === void 0 || !isNameStart(next)) {
+      yield { kind: "text", value: "<" };
+      position = angle + 1;
+      continue;
+    }
+    const tag = readStartTag(html, angle + 1);
+    if (tag === null) {
+      yield { kind: "text", value: "<" };
+      position = angle + 1;
+      continue;
+    }
+    yield { kind: "open", name: tag.name, attributes: tag.attributes, selfClosing: tag.selfClosing };
+    position = tag.end;
+    if (RAW_TEXT_ELEMENTS.has(tag.name) && !tag.selfClosing) {
+      lowerHtml ??= html.toLowerCase();
+      const close = lowerHtml.indexOf(`</${tag.name}`, position);
+      position = close === -1 ? html.length : close;
+      continue;
+    }
+    if (tag.selfClosing) yield { kind: "close", name: tag.name };
+  }
+}
+export {
+  MAX_HTML_INPUT,
+  tokenizeHtml
+};
+//# sourceMappingURL=tokenizer.js.map