@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/extract/content.ts

@@ -0,0 +1,136 @@
+import { BLOCK_ELEMENTS, NON_CONTENT_ELEMENTS, normalizeWhitespace } from './text'
+import { tokenizeHtml } from './tokenizer'
+
+const CANDIDATE_ELEMENTS = new Set(['article', 'main', 'section', 'div', 'td', 'body'])
+const SEMANTIC_BONUS = new Set(['article', 'main'])
+
+const NEGATIVE_HINT =
+  /(^|[\s_-])(nav|navigation|menu|sidebar|side-bar|footer|masthead|comment|share|social|advert|promo|banner|cookie|consent|related|breadcrumb|pagination|widget|newsletter|subscribe)([\s_-]|$)/i
+const POSITIVE_HINT = /(^|[\s_-])(article|content|post|story|entry|body-?text|main)([\s_-]|$)/i
+
+/** Below this a candidate is treated as chrome rather than an article. */
+const MIN_CONTENT_CHARS = 200
+
+type BestRange = {
+  readonly startPart: number
+  readonly endPart: number
+  readonly score: number
+}
+
+type Frame = {
+  readonly name: string
+  readonly startPart: number
+  readonly hintScore: number
+  textLength: number
+  linkTextLength: number
+  paragraphs: number
+}
+
+function hintScoreOf(attributes: ReadonlyMap<string, string>): number {
+  const descriptor = `${attributes.get('class') ?? ''} ${attributes.get('id') ?? ''} ${attributes.get('role') ?? ''}`
+  if (descriptor.trim().length === 0) return 0
+  let score = 0
+  if (NEGATIVE_HINT.test(descriptor)) score -= 600
+  if (POSITIVE_HINT.test(descriptor)) score += 300
+  return score
+}
+
+function scoreOf(frame: Frame): number {
+  const semantic = SEMANTIC_BONUS.has(frame.name) ? 400 : 0
+  return frame.textLength - 2 * frame.linkTextLength + 25 * frame.paragraphs + frame.hintScore + semantic
+}
+
+/**
+ * Picks the subtree that reads like the article, using text-to-link density plus
+ * paragraph count and class/id hints. Counters are accumulated during the single
+ * tokenizer pass and only the winning range is materialized, so the scan stays
+ * linear in time and the output allocates once.
+ */
+export function extractMainContent(html: string): string {
+  const parts: string[] = []
+  const stack: Frame[] = []
+  const best: { current: BestRange | null } = { current: null }
+  let linkDepth = 0
+  let skipName: string | null = null
+  let skipDepth = 0
+
+  const closeFrame = (): void => {
+    const frame = stack.pop()
+    if (!frame) return
+    const score = scoreOf(frame)
+    if (frame.textLength >= MIN_CONTENT_CHARS && (best.current === null || score > best.current.score)) {
+      best.current = { startPart: frame.startPart, endPart: parts.length, score }
+    }
+    const parent = stack[stack.length - 1]
+    if (parent) {
+      parent.textLength += frame.textLength
+      parent.linkTextLength += frame.linkTextLength
+      parent.paragraphs += frame.paragraphs
+    }
+  }
+
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === 'open') {
+      if (skipName !== null) {
+        if (token.name === skipName) skipDepth += 1
+        continue
+      }
+      if (NON_CONTENT_ELEMENTS.has(token.name)) {
+        skipName = token.name
+        skipDepth = 1
+        continue
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\n')
+      if (token.name === 'a') linkDepth += 1
+      if (token.name === 'p') {
+        const frame = stack[stack.length - 1]
+        if (frame) frame.paragraphs += 1
+      }
+      if (CANDIDATE_ELEMENTS.has(token.name) && !token.selfClosing) {
+        stack.push({
+          name: token.name,
+          startPart: parts.length,
+          hintScore: hintScoreOf(token.attributes),
+          textLength: 0,
+          linkTextLength: 0,
+          paragraphs: 0,
+        })
+      }
+      continue
+    }
+
+    if (token.kind === 'close') {
+      if (skipName !== null) {
+        if (token.name === skipName) {
+          skipDepth -= 1
+          if (skipDepth <= 0) skipName = null
+        }
+        continue
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\n')
+      if (token.name === 'a' && linkDepth > 0) linkDepth -= 1
+      if (CANDIDATE_ELEMENTS.has(token.name)) {
+        // Unbalanced markup is common; only unwind to the nearest matching frame.
+        const index = stack.findLastIndex((frame) => frame.name === token.name)
+        if (index !== -1) {
+          while (stack.length > index) closeFrame()
+        }
+      }
+      continue
+    }
+
+    if (skipName !== null) continue
+    parts.push(token.value)
+    const frame = stack[stack.length - 1]
+    if (!frame) continue
+    const length = token.value.trim().length
+    frame.textLength += length
+    if (linkDepth > 0) frame.linkTextLength += length
+  }
+
+  while (stack.length > 0) closeFrame()
+
+  const winner = best.current
+  const selected = winner === null ? parts : parts.slice(winner.startPart, winner.endPart)
+  return normalizeWhitespace(selected.join(''))
+}

package/src/extract/entities.ts

@@ -0,0 +1,98 @@
+const NAMED: Readonly<Record<string, string>> = {
+  amp: '&',
+  lt: '<',
+  gt: '>',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+  ensp: ' ',
+  emsp: ' ',
+  thinsp: ' ',
+  shy: '',
+  hellip: '…',
+  mdash: '—',
+  ndash: '–',
+  lsquo: '‘',
+  rsquo: '’',
+  ldquo: '“',
+  rdquo: '”',
+  laquo: '«',
+  raquo: '»',
+  bull: '•',
+  middot: '·',
+  copy: '©',
+  reg: '®',
+  trade: '™',
+  deg: '°',
+  euro: '€',
+  pound: '£',
+  yen: '¥',
+  cent: '¢',
+  sect: '§',
+  para: '¶',
+  dagger: '†',
+  permil: '‰',
+  prime: '′',
+  Prime: '″',
+  times: '×',
+  divide: '÷',
+  minus: '−',
+  plusmn: '±',
+  frac12: '½',
+  frac14: '¼',
+  frac34: '¾',
+  larr: '←',
+  rarr: '→',
+  harr: '↔',
+  darr: '↓',
+  uarr: '↑',
+}
+
+function codePointToString(codePoint: number): string {
+  if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff) return ''
+  // Surrogate halves are not valid standalone scalar values.
+  if (codePoint >= 0xd800 && codePoint <= 0xdfff) return '�'
+  return String.fromCodePoint(codePoint)
+}
+
+/**
+ * Decodes HTML character references in a single forward pass. Unterminated or
+ * unknown references are emitted verbatim, matching how browsers treat them.
+ */
+export function decodeEntities(input: string): string {
+  if (!input.includes('&')) return input
+  let output = ''
+  let position = 0
+  while (position < input.length) {
+    const start = input.indexOf('&', position)
+    if (start === -1) {
+      output += input.slice(position)
+      break
+    }
+    output += input.slice(position, start)
+    const end = input.indexOf(';', start + 1)
+    if (end === -1 || end - start > 32) {
+      output += '&'
+      position = start + 1
+      continue
+    }
+    const reference = input.slice(start + 1, end)
+    if (reference.startsWith('#')) {
+      const isHex = reference[1] === 'x' || reference[1] === 'X'
+      const digits = isHex ? reference.slice(2) : reference.slice(1)
+      const valid = isHex ? /^[0-9a-fA-F]+$/.test(digits) : /^[0-9]+$/.test(digits)
+      if (valid) {
+        output += codePointToString(Number.parseInt(digits, isHex ? 16 : 10))
+        position = end + 1
+        continue
+      }
+    } else if (Object.hasOwn(NAMED, reference)) {
+      output += NAMED[reference]
+      position = end + 1
+      continue
+    }
+    output += '&'
+    position = start + 1
+  }
+  return output
+}

package/src/extract/index.ts

@@ -0,0 +1,11 @@
+export { decodeEntities } from './entities'
+export { classifyPage, type ClassifyInput, type PageClassification } from './classify'
+export { extractMainContent } from './content'
+export {
+  BLOCK_ELEMENTS,
+  NON_CONTENT_ELEMENTS,
+  extractTitle,
+  htmlToText,
+  normalizeWhitespace,
+} from './text'
+export { MAX_HTML_INPUT, tokenizeHtml, type HtmlToken } from './tokenizer'

package/src/extract/text.ts

@@ -0,0 +1,124 @@
+import { tokenizeHtml } from './tokenizer'
+
+/** Elements whose text content is never part of the readable document. */
+export const NON_CONTENT_ELEMENTS = new Set([
+  'script',
+  'style',
+  'noscript',
+  'template',
+  'svg',
+  'math',
+  'title',
+  'iframe',
+  'object',
+])
+
+export const BLOCK_ELEMENTS = new Set([
+  'address',
+  'article',
+  'aside',
+  'blockquote',
+  'br',
+  'dd',
+  'div',
+  'dl',
+  'dt',
+  'figcaption',
+  'figure',
+  'footer',
+  'form',
+  'h1',
+  'h2',
+  'h3',
+  'h4',
+  'h5',
+  'h6',
+  'header',
+  'hr',
+  'li',
+  'main',
+  'nav',
+  'ol',
+  'p',
+  'pre',
+  'section',
+  'table',
+  'tbody',
+  'td',
+  'tfoot',
+  'th',
+  'thead',
+  'tr',
+  'ul',
+])
+
+/** Non-breaking and typographic spaces that must fold into an ordinary space. */
+const UNICODE_SPACES = new RegExp('[\\u00A0\\u1680\\u2000-\\u200A\\u202F\\u205F\\u3000]', 'g')
+
+export function normalizeWhitespace(text: string): string {
+  return text
+    .replace(UNICODE_SPACES, ' ')
+    .replace(/[ \t\f\r\v]+/g, ' ')
+    // The collapse above leaves at most one space on either side of a newline,
+    // so ` ?` is equivalent to ` *` here — and, unlike ` *`, does not backtrack
+    // quadratically over long runs of spaces (CodeQL polynomial ReDoS).
+    .replace(/ ?\n ?/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+}
+
+export function htmlToText(html: string): string {
+  const parts: string[] = []
+  let skipName: string | null = null
+  let skipDepth = 0
+
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === 'open') {
+      if (skipName !== null) {
+        if (token.name === skipName) skipDepth += 1
+        continue
+      }
+      if (NON_CONTENT_ELEMENTS.has(token.name)) {
+        skipName = token.name
+        skipDepth = 1
+        continue
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\n')
+      continue
+    }
+    if (token.kind === 'close') {
+      if (skipName !== null) {
+        if (token.name === skipName) {
+          skipDepth -= 1
+          if (skipDepth <= 0) skipName = null
+        }
+        continue
+      }
+      if (BLOCK_ELEMENTS.has(token.name)) parts.push('\n')
+      continue
+    }
+    if (skipName !== null) continue
+    parts.push(token.value)
+  }
+
+  return normalizeWhitespace(parts.join(''))
+}
+
+export function extractTitle(html: string): string | null {
+  let inTitle = false
+  let title = ''
+  for (const token of tokenizeHtml(html)) {
+    if (token.kind === 'open') {
+      if (token.name === 'title') inTitle = true
+      else if (token.name === 'body') break
+      continue
+    }
+    if (token.kind === 'close') {
+      if (token.name === 'title') break
+      continue
+    }
+    if (inTitle) title += token.value
+  }
+  const normalized = title.replace(/\s+/g, ' ').trim()
+  return normalized.length > 0 ? normalized : null
+}

package/src/extract/tokenizer.ts

@@ -0,0 +1,197 @@
+import { decodeEntities } from './entities'
+
+export type HtmlToken =
+  | { readonly kind: 'text'; readonly value: string }
+  | {
+      readonly kind: 'open'
+      readonly name: string
+      readonly attributes: ReadonlyMap<string, string>
+      readonly selfClosing: boolean
+    }
+  | { readonly kind: 'close'; readonly name: string }
+
+/**
+ * Bounds memory for a pathological document. The tokenizer itself is linear and
+ * cannot stall the event loop, so this is an allocation guard rather than the
+ * ReDoS mitigation a regex-based extractor would need it to be.
+ */
+export const MAX_HTML_INPUT = 8 * 1024 * 1024
+
+const VOID_ELEMENTS = new Set([
+  'area',
+  'base',
+  'br',
+  'col',
+  'embed',
+  'hr',
+  'img',
+  'input',
+  'link',
+  'meta',
+  'param',
+  'source',
+  'track',
+  'wbr',
+])
+
+const RAW_TEXT_ELEMENTS = new Set(['script', 'style'])
+
+function isNameStart(character: string): boolean {
+  return (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')
+}
+
+function isWhitespace(character: string): boolean {
+  return character === ' ' || character === '\t' || character === '\n' || character === '\r' || character === '\f'
+}
+
+type ParsedTag = {
+  readonly name: string
+  readonly attributes: Map<string, string>
+  readonly selfClosing: boolean
+  readonly end: number
+}
+
+/**
+ * Reads one start tag beginning at the character after `<`. Quote-aware so an
+ * attribute value containing `>` does not truncate the tag.
+ */
+function readStartTag(html: string, start: number): ParsedTag | null {
+  let position = start
+  while (position < html.length && !isWhitespace(html[position]) && html[position] !== '>' && html[position] !== '/') {
+    position += 1
+  }
+  const name = html.slice(start, position).toLowerCase()
+  if (name.length === 0) return null
+
+  const attributes = new Map<string, string>()
+  let selfClosing = false
+
+  while (position < html.length) {
+    while (position < html.length && isWhitespace(html[position])) position += 1
+    if (position >= html.length) break
+    if (html[position] === '>') {
+      position += 1
+      break
+    }
+    if (html[position] === '/') {
+      selfClosing = true
+      position += 1
+      continue
+    }
+
+    const nameStart = position
+    while (
+      position < html.length &&
+      !isWhitespace(html[position]) &&
+      html[position] !== '=' &&
+      html[position] !== '>' &&
+      html[position] !== '/'
+    ) {
+      position += 1
+    }
+    const attributeName = html.slice(nameStart, position).toLowerCase()
+    if (attributeName.length === 0) {
+      position += 1
+      continue
+    }
+
+    while (position < html.length && isWhitespace(html[position])) position += 1
+    if (html[position] !== '=') {
+      attributes.set(attributeName, '')
+      continue
+    }
+    position += 1
+    while (position < html.length && isWhitespace(html[position])) position += 1
+
+    const quote = html[position]
+    if (quote === '"' || quote === "'") {
+      const valueStart = position + 1
+      const valueEnd = html.indexOf(quote, valueStart)
+      if (valueEnd === -1) {
+        attributes.set(attributeName, decodeEntities(html.slice(valueStart)))
+        position = html.length
+        break
+      }
+      attributes.set(attributeName, decodeEntities(html.slice(valueStart, valueEnd)))
+      position = valueEnd + 1
+      continue
+    }
+
+    const valueStart = position
+    while (position < html.length && !isWhitespace(html[position]) && html[position] !== '>') position += 1
+    attributes.set(attributeName, decodeEntities(html.slice(valueStart, position)))
+  }
+
+  return { name, attributes, selfClosing: selfClosing || VOID_ELEMENTS.has(name), end: position }
+}
+
+/**
+ * Single forward scan over HTML emitting structural tokens. Every advance is
+ * driven by `indexOf` or a bounded character walk, so there is no backtracking
+ * and runtime is linear in the input length.
+ */
+export function* tokenizeHtml(rawHtml: string): Generator<HtmlToken> {
+  const html = rawHtml.length > MAX_HTML_INPUT ? rawHtml.slice(0, MAX_HTML_INPUT) : rawHtml
+  // Lowercased once: searching for a raw-text close tag inside the loop would
+  // otherwise copy the whole document per <script>/<style>.
+  let lowerHtml: string | null = null
+  let position = 0
+
+  while (position < html.length) {
+    const angle = html.indexOf('<', position)
+    if (angle === -1) {
+      const text = html.slice(position)
+      if (text.length > 0) yield { kind: 'text', value: decodeEntities(text) }
+      return
+    }
+    if (angle > position) {
+      yield { kind: 'text', value: decodeEntities(html.slice(position, angle)) }
+    }
+
+    const next = html[angle + 1]
+
+    if (next === '!') {
+      if (html.startsWith('<!--', angle)) {
+        const close = html.indexOf('-->', angle + 4)
+        position = close === -1 ? html.length : close + 3
+        continue
+      }
+      const close = html.indexOf('>', angle + 2)
+      position = close === -1 ? html.length : close + 1
+      continue
+    }
+
+    if (next === '/') {
+      const close = html.indexOf('>', angle + 2)
+      const end = close === -1 ? html.length : close
+      const name = html.slice(angle + 2, end).trim().toLowerCase()
+      if (name.length > 0) yield { kind: 'close', name }
+      position = close === -1 ? html.length : close + 1
+      continue
+    }
+
+    if (next === undefined || !isNameStart(next)) {
+      yield { kind: 'text', value: '<' }
+      position = angle + 1
+      continue
+    }
+
+    const tag = readStartTag(html, angle + 1)
+    if (tag === null) {
+      yield { kind: 'text', value: '<' }
+      position = angle + 1
+      continue
+    }
+    yield { kind: 'open', name: tag.name, attributes: tag.attributes, selfClosing: tag.selfClosing }
+    position = tag.end
+
+    if (RAW_TEXT_ELEMENTS.has(tag.name) && !tag.selfClosing) {
+      lowerHtml ??= html.toLowerCase()
+      const close = lowerHtml.indexOf(`</${tag.name}`, position)
+      position = close === -1 ? html.length : close
+      continue
+    }
+
+    if (tag.selfClosing) yield { kind: 'close', name: tag.name }
+  }
+}