@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/engine/index.ts

@@ -0,0 +1,22 @@
+export {
+  buildCacheKey,
+  createSingleFlight,
+  type CacheEntry,
+  type ResultCache,
+} from './cache'
+export { DEADLINE_REACHED, deadlineRace, linkSignals, sleep, type Cancellable } from './deadline'
+export { createSearchEngine } from './engine'
+export { outcomeReason, outcomeToStatus, runAdapter, type AdapterRunResult } from './runAdapter'
+export type {
+  AdapterDiagnostic,
+  AdapterDiagnosticStatus,
+  AdapterHealthReport,
+  AdapterResultSet,
+  EngineAdapterEntry,
+  HealthOptions,
+  RunOptions,
+  SearchDiagnostics,
+  SearchEngine,
+  SearchEngineOptions,
+  SearchEngineResult,
+} from './types'

package/src/engine/runAdapter.ts

@@ -0,0 +1,89 @@
+import type { AdapterContext } from '../contract/adapter'
+import { errorMessage } from '../contract/errors'
+import type { HttpClient, Logger } from '../contract/http'
+import { failed, type SearchOutcome } from '../contract/outcomes'
+import type { SearchRequest } from '../contract/results'
+import type { SearchStepEvent, SearchStepMetrics } from '../contract/steps'
+import { linkSignals } from './deadline'
+import type { AdapterDiagnosticStatus, EngineAdapterEntry } from './types'
+
+export type AdapterRunResult = {
+  readonly entry: EngineAdapterEntry
+  readonly outcome: SearchOutcome
+  readonly latencyMs: number
+}
+
+export type AdapterRunDeps = {
+  readonly http: HttpClient
+  readonly logger: Logger
+  readonly now: () => number
+  readonly report: (adapterId: string, event: SearchStepEvent, detail?: string, metrics?: SearchStepMetrics) => void
+  readonly parentSignal: AbortSignal
+  readonly deadlineAt: number
+  readonly adapterTimeoutMs: number
+}
+
+export function outcomeToStatus(outcome: SearchOutcome): AdapterDiagnosticStatus {
+  switch (outcome.status) {
+    case 'ok':
+      return 'ok'
+    case 'empty':
+      return 'empty'
+    case 'unavailable':
+      return 'unavailable'
+    case 'blocked':
+      return 'blocked'
+    case 'timeout':
+      return 'timeout'
+    case 'error':
+      return 'error'
+  }
+}
+
+export function outcomeReason(outcome: SearchOutcome): string | undefined {
+  if (outcome.status === 'ok' || outcome.status === 'empty') return undefined
+  return outcome.reason
+}
+
+/**
+ * Runs one adapter under its own timeout and abort linkage.
+ *
+ * The contract says adapters return outcomes rather than throwing, but a
+ * third-party adapter is not trusted to honour that — a throw here is converted
+ * into an `error` outcome so one misbehaving package cannot abort the whole run.
+ */
+export async function runAdapter(
+  entry: EngineAdapterEntry,
+  request: SearchRequest,
+  deps: AdapterRunDeps,
+): Promise<AdapterRunResult> {
+  const startedAt = deps.now()
+  const allowance = entry.timeoutMs ?? deps.adapterTimeoutMs
+  const budget = Math.max(0, Math.min(allowance, deps.deadlineAt - startedAt))
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), budget)
+  timer.unref?.()
+  const linked = linkSignals([deps.parentSignal, controller.signal])
+
+  const context: AdapterContext = {
+    signal: linked.signal,
+    http: deps.http,
+    report: (event, detail, metrics) => deps.report(entry.adapter.id, event, detail, metrics),
+    deadlineAt: startedAt + budget,
+    logger: deps.logger,
+  }
+
+  deps.report(entry.adapter.id, 'started')
+
+  try {
+    const outcome = await entry.adapter.search(request, context)
+    return { entry, outcome, latencyMs: deps.now() - startedAt }
+  } catch (error) {
+    const detail = `adapter threw instead of returning an outcome: ${errorMessage(error)}`
+    deps.logger.warn(`[web-research] ${entry.adapter.id} ${detail}`)
+    return { entry, outcome: failed(detail, false), latencyMs: deps.now() - startedAt }
+  } finally {
+    clearTimeout(timer)
+    linked.dispose()
+  }
+}

package/src/engine/types.ts

@@ -0,0 +1,137 @@
+import type { AdapterHealth, ProbeCost, SearchAdapter } from '../contract/adapter'
+import type { HttpClient, Logger } from '../contract/http'
+import type { FetchOutcome } from '../contract/outcomes'
+import type { SearchPolicy } from '../contract/policy'
+import type { FetchRequest, RawResult, SearchRequestInput, SearchResult } from '../contract/results'
+import type { StepSink } from '../contract/steps'
+import type { ResultCache } from './cache'
+
+export type AdapterDiagnosticStatus =
+  | 'ok'
+  | 'empty'
+  | 'unavailable'
+  | 'blocked'
+  | 'timeout'
+  | 'error'
+  | 'skipped'
+  | 'cancelled'
+
+export type AdapterDiagnostic = {
+  readonly id: string
+  readonly status: AdapterDiagnosticStatus
+  readonly latencyMs: number
+  readonly resultCount: number
+  readonly reason?: string
+}
+
+export type SearchDiagnostics = {
+  readonly adapters: readonly AdapterDiagnostic[]
+  readonly degraded: boolean
+  readonly cached: boolean
+  readonly escalated: boolean
+  readonly elapsedMs: number
+  /** Pages fetched for inline content; what the host charges a fetch budget for. */
+  readonly pagesRead: number
+}
+
+/**
+ * One adapter's own answer, before fusion touched it.
+ *
+ * Fusion is lossy on purpose: dedup, the per-domain cap and the limit all drop
+ * hits, and only the first prose answer survives. That is right for an agent,
+ * which wants one ranked list rather than N to reconcile — but it leaves an
+ * operator comparing sources with nothing to compare. This is that view, and it
+ * is opt-in for the same reason it is not the default.
+ */
+export type AdapterResultSet = {
+  readonly adapterId: string
+  readonly weight: number
+  readonly answer: string | null
+  /** Exactly what the adapter returned, in its own order. */
+  readonly results: readonly RawResult[]
+}
+
+export type SearchEngineResult = {
+  readonly results: readonly SearchResult[]
+  /** Prose synthesis when an adapter produced one; never fabricated by the engine. */
+  readonly answer: string | null
+  readonly diagnostics: SearchDiagnostics
+  /** Present only when the caller asked for `includeAdapterResults`. */
+  readonly byAdapter?: readonly AdapterResultSet[]
+}
+
+export type AdapterHealthReport = AdapterHealth & {
+  readonly id: string
+  readonly ready: boolean
+  /** What probing this adapter costs, resolved through the engine's default. */
+  readonly probeCost?: ProbeCost
+  /** False when the row reports configuration only, with no call made. */
+  readonly probed?: boolean
+}
+
+export type EngineAdapterEntry = {
+  readonly adapter: SearchAdapter
+  readonly weight: number
+  readonly order: number
+  /**
+   * Whether the normal waves may use it. A disabled adapter is still reachable
+   * as `policy.lastResort`, which is what lets an operator keep an expensive
+   * source out of every query while retaining it as the safety net.
+   */
+  readonly enabled: boolean
+  /** Overrides `policy.adapterTimeoutMs` for this adapter alone. */
+  readonly timeoutMs?: number
+}
+
+export type SearchEngineOptions = {
+  readonly policy: SearchPolicy
+  readonly adapters: readonly EngineAdapterEntry[]
+  readonly http: HttpClient
+  readonly logger?: Logger
+  readonly cache?: ResultCache<SearchEngineResult>
+  readonly onStep?: StepSink
+  readonly now?: () => number
+}
+
+export type RunOptions = {
+  readonly signal?: AbortSignal
+  readonly onStep?: StepSink
+  /**
+   * Also return each adapter's untouched result list and prose. An operator
+   * comparison aid, deliberately separate from `settleMode`: that decides when
+   * to stop, this decides what to hand back. Pair it with `exhaustive` or the
+   * scheduler will cancel the very adapters you meant to compare.
+   */
+  readonly includeAdapterResults?: boolean
+}
+
+export type HealthOptions = RunOptions & {
+  /**
+   * Whether to actually call `healthCheck()` on each adapter.
+   *
+   * Off means readiness only, which is synchronous and I/O-free. A live probe is
+   * not free for every adapter — a metered source spends a real search credit and
+   * the browser tier spawns a process — so a screen that merely displays status
+   * must be able to ask without billing anyone.
+   */
+  readonly probe?: boolean
+  /**
+   * Ceiling on what may be spent probing. `'free'` calls only the adapters whose
+   * probe costs nothing; the rest come back `probed: false` with their readiness
+   * intact. Ignored when `probe` is false. Defaults to `'billable'`, which is the
+   * behaviour every existing caller already gets.
+   */
+  readonly maxProbeCost?: ProbeCost
+  /**
+   * Restrict probing to these adapter ids. Everything else still reports
+   * readiness, so a targeted re-test of one adapter cannot bill the others.
+   */
+  readonly only?: readonly string[]
+}
+
+export interface SearchEngine {
+  search(input: SearchRequestInput, options?: RunOptions): Promise<SearchEngineResult>
+  fetch(request: FetchRequest, options?: RunOptions): Promise<FetchOutcome>
+  health(options?: HealthOptions): Promise<readonly AdapterHealthReport[]>
+  dispose(): Promise<void>
+}

package/src/extract/__tests__/content.test.ts

@@ -0,0 +1,81 @@
+import { classifyPage } from '../classify'
+import { extractMainContent } from '../content'
+
+const paragraph = (marker: string): string =>
+  `<p>${marker} ${'lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod. '.repeat(4)}</p>`
+
+describe('extractMainContent', () => {
+  it('prefers the article over navigation chrome', () => {
+    const html = `
+      <body>
+        <nav class="site-nav"><a href="/a">Home</a><a href="/b">Products</a><a href="/c">Pricing</a></nav>
+        <div class="sidebar"><a href="/x">Related one</a><a href="/y">Related two</a></div>
+        <article class="post-body">${paragraph('ARTICLE')}${paragraph('ARTICLE')}</article>
+        <footer class="site-footer"><a href="/tos">Terms</a></footer>
+      </body>`
+    const text = extractMainContent(html)
+    expect(text).toContain('ARTICLE')
+    expect(text).not.toContain('Terms')
+    expect(text).not.toContain('Pricing')
+  })
+
+  it('penalizes a link-dense block against a prose block of similar length', () => {
+    const links = `<div class="menu">${'<a href="/l">navigation link text here</a>'.repeat(30)}</div>`
+    const prose = `<div class="entry-content">${paragraph('PROSE')}</div>`
+    const text = extractMainContent(`<body>${links}${prose}</body>`)
+    expect(text).toContain('PROSE')
+  })
+
+  it('falls back to the whole document when nothing scores well', () => {
+    expect(extractMainContent('<body><p>tiny</p></body>')).toBe('tiny')
+  })
+
+  it('tolerates unbalanced markup', () => {
+    const html = `<body><div class="content"><article>${paragraph('X')}</div></body>`
+    expect(() => extractMainContent(html)).not.toThrow()
+    expect(extractMainContent(html)).toContain('X')
+  })
+})
+
+describe('classifyPage', () => {
+  const base = { status: 200, contentType: 'text/html' }
+
+  it('accepts a page with real text', () => {
+    const text = 'a'.repeat(500)
+    expect(classifyPage({ ...base, html: `<p>${text}</p>`, text }).verdict).toBe('ok')
+  })
+
+  it.each([403, 429, 503])('treats HTTP %s as blocked', (status) => {
+    expect(classifyPage({ ...base, status, html: '<html></html>', text: '' }).verdict).toBe('blocked')
+  })
+
+  it('detects an anti-bot interstitial', () => {
+    const html = '<html><body><h1>Just a moment...</h1><div id="cf_chl_opt"></div></body></html>'
+    expect(classifyPage({ ...base, html, text: 'Just a moment...' }).verdict).toBe('blocked')
+  })
+
+  // Observed against a real LinkedIn company page: 12,987 chars of usable text,
+  // but the word "captcha" appears in a bundled script. Flagging that as blocked
+  // triggers a browser render for a page that already worked.
+  it('does not call a content-bearing page blocked just because its markup mentions a challenge', () => {
+    const text = 'Stripe. Technology, Information and Internet. '.repeat(40)
+    const html = `<html><body><script>var x="captcha";</script><article><p>${text}</p></article></body></html>`
+    expect(classifyPage({ ...base, html, text }).verdict).toBe('ok')
+  })
+
+  it('detects a client-side render shell', () => {
+    const html = `<html><body><div id="root"></div>${'<script>x</script>'.repeat(400)}</body></html>`
+    expect(classifyPage({ ...base, html, text: '' }).verdict).toBe('js-shell')
+  })
+
+  it('detects a JavaScript-required notice', () => {
+    const html = '<html><body><noscript>You need to enable JavaScript to run this app.</noscript></body></html>'
+    const result = classifyPage({ ...base, html, text: '' })
+    expect(result.verdict).toBe('js-shell')
+    expect(result.reason).toContain('JavaScript')
+  })
+
+  it('reports an empty body as empty rather than blocked', () => {
+    expect(classifyPage({ ...base, html: '', text: '' }).verdict).toBe('empty')
+  })
+})

package/src/extract/__tests__/text.test.ts

@@ -0,0 +1,61 @@
+import { decodeEntities } from '../entities'
+import { extractTitle, htmlToText, normalizeWhitespace } from '../text'
+
+describe('decodeEntities', () => {
+  it('decodes named, decimal and hex references', () => {
+    expect(decodeEntities('a&b A B')).toBe('a&b A B')
+  })
+
+  it('leaves unknown or unterminated references alone', () => {
+    expect(decodeEntities('&notareal; &amp')).toBe('&notareal; &amp')
+  })
+
+  it('maps nbsp to a plain space', () => {
+    expect(decodeEntities('a b')).toBe('a b')
+  })
+
+  it('rejects lone surrogates', () => {
+    expect(decodeEntities('�')).toBe('�')
+  })
+})
+
+describe('htmlToText', () => {
+  it('strips markup and keeps readable text', () => {
+    expect(htmlToText('<p>Hello <b>world</b></p>')).toBe('Hello world')
+  })
+
+  it('drops script and style content', () => {
+    const html = '<style>.a{color:red}</style><script>alert(1)</script><p>Body</p>'
+    expect(htmlToText(html)).toBe('Body')
+  })
+
+  it('does not leak the document title into body text', () => {
+    expect(htmlToText('<html><head><title>Tab</title></head><body><p>Body</p></body></html>')).toBe('Body')
+  })
+
+  it('separates paragraphs with a blank line', () => {
+    expect(htmlToText('<p>one</p><p>two</p>')).toBe('one\n\ntwo')
+  })
+
+  it('keeps list items on separate lines', () => {
+    expect(htmlToText('<ul><li>a</li><li>b</li></ul>')).toBe('a\n\nb')
+  })
+
+  it('collapses runs of whitespace', () => {
+    expect(normalizeWhitespace('a   \t b\n\n\n\nc')).toBe('a b\n\nc')
+  })
+})
+
+describe('extractTitle', () => {
+  it('reads the document title', () => {
+    expect(extractTitle('<html><head><title> Hello  World </title></head></html>')).toBe('Hello World')
+  })
+
+  it('returns null when there is no title', () => {
+    expect(extractTitle('<html><body><p>x</p></body></html>')).toBeNull()
+  })
+
+  it('does not scan past the start of the body', () => {
+    expect(extractTitle('<body><title>late</title></body>')).toBeNull()
+  })
+})

package/src/extract/__tests__/tokenizer.test.ts

@@ -0,0 +1,85 @@
+import { tokenizeHtml, type HtmlToken } from '../tokenizer'
+
+const collect = (html: string): HtmlToken[] => [...tokenizeHtml(html)]
+
+describe('tokenizeHtml', () => {
+  it('emits open, text and close tokens in document order', () => {
+    expect(collect('<p>hi</p>')).toEqual([
+      { kind: 'open', name: 'p', attributes: new Map(), selfClosing: false },
+      { kind: 'text', value: 'hi' },
+      { kind: 'close', name: 'p' },
+    ])
+  })
+
+  it('lowercases tag and attribute names', () => {
+    const [token] = collect('<DIV CLASS="Body">')
+    expect(token).toMatchObject({ kind: 'open', name: 'div' })
+    if (token.kind !== 'open') throw new Error('expected an open token')
+    expect(token.attributes.get('class')).toBe('Body')
+  })
+
+  it('parses single-quoted, double-quoted and bare attribute values', () => {
+    const [token] = collect(`<a href='/x' title="y" data-n=3>`)
+    if (token.kind !== 'open') throw new Error('expected an open token')
+    expect(token.attributes.get('href')).toBe('/x')
+    expect(token.attributes.get('title')).toBe('y')
+    expect(token.attributes.get('data-n')).toBe('3')
+  })
+
+  it('does not truncate a tag on a > inside a quoted attribute value', () => {
+    const [token] = collect('<a title="a > b" href="/ok">text</a>')
+    if (token.kind !== 'open') throw new Error('expected an open token')
+    expect(token.attributes.get('title')).toBe('a > b')
+    expect(token.attributes.get('href')).toBe('/ok')
+  })
+
+  it('decodes entities in attribute values and text', () => {
+    const tokens = collect('<a title="a&amp;b">x&lt;y</a>')
+    const [open, text] = tokens
+    if (open.kind !== 'open') throw new Error('expected an open token')
+    expect(open.attributes.get('title')).toBe('a&b')
+    expect(text).toEqual({ kind: 'text', value: 'x<y' })
+  })
+
+  it('skips comments and doctypes', () => {
+    expect(collect('<!DOCTYPE html><!-- note --><p>a</p>').map((token) => token.kind)).toEqual([
+      'open',
+      'text',
+      'close',
+    ])
+  })
+
+  it('drops raw text inside script and style', () => {
+    const tokens = collect('<script>var a = "<p>no</p>";</script><p>yes</p>')
+    const text = tokens.filter((token) => token.kind === 'text').map((token) => token.value)
+    expect(text).toEqual(['yes'])
+  })
+
+  it('synthesizes a close token for void elements', () => {
+    expect(collect('<br>').map((token) => token.kind)).toEqual(['open', 'close'])
+  })
+
+  it('treats a stray angle bracket as text', () => {
+    expect(collect('5 < 7')).toEqual([
+      { kind: 'text', value: '5 ' },
+      { kind: 'text', value: '<' },
+      { kind: 'text', value: ' 7' },
+    ])
+  })
+
+  it('survives unterminated markup without hanging', () => {
+    expect(() => collect('<div class="x')).not.toThrow()
+    expect(() => collect('<!-- never closed')).not.toThrow()
+    expect(() => collect('<script>forever')).not.toThrow()
+  })
+
+  it('runs in linear time on adversarial nesting', () => {
+    // A regex-based extractor with lazy quantifiers stalls on this shape; the
+    // linear scan must not.
+    const hostile = `${'<div class="a">'.repeat(20_000)}text`
+    const started = Date.now()
+    const tokens = collect(hostile)
+    expect(tokens.length).toBeGreaterThan(20_000)
+    expect(Date.now() - started).toBeLessThan(2_000)
+  })
+})

package/src/extract/classify.ts

@@ -0,0 +1,81 @@
+import type { PageVerdict } from '../contract/results'
+
+export type PageClassification = {
+  readonly verdict: PageVerdict
+  readonly reason?: string
+}
+
+export type ClassifyInput = {
+  readonly status: number
+  readonly contentType: string | null
+  readonly html: string
+  readonly text: string
+}
+
+const CHALLENGE_MARKERS = [
+  'cf-browser-verification',
+  'cf_chl_opt',
+  'just a moment',
+  'checking your browser',
+  'enable cookies to continue',
+  'ddos protection by',
+  'access denied',
+  'captcha',
+  'are you a robot',
+]
+
+const JS_REQUIRED_MARKERS = [
+  'enable javascript',
+  'javascript is required',
+  'javascript is disabled',
+  "doesn't work properly without javascript",
+  'you need to enable javascript to run this app',
+]
+
+const EMPTY_ROOT = /<div[^<>]{0,200}\bid=["']?(root|__next|app|__nuxt)["']?[^<>]{0,200}>\s*<\/div>/i
+
+/** Below this, a text-bearing HTML page is almost certainly a shell, not content. */
+const MIN_MEANINGFUL_TEXT = 200
+const SHELL_HTML_THRESHOLD = 4_000
+
+/**
+ * Decides whether an HTTP read actually produced readable content. The verdict
+ * drives browser escalation in code rather than leaving the judgement to the
+ * model, which costs a full round-trip and answers differently each time.
+ */
+export function classifyPage(input: ClassifyInput): PageClassification {
+  const haystack = input.html.slice(0, 200_000).toLowerCase()
+
+  if (input.status === 403 || input.status === 429 || input.status === 503) {
+    return { verdict: 'blocked', reason: `HTTP ${input.status}` }
+  }
+
+  const text = input.text.trim()
+  if (text.length === 0 && input.html.trim().length === 0) {
+    return { verdict: 'empty', reason: 'no response body' }
+  }
+
+  // Marker matching runs over raw markup, including script bundles, so plenty of
+  // ordinary pages mention "captcha" or "access denied" without being one. A
+  // challenge page is content-free by definition, so the markers only count when
+  // the page also failed to yield readable text — otherwise a working page gets
+  // flagged and triggers a pointless browser escalation.
+  if (text.length < MIN_MEANINGFUL_TEXT) {
+    const challenge = CHALLENGE_MARKERS.find((marker) => haystack.includes(marker))
+    if (challenge) return { verdict: 'blocked', reason: `anti-bot challenge (${challenge})` }
+  }
+
+  const jsMarker = JS_REQUIRED_MARKERS.find((marker) => haystack.includes(marker))
+  if (jsMarker && text.length < MIN_MEANINGFUL_TEXT) {
+    return { verdict: 'js-shell', reason: `page requires JavaScript (${jsMarker})` }
+  }
+  if (EMPTY_ROOT.test(input.html) && text.length < MIN_MEANINGFUL_TEXT) {
+    return { verdict: 'js-shell', reason: 'empty client-side render root' }
+  }
+  if (text.length < MIN_MEANINGFUL_TEXT && input.html.length > SHELL_HTML_THRESHOLD) {
+    return { verdict: 'js-shell', reason: `only ${text.length} chars of text in ${input.html.length} of markup` }
+  }
+  if (text.length === 0) return { verdict: 'empty', reason: 'no extractable text' }
+
+  return { verdict: 'ok' }
+}