@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82

package/src/fusion/__tests__/fusion.test.ts

@@ -0,0 +1,198 @@
+import { canonicalizeUrl } from '../canonicalize'
+import { fuseResults, type FusionInput } from '../fuse'
+
+const OPTIONS = { limit: 10, minResults: 1, minConfidence: 0, maxPerDomain: 10 }
+
+/**
+ * Matches on the registrable host rather than a URL substring: `includes()`
+ * would also match a host that merely embeds the name (`farm.com.evil.test`),
+ * which is both a weaker assertion and a CodeQL finding.
+ */
+const hostOf = (url: string): string => new URL(url).hostname.replace(/^www\./, '')
+
+describe('canonicalizeUrl', () => {
+  it('strips tracking parameters but keeps meaningful ones', () => {
+    const canonical = canonicalizeUrl('https://example.com/a?utm_source=x&id=7&fbclid=y')
+    expect(canonical?.url).toBe('https://example.com/a?id=7')
+  })
+
+  it('drops the fragment and trailing slash', () => {
+    expect(canonicalizeUrl('https://example.com/a/#section')?.url).toBe('https://example.com/a')
+  })
+
+  it('orders query parameters so equivalent URLs collapse', () => {
+    const left = canonicalizeUrl('https://example.com/a?b=2&a=1')
+    const right = canonicalizeUrl('https://example.com/a?a=1&b=2')
+    expect(left?.key).toBe(right?.key)
+  })
+
+  it('folds www into the dedup key while keeping the URL fetchable', () => {
+    const withWww = canonicalizeUrl('https://www.example.com/a')
+    const without = canonicalizeUrl('https://example.com/a')
+    expect(withWww?.key).toBe(without?.key)
+    expect(withWww?.url).toBe('https://www.example.com/a')
+  })
+
+  it('unwraps a DuckDuckGo redirect', () => {
+    const canonical = canonicalizeUrl(
+      'https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Freal&rut=abc',
+    )
+    expect(canonical?.url).toBe('https://example.com/real')
+  })
+
+  it('rejects a DuckDuckGo sponsored link', () => {
+    // Ads sit in the same markup as organic hits, so the SERP parser picks them
+    // up; left alone they survive a `site:` filter and an agent cites an advert.
+    const advert =
+      'https://duckduckgo.com/y.js?ad_domain=udemy.com&ad_provider=bingv7aa&ad_type=txad&u3=https%3A%2F%2Fwww.bing.com%2Faclick'
+    expect(canonicalizeUrl(advert)).toBeNull()
+  })
+
+  it('keeps an ordinary duckduckgo.com page', () => {
+    expect(canonicalizeUrl('https://duckduckgo.com/about')?.url).toBe('https://duckduckgo.com/about')
+  })
+
+  it('unwraps a Google redirect', () => {
+    expect(canonicalizeUrl('https://www.google.com/url?q=https://example.com/x')?.url).toBe(
+      'https://example.com/x',
+    )
+  })
+
+  it('rejects non-http schemes and malformed input', () => {
+    expect(canonicalizeUrl('javascript:alert(1)')).toBeNull()
+    expect(canonicalizeUrl('not a url')).toBeNull()
+  })
+})
+
+describe('fuseResults', () => {
+  it('ranks a hit found by two adapters above one found by a single adapter first', () => {
+    const inputs: FusionInput[] = [
+      {
+        adapterId: 'alpha',
+        weight: 1,
+        results: [{ url: 'https://a.com/1' }, { url: 'https://shared.com/x' }],
+      },
+      {
+        adapterId: 'beta',
+        weight: 1,
+        results: [{ url: 'https://b.com/1' }, { url: 'https://shared.com/x' }],
+      },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+    expect(results[0].url).toBe('https://shared.com/x')
+    expect(results[0].sources).toHaveLength(2)
+  })
+
+  it('does not let one adapter outvote genuine agreement by repeating a URL', () => {
+    const inputs: FusionInput[] = [
+      {
+        adapterId: 'noisy',
+        weight: 1,
+        results: [
+          { url: 'https://noisy.com/x' },
+          { url: 'https://noisy.com/x?utm_source=feed' },
+          { url: 'https://www.noisy.com/x/' },
+        ],
+      },
+      { adapterId: 'alpha', weight: 1, results: [{ url: 'https://shared.com/y' }] },
+      { adapterId: 'beta', weight: 1, results: [{ url: 'https://shared.com/y' }] },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+
+    expect(results[0].url).toBe('https://shared.com/y')
+    expect(results.find((hit) => hostOf(hit.url) === 'noisy.com')?.sources).toHaveLength(1)
+  })
+
+  it('deduplicates across adapters and merges provenance', () => {
+    const inputs: FusionInput[] = [
+      { adapterId: 'alpha', weight: 1, results: [{ url: 'https://example.com/a?utm_source=x' }] },
+      { adapterId: 'beta', weight: 1, results: [{ url: 'https://www.example.com/a/' }] },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+    expect(results).toHaveLength(1)
+    expect(results[0].sources.map((source) => source.adapterId)).toEqual(['alpha', 'beta'])
+  })
+
+  it('prefers the longest title and snippet when adapters disagree', () => {
+    const inputs: FusionInput[] = [
+      { adapterId: 'alpha', weight: 1, results: [{ url: 'https://e.com/a', title: 'Short', snippet: 'a' }] },
+      {
+        adapterId: 'beta',
+        weight: 1,
+        results: [{ url: 'https://e.com/a', title: 'A much longer title', snippet: 'a longer snippet' }],
+      },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+    expect(results[0].title).toBe('A much longer title')
+    expect(results[0].snippet).toBe('a longer snippet')
+  })
+
+  it('applies adapter weight to the fused score', () => {
+    const inputs: FusionInput[] = [
+      { adapterId: 'trusted', weight: 5, results: [{ url: 'https://trusted.com/x' }] },
+      { adapterId: 'weak', weight: 1, results: [{ url: 'https://weak.com/x' }] },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+    expect(results[0].url).toBe('https://trusted.com/x')
+    expect(results[0].confidence).toBe(1)
+  })
+
+  it('caps results per domain', () => {
+    const inputs: FusionInput[] = [
+      {
+        adapterId: 'alpha',
+        weight: 1,
+        results: [
+          { url: 'https://farm.com/1' },
+          { url: 'https://farm.com/2' },
+          { url: 'https://farm.com/3' },
+          { url: 'https://other.com/1' },
+        ],
+      },
+    ]
+    const { results } = fuseResults(inputs, { ...OPTIONS, maxPerDomain: 2, minResults: 1 })
+    const farmHits = results.filter((result) => hostOf(result.url) === 'farm.com')
+    expect(farmHits).toHaveLength(2)
+    expect(results.map((result) => result.url)).toContain('https://other.com/1')
+  })
+
+  it('skips results whose URL cannot be canonicalized', () => {
+    const inputs: FusionInput[] = [
+      { adapterId: 'alpha', weight: 1, results: [{ url: 'garbage' }, { url: 'https://ok.com/1' }] },
+    ]
+    const { results } = fuseResults(inputs, OPTIONS)
+    expect(results.map((result) => result.url)).toEqual(['https://ok.com/1'])
+  })
+
+  it('relaxes the confidence threshold rather than returning nothing', () => {
+    const inputs: FusionInput[] = [
+      {
+        adapterId: 'alpha',
+        weight: 1,
+        results: [{ url: 'https://a.com/1' }, { url: 'https://b.com/2' }, { url: 'https://c.com/3' }],
+      },
+    ]
+    const { results, degraded } = fuseResults(inputs, {
+      ...OPTIONS,
+      minConfidence: 0.99,
+      minResults: 3,
+    })
+    expect(results.length).toBe(3)
+    expect(degraded).toBe(true)
+  })
+
+  it('is deterministic for equally scored hits', () => {
+    const inputs: FusionInput[] = [
+      { adapterId: 'alpha', weight: 1, results: [{ url: 'https://z.com/1' }] },
+      { adapterId: 'beta', weight: 1, results: [{ url: 'https://a.com/1' }] },
+    ]
+    const first = fuseResults(inputs, OPTIONS).results.map((result) => result.url)
+    const second = fuseResults(inputs, OPTIONS).results.map((result) => result.url)
+    expect(first).toEqual(second)
+    expect(first[0]).toBe('https://a.com/1')
+  })
+
+  it('returns nothing when every adapter returned nothing', () => {
+    expect(fuseResults([], OPTIONS)).toEqual({ results: [], degraded: false })
+  })
+})

package/src/fusion/canonicalize.ts

@@ -0,0 +1,119 @@
+export type CanonicalUrl = {
+  /** Cleaned URL suitable for display and re-fetching. */
+  readonly url: string
+  /** Aggressively normalized form used only for dedup. */
+  readonly key: string
+  readonly domain: string
+}
+
+const TRACKING_PARAMS = new Set([
+  'fbclid',
+  'gclid',
+  'gclsrc',
+  'dclid',
+  'msclkid',
+  'yclid',
+  'twclid',
+  'igshid',
+  'mc_cid',
+  'mc_eid',
+  'ref_src',
+  'ref_url',
+  's_kwcid',
+  'vero_id',
+  'wickedid',
+  '_ga',
+  '_gl',
+  'spm',
+])
+
+/** Query keys carrying the real destination on known redirect hops. */
+const REDIRECT_HOSTS: ReadonlyArray<{ host: RegExp; path?: RegExp; params: readonly string[] }> = [
+  { host: /(^|\.)duckduckgo\.com$/, path: /^\/l\/?$/, params: ['uddg'] },
+  { host: /(^|\.)google\.[a-z.]+$/, path: /^\/url$/, params: ['q', 'url'] },
+  { host: /(^|\.)out\.reddit\.com$/, params: ['url'] },
+  { host: /(^|\.)steamcommunity\.com$/, path: /^\/linkfilter\/?$/, params: ['url'] },
+]
+
+/**
+ * Endpoints that are never a real result. DuckDuckGo renders sponsored slots as
+ * `y.js` click-tracking links carrying `ad_domain`, and they sit in the same
+ * markup as organic hits — so a SERP parser picks them up, they survive a
+ * `site:` filter that should have excluded them, and an agent ends up citing or
+ * fetching an advert. There is nothing to unwrap here that is worth keeping.
+ */
+const AD_ENDPOINTS: ReadonlyArray<{ host: RegExp; path: RegExp }> = [
+  { host: /(^|\.)duckduckgo\.com$/, path: /^\/y\.js$/ },
+]
+
+function isAdEndpoint(url: URL): boolean {
+  return AD_ENDPOINTS.some((rule) => rule.host.test(url.hostname) && rule.path.test(url.pathname))
+}
+
+function isTracking(key: string): boolean {
+  const lower = key.toLowerCase()
+  return lower.startsWith('utm_') || TRACKING_PARAMS.has(lower)
+}
+
+function unwrapRedirect(url: URL): URL | null {
+  for (const rule of REDIRECT_HOSTS) {
+    if (!rule.host.test(url.hostname)) continue
+    if (rule.path && !rule.path.test(url.pathname)) continue
+    for (const param of rule.params) {
+      const target = url.searchParams.get(param)
+      if (!target) continue
+      try {
+        const parsed = new URL(target)
+        if (parsed.protocol === 'http:' || parsed.protocol === 'https:') return parsed
+      } catch {
+        continue
+      }
+    }
+  }
+  return null
+}
+
+/**
+ * Normalizes a result URL for display and produces a separate dedup key. The two
+ * differ on purpose: the key folds `www.` and the scheme so the same page found
+ * by two adapters collapses into one hit, while the displayed URL stays faithful
+ * enough to fetch.
+ */
+export function canonicalizeUrl(raw: string): CanonicalUrl | null {
+  let parsed: URL
+  try {
+    parsed = new URL(raw.trim())
+  } catch {
+    return null
+  }
+
+  for (let hop = 0; hop < 3; hop += 1) {
+    const unwrapped = unwrapRedirect(parsed)
+    if (!unwrapped) break
+    parsed = unwrapped
+  }
+
+  if (isAdEndpoint(parsed)) return null
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null
+  if (parsed.hostname.length === 0) return null
+
+  parsed.hash = ''
+  parsed.username = ''
+  parsed.password = ''
+
+  const retained = [...parsed.searchParams.entries()].filter(([key]) => !isTracking(key))
+  retained.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+  parsed.search = ''
+  for (const [key, value] of retained) parsed.searchParams.append(key, value)
+
+  const host = parsed.hostname.toLowerCase()
+  parsed.hostname = host
+  if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+    parsed.pathname = parsed.pathname.replace(/\/+$/, '')
+  }
+
+  const domain = host.startsWith('www.') ? host.slice(4) : host
+  const key = `${domain}${parsed.pathname}${parsed.search}`
+
+  return { url: parsed.toString(), key, domain }
+}

package/src/fusion/fuse.ts

@@ -0,0 +1,145 @@
+import type { RawResult, ResultProvenance, SearchResult } from '../contract/results'
+import { canonicalizeUrl } from './canonicalize'
+
+/**
+ * Reciprocal-rank-fusion damping constant. 60 is the value from the original
+ * Cormack et al. formulation and is deliberately large relative to typical result
+ * counts, so a hit ranked 1 by one adapter cannot dominate a hit ranked 2 by two.
+ */
+export const RRF_K = 60
+
+export type FusionInput = {
+  readonly adapterId: string
+  readonly weight: number
+  readonly results: readonly RawResult[]
+}
+
+export type FusionOptions = {
+  readonly limit: number
+  readonly minResults: number
+  readonly minConfidence: number
+  readonly maxPerDomain: number
+}
+
+export type FusionOutput = {
+  readonly results: readonly SearchResult[]
+  /** True when the confidence threshold or domain cap had to be relaxed. */
+  readonly degraded: boolean
+}
+
+type Accumulator = {
+  readonly key: string
+  readonly domain: string
+  url: string
+  title: string | null
+  snippet: string | null
+  content: string | null
+  publishedAt: string | null
+  score: number
+  readonly sources: ResultProvenance[]
+}
+
+function longest(current: string | null, candidate: string | null | undefined): string | null {
+  const next = candidate?.trim()
+  if (!next) return current
+  if (current === null || next.length > current.length) return next
+  return current
+}
+
+/**
+ * Merges per-adapter result lists into one ranked set.
+ *
+ * RRF is used rather than blending native scores because the sources are not
+ * comparable: an LLM citation list has no score at all, a SERP has an opaque
+ * one, and a paid API has its own scale. Rank is the only signal every adapter
+ * genuinely agrees on.
+ */
+export function fuseResults(inputs: readonly FusionInput[], options: FusionOptions): FusionOutput {
+  const accumulators = new Map<string, Accumulator>()
+
+  for (const input of inputs) {
+    const weight = input.weight > 0 ? input.weight : 1
+    // Two spellings of one URL inside a single adapter's list must not each earn
+    // that adapter a rank slot: the duplicate would compound its own score and
+    // let one source outvote the agreement between two independent ones.
+    const contributed = new Set<string>()
+    let rank = 0
+    for (const result of input.results) {
+      const canonical = canonicalizeUrl(result.url)
+      if (canonical === null) continue
+      if (contributed.has(canonical.key)) continue
+      contributed.add(canonical.key)
+      rank += 1
+      const existing = accumulators.get(canonical.key)
+      const accumulator: Accumulator = existing ?? {
+        key: canonical.key,
+        domain: canonical.domain,
+        url: canonical.url,
+        title: null,
+        snippet: null,
+        content: null,
+        publishedAt: null,
+        score: 0,
+        sources: [],
+      }
+      accumulator.title = longest(accumulator.title, result.title)
+      accumulator.snippet = longest(accumulator.snippet, result.snippet)
+      accumulator.content = accumulator.content ?? (result.content?.trim() || null)
+      accumulator.publishedAt = accumulator.publishedAt ?? (result.publishedAt?.trim() || null)
+      accumulator.score += weight / (RRF_K + rank)
+      accumulator.sources.push({ adapterId: input.adapterId, rank })
+      if (!existing) accumulators.set(canonical.key, accumulator)
+    }
+  }
+
+  if (accumulators.size === 0) return { results: [], degraded: false }
+
+  const maxScore = Math.max(...[...accumulators.values()].map((entry) => entry.score))
+  const ranked = [...accumulators.values()]
+    .map((entry) => ({
+      entry,
+      result: {
+        url: entry.url,
+        title: entry.title ?? entry.url,
+        snippet: entry.snippet,
+        content: entry.content,
+        publishedAt: entry.publishedAt,
+        confidence: maxScore > 0 ? Number((entry.score / maxScore).toFixed(6)) : 0,
+        sources: entry.sources,
+      } satisfies SearchResult,
+    }))
+    .sort((left, right) => {
+      if (right.result.confidence !== left.result.confidence) {
+        return right.result.confidence - left.result.confidence
+      }
+      return left.entry.key < right.entry.key ? -1 : left.entry.key > right.entry.key ? 1 : 0
+    })
+
+  const perDomain = new Map<string, number>()
+  const primary: SearchResult[] = []
+  const overflow: SearchResult[] = []
+
+  for (const { entry, result } of ranked) {
+    const used = perDomain.get(entry.domain) ?? 0
+    if (used >= options.maxPerDomain || result.confidence < options.minConfidence) {
+      overflow.push(result)
+      continue
+    }
+    perDomain.set(entry.domain, used + 1)
+    primary.push(result)
+  }
+
+  // A confidence threshold or a domain cap must never manufacture an empty
+  // answer: fall back to the best rejected hits and report the relaxation.
+  let degraded = false
+  const selected = primary.slice(0, options.limit)
+  if (selected.length < Math.min(options.minResults, ranked.length)) {
+    const needed = Math.min(options.minResults, options.limit) - selected.length
+    if (needed > 0 && overflow.length > 0) {
+      degraded = true
+      selected.push(...overflow.slice(0, needed))
+    }
+  }
+
+  return { results: selected, degraded }
+}

package/src/fusion/index.ts

@@ -0,0 +1,8 @@
+export { canonicalizeUrl, type CanonicalUrl } from './canonicalize'
+export {
+  RRF_K,
+  fuseResults,
+  type FusionInput,
+  type FusionOptions,
+  type FusionOutput,
+} from './fuse'

package/src/index.ts

@@ -0,0 +1,15 @@
+export * from './contract'
+export * from './engine'
+export * from './extract'
+export * from './fusion'
+export * from './loader'
+export { createHttpClient, createPinnedLookup, type HttpClientOptions } from './net/client'
+export { stripTrailingSlashes } from './net/url'
+export {
+  assertPublicUrl,
+  isLoopbackHostname,
+  isPrivateAddress,
+  type AssertPublicUrlOptions,
+  type LookupFn,
+  type VettedUrl,
+} from './net/ssrf'

package/src/loader/__tests__/describe.test.ts

@@ -0,0 +1,119 @@
+import { z } from 'zod'
+import { defineAdapterModule, type AdapterModule } from '../../contract/adapter'
+import { CONTRACT_VERSION } from '../../contract/version'
+import { createFakeAdapter } from '../../testing/fakeAdapter'
+import { SECRET_PLACEHOLDER, describeOptionsSchema, maskSecrets, unmaskSecrets } from '../describe'
+
+const moduleWith = (schema: z.ZodType<Record<string, unknown>>): AdapterModule<unknown> =>
+  defineAdapterModule({
+    id: 'fixture',
+    kind: 'api',
+    contractVersion: CONTRACT_VERSION,
+    optionsSchema: schema,
+    createAdapter: () => createFakeAdapter({ id: 'fixture' }),
+  }) as unknown as AdapterModule<unknown>
+
+describe('describeOptionsSchema', () => {
+  it('describes each primitive kind', () => {
+    const fields = describeOptionsSchema(
+      moduleWith(
+        z.object({
+          apiKey: z.string().optional(),
+          depth: z.enum(['basic', 'advanced']).optional(),
+          timeoutMs: z.number().int().optional(),
+          enabled: z.boolean().optional(),
+          engines: z.array(z.string()).optional(),
+        }),
+      ),
+    )
+
+    expect(fields.map((field) => [field.name, field.kind])).toEqual([
+      ['apiKey', 'string'],
+      ['depth', 'enum'],
+      ['timeoutMs', 'number'],
+      ['enabled', 'boolean'],
+      ['engines', 'stringList'],
+    ])
+    expect(fields.find((field) => field.name === 'depth')?.choices).toEqual(['basic', 'advanced'])
+  })
+
+  // A host capability (a model resolver, a sidecar handle) is declared with
+  // z.custom. Rendering it would put a function in an admin form.
+  it('skips host-capability fields declared with z.custom', () => {
+    const fields = describeOptionsSchema(
+      moduleWith(
+        z.object({
+          apiKey: z.string().optional(),
+          resolveModel: z.custom<() => void>((value) => typeof value === 'function').optional(),
+          generate: z.custom<() => void>((value) => typeof value === 'function').optional(),
+        }),
+      ),
+    )
+    expect(fields.map((field) => field.name)).toEqual(['apiKey'])
+  })
+
+  it('marks credential-shaped names as secret', () => {
+    const fields = describeOptionsSchema(
+      moduleWith(
+        z.object({
+          apiKey: z.string().optional(),
+          accessToken: z.string().optional(),
+          password: z.string().optional(),
+          baseUrl: z.string().optional(),
+          engines: z.array(z.string()).optional(),
+        }),
+      ),
+    )
+    const secrets = fields.filter((field) => field.secret).map((field) => field.name)
+    expect(secrets).toEqual(['apiKey', 'accessToken', 'password'])
+  })
+
+  it('reports required vs optional and carries the string format', () => {
+    const fields = describeOptionsSchema(
+      moduleWith(z.object({ baseUrl: z.url(), label: z.string().optional() })),
+    )
+    expect(fields.find((field) => field.name === 'baseUrl')).toMatchObject({
+      required: true,
+      format: 'url',
+    })
+    expect(fields.find((field) => field.name === 'label')?.required).toBe(false)
+  })
+
+  it('returns nothing for a schema that is not an object', () => {
+    expect(describeOptionsSchema(moduleWith(z.string() as never))).toEqual([])
+  })
+})
+
+describe('secret round-trip', () => {
+  const fields = describeOptionsSchema(
+    moduleWith(z.object({ apiKey: z.string().optional(), baseUrl: z.string().optional() })),
+  )
+
+  it('never echoes a stored secret back to the client', () => {
+    const masked = maskSecrets({ apiKey: 'sk-live-123', baseUrl: 'https://x' }, fields)
+    expect(masked.apiKey).toBe(SECRET_PLACEHOLDER)
+    expect(masked.baseUrl).toBe('https://x')
+  })
+
+  it('leaves an unset secret unset rather than masking nothing', () => {
+    expect(maskSecrets({ baseUrl: 'https://x' }, fields).apiKey).toBeUndefined()
+  })
+
+  it('keeps the stored secret when the client sends the placeholder back', () => {
+    const merged = unmaskSecrets(
+      { apiKey: SECRET_PLACEHOLDER, baseUrl: 'https://new' },
+      { apiKey: 'sk-live-123', baseUrl: 'https://old' },
+      fields,
+    )
+    expect(merged).toEqual({ apiKey: 'sk-live-123', baseUrl: 'https://new' })
+  })
+
+  it('accepts a genuinely new secret', () => {
+    const merged = unmaskSecrets({ apiKey: 'sk-live-new' }, { apiKey: 'sk-live-old' }, fields)
+    expect(merged.apiKey).toBe('sk-live-new')
+  })
+
+  it('drops the placeholder when nothing was stored, rather than persisting it', () => {
+    expect(unmaskSecrets({ apiKey: SECRET_PLACEHOLDER }, {}, fields).apiKey).toBeUndefined()
+  })
+})