bazilion 0.0.0

package/apps/daemon/src/runtime/tools/web-ssrf.ts

@@ -0,0 +1,245 @@
+import { lookup as dnsLookupCb, type LookupAddress } from 'node:dns'
+import { lookup as dnsLookup } from 'node:dns/promises'
+import { Agent, type Dispatcher } from 'undici'
+
+export class SsrFBlockedError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = 'SsrFBlockedError'
+  }
+}
+
+const BLOCKED_HOSTNAMES = new Set(['localhost', 'metadata.google.internal'])
+const PRIVATE_IPV6_PREFIXES = ['fe80:', 'fec0:', 'fc', 'fd']
+
+function normalizeHostname(hostname: string): string {
+  let h = hostname.trim().toLowerCase().replace(/\.$/, '')
+  if (h.startsWith('[') && h.endsWith(']')) h = h.slice(1, -1)
+  return h
+}
+
+function parseIpv4(address: string): number[] | null {
+  const parts = address.split('.')
+  if (parts.length !== 4) return null
+  const nums = parts.map((p) => Number.parseInt(p, 10))
+  if (nums.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return null
+  return nums
+}
+
+function isPrivateIpv4(parts: number[]): boolean {
+  const [a, b] = parts as [number, number, number, number]
+  if (a === 0 || a === 10 || a === 127) return true
+  if (a === 169 && b === 254) return true
+  if (a === 172 && b >= 16 && b <= 31) return true
+  if (a === 192 && b === 168) return true
+  if (a === 100 && b >= 64 && b <= 127) return true
+  return false
+}
+
+export function isPrivateIpAddress(address: string): boolean {
+  let norm = address.trim().toLowerCase()
+  if (norm.startsWith('[') && norm.endsWith(']')) norm = norm.slice(1, -1)
+  if (!norm) return false
+  if (norm.startsWith('::ffff:')) {
+    const mapped = norm.slice('::ffff:'.length)
+    const ipv4 = parseIpv4(mapped)
+    if (ipv4) return isPrivateIpv4(ipv4)
+  }
+  if (norm.includes(':')) {
+    if (norm === '::' || norm === '::1') return true
+    return PRIVATE_IPV6_PREFIXES.some((p) => norm.startsWith(p))
+  }
+  const ipv4 = parseIpv4(norm)
+  return ipv4 ? isPrivateIpv4(ipv4) : false
+}
+
+export function isBlockedHostname(hostname: string): boolean {
+  const h = normalizeHostname(hostname)
+  if (!h) return false
+  if (BLOCKED_HOSTNAMES.has(h)) return true
+  return h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')
+}
+
+type LookupCallback = (
+  err: NodeJS.ErrnoException | null,
+  address: string | LookupAddress[],
+  family?: number,
+) => void
+
+function createPinnedLookup(hostname: string, addresses: string[]): typeof dnsLookupCb {
+  const normalized = normalizeHostname(hostname)
+  const records = addresses.map((address) => ({
+    address,
+    family: (address.includes(':') ? 6 : 4) as 4 | 6,
+  }))
+  let index = 0
+  return ((host: string, options?: unknown, callback?: unknown) => {
+    const cb: LookupCallback =
+      typeof options === 'function' ? (options as LookupCallback) : (callback as LookupCallback)
+    if (!cb) return
+    if (normalizeHostname(host) !== normalized) {
+      if (typeof options === 'function' || options === undefined) {
+        return (dnsLookupCb as unknown as (h: string, cb: LookupCallback) => void)(host, cb)
+      }
+      return (dnsLookupCb as unknown as (h: string, o: unknown, cb: LookupCallback) => void)(
+        host,
+        options,
+        cb,
+      )
+    }
+    const opts =
+      typeof options === 'object' && options !== null
+        ? (options as { all?: boolean; family?: number })
+        : {}
+    const family = typeof options === 'number' ? options : (opts.family ?? 0)
+    const candidates =
+      family === 4 || family === 6 ? records.filter((r) => r.family === family) : records
+    const usable = candidates.length > 0 ? candidates : records
+    if (opts.all) {
+      cb(null, usable as LookupAddress[])
+      return
+    }
+    const chosen = usable[index % usable.length]
+    if (!chosen) return
+    index += 1
+    cb(null, chosen.address, chosen.family)
+  }) as typeof dnsLookupCb
+}
+
+async function resolveAndCheck(hostname: string): Promise<string[]> {
+  const norm = normalizeHostname(hostname)
+  if (!norm) throw new SsrFBlockedError('Invalid hostname')
+  if (isBlockedHostname(norm)) throw new SsrFBlockedError(`Blocked hostname: ${hostname}`)
+  if (isPrivateIpAddress(norm)) throw new SsrFBlockedError('Blocked: private IP literal')
+  const results = await dnsLookup(norm, { all: true })
+  if (results.length === 0) throw new SsrFBlockedError(`Cannot resolve: ${hostname}`)
+  for (const r of results) {
+    if (isPrivateIpAddress(r.address)) throw new SsrFBlockedError('Blocked: resolves to private IP')
+  }
+  return Array.from(new Set(results.map((r) => r.address)))
+}
+
+type FetchLike = (input: string | URL, init?: RequestInit) => Promise<Response>
+
+export interface GuardedFetchOptions {
+  url: string
+  fetchImpl?: FetchLike
+  init?: RequestInit
+  maxRedirects?: number
+  timeoutMs?: number
+  signal?: AbortSignal
+  /** If true, skip all SSRF checks (for tests hitting 127.0.0.1 mocks). */
+  allowPrivate?: boolean
+}
+
+export interface GuardedFetchResult {
+  response: Response
+  finalUrl: string
+  release: () => Promise<void>
+}
+
+function isRedirectStatus(s: number): boolean {
+  return s === 301 || s === 302 || s === 303 || s === 307 || s === 308
+}
+
+async function closeDispatcher(d: Dispatcher | null): Promise<void> {
+  if (!d) return
+  try {
+    await d.close()
+  } catch {
+    // ignore
+  }
+}
+
+export async function guardedFetch(opts: GuardedFetchOptions): Promise<GuardedFetchResult> {
+  const fetcher: FetchLike = opts.fetchImpl ?? (globalThis.fetch as FetchLike)
+  const maxRedirects = opts.maxRedirects ?? 3
+  const abortController = new AbortController()
+  const timeoutId = opts.timeoutMs
+    ? setTimeout(() => abortController.abort(new Error('timeout')), opts.timeoutMs)
+    : null
+  if (opts.signal) {
+    if (opts.signal.aborted) abortController.abort(opts.signal.reason)
+    else
+      opts.signal.addEventListener('abort', () => abortController.abort(opts.signal?.reason), {
+        once: true,
+      })
+  }
+
+  let current = opts.url
+  const visited = new Set<string>()
+  let redirects = 0
+  let dispatcher: Dispatcher | null = null
+
+  const release = async (): Promise<void> => {
+    if (timeoutId) clearTimeout(timeoutId)
+    await closeDispatcher(dispatcher)
+    dispatcher = null
+  }
+
+  while (true) {
+    let parsed: URL
+    try {
+      parsed = new URL(current)
+    } catch {
+      await release()
+      throw new Error('Invalid URL')
+    }
+    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+      await release()
+      throw new Error('Invalid URL: must be http or https')
+    }
+
+    // Pin DNS only when we control the dispatcher (native fetch path).
+    const canPin = !opts.fetchImpl
+    if (!opts.allowPrivate) {
+      const addrs = await resolveAndCheck(parsed.hostname)
+      await closeDispatcher(dispatcher)
+      dispatcher = canPin
+        ? new Agent({ connect: { lookup: createPinnedLookup(parsed.hostname, addrs) } })
+        : null
+    }
+
+    const init: RequestInit = {
+      ...(opts.init ?? {}),
+      redirect: 'manual',
+      signal: abortController.signal,
+    }
+    // `dispatcher` is a Node fetch extension not reflected in the DOM
+    // RequestInit type — and the undici Dispatcher type brand can diverge
+    // from the one @types/node bundles. Stamp it on via a cast.
+    if (dispatcher) (init as unknown as { dispatcher: Dispatcher }).dispatcher = dispatcher
+
+    let res: Response
+    try {
+      res = await fetcher(parsed.toString(), init)
+    } catch (err) {
+      await release()
+      throw err
+    }
+
+    if (isRedirectStatus(res.status)) {
+      const loc = res.headers.get('location')
+      if (!loc) {
+        await release()
+        throw new Error(`Redirect ${res.status} missing Location header`)
+      }
+      redirects += 1
+      if (redirects > maxRedirects) {
+        await release()
+        throw new Error(`Too many redirects (> ${maxRedirects})`)
+      }
+      const next = new URL(loc, parsed).toString()
+      if (visited.has(next)) {
+        await release()
+        throw new Error('Redirect loop')
+      }
+      visited.add(next)
+      void res.body?.cancel()
+      current = next
+      continue
+    }
+
+    return { response: res, finalUrl: parsed.toString(), release }
+  }
+}

package/apps/daemon/src/runtime/tools/web.ts

@@ -0,0 +1,273 @@
+import type { ToolHandler } from './types.ts'
+import { type ExtractMode, type ExtractResult, extractReadable } from './web-extract.ts'
+import { guardedFetch, SsrFBlockedError } from './web-ssrf.ts'
+
+const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
+const DEFAULT_CACHE_TTL_MS = 15 * 60_000
+const DEFAULT_CACHE_MAX = 100
+const DEFAULT_MAX_LENGTH = 20_000
+const DEFAULT_TIMEOUT_MS = 20_000
+
+interface SearchResult {
+  title: string
+  url: string
+  snippet: string
+}
+
+function stripHtml(s: string): string {
+  return s
+    .replace(/<[^>]*>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .trim()
+}
+
+// --- search backends ---
+
+async function braveSearch(
+  query: string,
+  limit: number,
+  apiKey: string,
+  fetchFn: typeof fetch,
+): Promise<SearchResult[]> {
+  const params = new URLSearchParams({ q: query, count: String(limit) })
+  const res = await fetchFn(`https://api.search.brave.com/res/v1/web/search?${params}`, {
+    headers: {
+      accept: 'application/json',
+      'accept-encoding': 'gzip',
+      'x-subscription-token': apiKey,
+    },
+  })
+  if (!res.ok) throw new Error(`Brave Search: ${res.status} ${await res.text()}`)
+  const data = (await res.json()) as {
+    web?: { results?: { title?: string; url?: string; description?: string }[] }
+  }
+  return (data.web?.results ?? []).slice(0, limit).map((r) => ({
+    title: r.title ?? '',
+    url: r.url ?? '',
+    snippet: r.description ? stripHtml(r.description) : '',
+  }))
+}
+
+async function searxngSearch(
+  query: string,
+  limit: number,
+  baseURL: string,
+  fetchFn: typeof fetch,
+): Promise<SearchResult[]> {
+  const params = new URLSearchParams({ q: query, format: 'json' })
+  const res = await fetchFn(`${baseURL}/search?${params}`, {
+    headers: { accept: 'application/json' },
+  })
+  if (!res.ok) throw new Error(`SearXNG: ${res.status} ${await res.text()}`)
+  const data = (await res.json()) as {
+    results?: { title?: string; url?: string; content?: string }[]
+  }
+  return (data.results ?? []).slice(0, limit).map((r) => ({
+    title: r.title ?? '',
+    url: r.url ?? '',
+    snippet: r.content ?? '',
+  }))
+}
+
+// --- per-URL cache ---
+
+interface CacheEntry {
+  value: ExtractResult
+  expiresAt: number
+}
+
+function cacheGet(cache: Map<string, CacheEntry>, key: string): ExtractResult | null {
+  const entry = cache.get(key)
+  if (!entry) return null
+  if (entry.expiresAt < Date.now()) {
+    cache.delete(key)
+    return null
+  }
+  // Refresh LRU position
+  cache.delete(key)
+  cache.set(key, entry)
+  return entry.value
+}
+
+function cacheSet(
+  cache: Map<string, CacheEntry>,
+  key: string,
+  value: ExtractResult,
+  ttlMs: number,
+  maxEntries: number,
+): void {
+  cache.set(key, { value, expiresAt: Date.now() + ttlMs })
+  while (cache.size > maxEntries) {
+    const oldest = cache.keys().next().value
+    if (!oldest) break
+    cache.delete(oldest)
+  }
+}
+
+// --- tool factory ---
+
+export interface WebToolsOpts {
+  fetchImpl?: typeof fetch
+  env?: NodeJS.ProcessEnv
+  /** Disable SSRF checks (tests against 127.0.0.1 mock servers). Default false. */
+  allowPrivate?: boolean
+  /** Cache TTL in ms. Default 15 min. */
+  cacheTtlMs?: number
+  /** Max cache entries. Default 100. */
+  cacheMax?: number
+  /** Fetch timeout in ms. Default 20s. */
+  timeoutMs?: number
+}
+
+/**
+ * web_search backends (tried in order):
+ *  1. Brave Search (env: BRAVE_API_KEY) — free tier, 2000 req/month
+ *  2. SearXNG (env: SEARXNG_URL) — self-hosted, unlimited
+ *  3. Error with setup instructions if neither is configured
+ *
+ * web_fetch: guarded fetch (SSRF-blocked private IPs) + Readability + markdown,
+ * with per-URL in-memory cache (15 min TTL).
+ */
+export function webTools(opts?: WebToolsOpts): ToolHandler[] {
+  const fetchFn = opts?.fetchImpl ?? fetch
+  const env = opts?.env ?? process.env
+  const allowPrivate = opts?.allowPrivate ?? false
+  const cacheTtlMs = opts?.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS
+  const cacheMax = opts?.cacheMax ?? DEFAULT_CACHE_MAX
+  const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS
+  const cache = new Map<string, CacheEntry>()
+
+  return [
+    {
+      def: {
+        name: 'web_search',
+        description:
+          'Search the internet. Returns a list of titles, URLs, and snippets. Requires BRAVE_API_KEY or SEARXNG_URL to be configured.',
+        parameters: {
+          type: 'object',
+          properties: {
+            query: { type: 'string', description: 'Search query' },
+            limit: { type: 'number', description: 'Max results to return (default 5)' },
+          },
+          required: ['query'],
+        },
+      },
+      async invoke(args) {
+        const query = String(args.query ?? '')
+        if (!query) throw new Error('web_search: query is required')
+        const limit = typeof args.limit === 'number' ? args.limit : 5
+
+        const braveKey = env.BRAVE_API_KEY
+        if (braveKey) {
+          const results = await braveSearch(query, limit, braveKey, fetchFn)
+          return results.length === 0 ? 'no results' : formatResults(results)
+        }
+
+        const searxngUrl = env.SEARXNG_URL
+        if (searxngUrl) {
+          const results = await searxngSearch(query, limit, searxngUrl, fetchFn)
+          return results.length === 0 ? 'no results' : formatResults(results)
+        }
+
+        throw new Error(
+          'web_search: no search backend configured. Set BRAVE_API_KEY (free at https://brave.com/search/api/) or SEARXNG_URL.',
+        )
+      },
+    },
+    {
+      def: {
+        name: 'web_fetch',
+        description:
+          'Fetch a URL and return its readable content. HTML is extracted via Readability and converted to markdown. Results are cached for 15 minutes.',
+        parameters: {
+          type: 'object',
+          properties: {
+            url: { type: 'string', description: 'The URL to fetch (http or https)' },
+            max_length: {
+              type: 'number',
+              description: 'Max characters to return (default 20000)',
+            },
+            extract_mode: {
+              type: 'string',
+              enum: ['markdown', 'text'],
+              description: 'Output format for HTML pages. Default "markdown".',
+            },
+          },
+          required: ['url'],
+        },
+      },
+      async invoke(args) {
+        const url = String(args.url ?? '')
+        if (!url) throw new Error('web_fetch: url is required')
+        const maxLen = typeof args.max_length === 'number' ? args.max_length : DEFAULT_MAX_LENGTH
+        const mode: ExtractMode = args.extract_mode === 'text' ? 'text' : 'markdown'
+        const cacheKey = `${mode}|${url}`
+
+        const cached = cacheGet(cache, cacheKey)
+        if (cached) return formatOutput(cached, maxLen)
+
+        let result: GuardedFetchResultShape
+        try {
+          result = await guardedFetch({
+            url,
+            fetchImpl: opts?.fetchImpl,
+            allowPrivate,
+            timeoutMs,
+            init: {
+              headers: {
+                'user-agent': DEFAULT_USER_AGENT,
+                accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*',
+                'accept-language': 'en-US,en;q=0.9',
+              },
+            },
+          })
+        } catch (err) {
+          if (err instanceof SsrFBlockedError) throw new Error(`web_fetch: ${err.message}`)
+          throw err
+        }
+
+        try {
+          if (!result.response.ok) {
+            throw new Error(`web_fetch: ${result.response.status} ${result.response.statusText}`)
+          }
+          const ct = result.response.headers.get('content-type') ?? ''
+          const body = await result.response.text()
+          let extracted: ExtractResult
+          if (ct.includes('text/html') || ct.includes('xhtml')) {
+            extracted = extractReadable(body, result.finalUrl, mode)
+          } else if (ct.includes('application/json')) {
+            try {
+              extracted = { text: JSON.stringify(JSON.parse(body), null, 2) }
+            } catch {
+              extracted = { text: body }
+            }
+          } else {
+            extracted = { text: body }
+          }
+          cacheSet(cache, cacheKey, extracted, cacheTtlMs, cacheMax)
+          return formatOutput(extracted, maxLen)
+        } finally {
+          await result.release()
+        }
+      },
+    },
+  ]
+}
+
+type GuardedFetchResultShape = Awaited<ReturnType<typeof guardedFetch>>
+
+function formatOutput(r: ExtractResult, maxLen: number): string {
+  const body = r.title ? `# ${r.title}\n\n${r.text}` : r.text
+  if (body.length > maxLen) return `${body.slice(0, maxLen)}\n\n[truncated at ${maxLen} chars]`
+  return body
+}
+
+function formatResults(results: SearchResult[]): string {
+  return results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}\n   ${r.snippet}`).join('\n\n')
+}