bazilion 0.0.0 → 0.1.1

package/apps/daemon/src/runtime/tools/web-ssrf.ts

@@ -1,245 +0,0 @@
-import { lookup as dnsLookupCb, type LookupAddress } from 'node:dns'
-import { lookup as dnsLookup } from 'node:dns/promises'
-import { Agent, type Dispatcher } from 'undici'
-
-export class SsrFBlockedError extends Error {
-  constructor(message: string) {
-    super(message)
-    this.name = 'SsrFBlockedError'
-  }
-}
-
-const BLOCKED_HOSTNAMES = new Set(['localhost', 'metadata.google.internal'])
-const PRIVATE_IPV6_PREFIXES = ['fe80:', 'fec0:', 'fc', 'fd']
-
-function normalizeHostname(hostname: string): string {
-  let h = hostname.trim().toLowerCase().replace(/\.$/, '')
-  if (h.startsWith('[') && h.endsWith(']')) h = h.slice(1, -1)
-  return h
-}
-
-function parseIpv4(address: string): number[] | null {
-  const parts = address.split('.')
-  if (parts.length !== 4) return null
-  const nums = parts.map((p) => Number.parseInt(p, 10))
-  if (nums.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return null
-  return nums
-}
-
-function isPrivateIpv4(parts: number[]): boolean {
-  const [a, b] = parts as [number, number, number, number]
-  if (a === 0 || a === 10 || a === 127) return true
-  if (a === 169 && b === 254) return true
-  if (a === 172 && b >= 16 && b <= 31) return true
-  if (a === 192 && b === 168) return true
-  if (a === 100 && b >= 64 && b <= 127) return true
-  return false
-}
-
-export function isPrivateIpAddress(address: string): boolean {
-  let norm = address.trim().toLowerCase()
-  if (norm.startsWith('[') && norm.endsWith(']')) norm = norm.slice(1, -1)
-  if (!norm) return false
-  if (norm.startsWith('::ffff:')) {
-    const mapped = norm.slice('::ffff:'.length)
-    const ipv4 = parseIpv4(mapped)
-    if (ipv4) return isPrivateIpv4(ipv4)
-  }
-  if (norm.includes(':')) {
-    if (norm === '::' || norm === '::1') return true
-    return PRIVATE_IPV6_PREFIXES.some((p) => norm.startsWith(p))
-  }
-  const ipv4 = parseIpv4(norm)
-  return ipv4 ? isPrivateIpv4(ipv4) : false
-}
-
-export function isBlockedHostname(hostname: string): boolean {
-  const h = normalizeHostname(hostname)
-  if (!h) return false
-  if (BLOCKED_HOSTNAMES.has(h)) return true
-  return h.endsWith('.localhost') || h.endsWith('.local') || h.endsWith('.internal')
-}
-
-type LookupCallback = (
-  err: NodeJS.ErrnoException | null,
-  address: string | LookupAddress[],
-  family?: number,
-) => void
-
-function createPinnedLookup(hostname: string, addresses: string[]): typeof dnsLookupCb {
-  const normalized = normalizeHostname(hostname)
-  const records = addresses.map((address) => ({
-    address,
-    family: (address.includes(':') ? 6 : 4) as 4 | 6,
-  }))
-  let index = 0
-  return ((host: string, options?: unknown, callback?: unknown) => {
-    const cb: LookupCallback =
-      typeof options === 'function' ? (options as LookupCallback) : (callback as LookupCallback)
-    if (!cb) return
-    if (normalizeHostname(host) !== normalized) {
-      if (typeof options === 'function' || options === undefined) {
-        return (dnsLookupCb as unknown as (h: string, cb: LookupCallback) => void)(host, cb)
-      }
-      return (dnsLookupCb as unknown as (h: string, o: unknown, cb: LookupCallback) => void)(
-        host,
-        options,
-        cb,
-      )
-    }
-    const opts =
-      typeof options === 'object' && options !== null
-        ? (options as { all?: boolean; family?: number })
-        : {}
-    const family = typeof options === 'number' ? options : (opts.family ?? 0)
-    const candidates =
-      family === 4 || family === 6 ? records.filter((r) => r.family === family) : records
-    const usable = candidates.length > 0 ? candidates : records
-    if (opts.all) {
-      cb(null, usable as LookupAddress[])
-      return
-    }
-    const chosen = usable[index % usable.length]
-    if (!chosen) return
-    index += 1
-    cb(null, chosen.address, chosen.family)
-  }) as typeof dnsLookupCb
-}
-
-async function resolveAndCheck(hostname: string): Promise<string[]> {
-  const norm = normalizeHostname(hostname)
-  if (!norm) throw new SsrFBlockedError('Invalid hostname')
-  if (isBlockedHostname(norm)) throw new SsrFBlockedError(`Blocked hostname: ${hostname}`)
-  if (isPrivateIpAddress(norm)) throw new SsrFBlockedError('Blocked: private IP literal')
-  const results = await dnsLookup(norm, { all: true })
-  if (results.length === 0) throw new SsrFBlockedError(`Cannot resolve: ${hostname}`)
-  for (const r of results) {
-    if (isPrivateIpAddress(r.address)) throw new SsrFBlockedError('Blocked: resolves to private IP')
-  }
-  return Array.from(new Set(results.map((r) => r.address)))
-}
-
-type FetchLike = (input: string | URL, init?: RequestInit) => Promise<Response>
-
-export interface GuardedFetchOptions {
-  url: string
-  fetchImpl?: FetchLike
-  init?: RequestInit
-  maxRedirects?: number
-  timeoutMs?: number
-  signal?: AbortSignal
-  /** If true, skip all SSRF checks (for tests hitting 127.0.0.1 mocks). */
-  allowPrivate?: boolean
-}
-
-export interface GuardedFetchResult {
-  response: Response
-  finalUrl: string
-  release: () => Promise<void>
-}
-
-function isRedirectStatus(s: number): boolean {
-  return s === 301 || s === 302 || s === 303 || s === 307 || s === 308
-}
-
-async function closeDispatcher(d: Dispatcher | null): Promise<void> {
-  if (!d) return
-  try {
-    await d.close()
-  } catch {
-    // ignore
-  }
-}
-
-export async function guardedFetch(opts: GuardedFetchOptions): Promise<GuardedFetchResult> {
-  const fetcher: FetchLike = opts.fetchImpl ?? (globalThis.fetch as FetchLike)
-  const maxRedirects = opts.maxRedirects ?? 3
-  const abortController = new AbortController()
-  const timeoutId = opts.timeoutMs
-    ? setTimeout(() => abortController.abort(new Error('timeout')), opts.timeoutMs)
-    : null
-  if (opts.signal) {
-    if (opts.signal.aborted) abortController.abort(opts.signal.reason)
-    else
-      opts.signal.addEventListener('abort', () => abortController.abort(opts.signal?.reason), {
-        once: true,
-      })
-  }
-
-  let current = opts.url
-  const visited = new Set<string>()
-  let redirects = 0
-  let dispatcher: Dispatcher | null = null
-
-  const release = async (): Promise<void> => {
-    if (timeoutId) clearTimeout(timeoutId)
-    await closeDispatcher(dispatcher)
-    dispatcher = null
-  }
-
-  while (true) {
-    let parsed: URL
-    try {
-      parsed = new URL(current)
-    } catch {
-      await release()
-      throw new Error('Invalid URL')
-    }
-    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
-      await release()
-      throw new Error('Invalid URL: must be http or https')
-    }
-
-    // Pin DNS only when we control the dispatcher (native fetch path).
-    const canPin = !opts.fetchImpl
-    if (!opts.allowPrivate) {
-      const addrs = await resolveAndCheck(parsed.hostname)
-      await closeDispatcher(dispatcher)
-      dispatcher = canPin
-        ? new Agent({ connect: { lookup: createPinnedLookup(parsed.hostname, addrs) } })
-        : null
-    }
-
-    const init: RequestInit = {
-      ...(opts.init ?? {}),
-      redirect: 'manual',
-      signal: abortController.signal,
-    }
-    // `dispatcher` is a Node fetch extension not reflected in the DOM
-    // RequestInit type — and the undici Dispatcher type brand can diverge
-    // from the one @types/node bundles. Stamp it on via a cast.
-    if (dispatcher) (init as unknown as { dispatcher: Dispatcher }).dispatcher = dispatcher
-
-    let res: Response
-    try {
-      res = await fetcher(parsed.toString(), init)
-    } catch (err) {
-      await release()
-      throw err
-    }
-
-    if (isRedirectStatus(res.status)) {
-      const loc = res.headers.get('location')
-      if (!loc) {
-        await release()
-        throw new Error(`Redirect ${res.status} missing Location header`)
-      }
-      redirects += 1
-      if (redirects > maxRedirects) {
-        await release()
-        throw new Error(`Too many redirects (> ${maxRedirects})`)
-      }
-      const next = new URL(loc, parsed).toString()
-      if (visited.has(next)) {
-        await release()
-        throw new Error('Redirect loop')
-      }
-      visited.add(next)
-      void res.body?.cancel()
-      current = next
-      continue
-    }
-
-    return { response: res, finalUrl: parsed.toString(), release }
-  }
-}

package/apps/daemon/src/runtime/tools/web.ts

@@ -1,273 +0,0 @@
-import type { ToolHandler } from './types.ts'
-import { type ExtractMode, type ExtractResult, extractReadable } from './web-extract.ts'
-import { guardedFetch, SsrFBlockedError } from './web-ssrf.ts'
-
-const DEFAULT_USER_AGENT =
-  'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
-const DEFAULT_CACHE_TTL_MS = 15 * 60_000
-const DEFAULT_CACHE_MAX = 100
-const DEFAULT_MAX_LENGTH = 20_000
-const DEFAULT_TIMEOUT_MS = 20_000
-
-interface SearchResult {
-  title: string
-  url: string
-  snippet: string
-}
-
-function stripHtml(s: string): string {
-  return s
-    .replace(/<[^>]*>/g, '')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&nbsp;/g, ' ')
-    .trim()
-}
-
-// --- search backends ---
-
-async function braveSearch(
-  query: string,
-  limit: number,
-  apiKey: string,
-  fetchFn: typeof fetch,
-): Promise<SearchResult[]> {
-  const params = new URLSearchParams({ q: query, count: String(limit) })
-  const res = await fetchFn(`https://api.search.brave.com/res/v1/web/search?${params}`, {
-    headers: {
-      accept: 'application/json',
-      'accept-encoding': 'gzip',
-      'x-subscription-token': apiKey,
-    },
-  })
-  if (!res.ok) throw new Error(`Brave Search: ${res.status} ${await res.text()}`)
-  const data = (await res.json()) as {
-    web?: { results?: { title?: string; url?: string; description?: string }[] }
-  }
-  return (data.web?.results ?? []).slice(0, limit).map((r) => ({
-    title: r.title ?? '',
-    url: r.url ?? '',
-    snippet: r.description ? stripHtml(r.description) : '',
-  }))
-}
-
-async function searxngSearch(
-  query: string,
-  limit: number,
-  baseURL: string,
-  fetchFn: typeof fetch,
-): Promise<SearchResult[]> {
-  const params = new URLSearchParams({ q: query, format: 'json' })
-  const res = await fetchFn(`${baseURL}/search?${params}`, {
-    headers: { accept: 'application/json' },
-  })
-  if (!res.ok) throw new Error(`SearXNG: ${res.status} ${await res.text()}`)
-  const data = (await res.json()) as {
-    results?: { title?: string; url?: string; content?: string }[]
-  }
-  return (data.results ?? []).slice(0, limit).map((r) => ({
-    title: r.title ?? '',
-    url: r.url ?? '',
-    snippet: r.content ?? '',
-  }))
-}
-
-// --- per-URL cache ---
-
-interface CacheEntry {
-  value: ExtractResult
-  expiresAt: number
-}
-
-function cacheGet(cache: Map<string, CacheEntry>, key: string): ExtractResult | null {
-  const entry = cache.get(key)
-  if (!entry) return null
-  if (entry.expiresAt < Date.now()) {
-    cache.delete(key)
-    return null
-  }
-  // Refresh LRU position
-  cache.delete(key)
-  cache.set(key, entry)
-  return entry.value
-}
-
-function cacheSet(
-  cache: Map<string, CacheEntry>,
-  key: string,
-  value: ExtractResult,
-  ttlMs: number,
-  maxEntries: number,
-): void {
-  cache.set(key, { value, expiresAt: Date.now() + ttlMs })
-  while (cache.size > maxEntries) {
-    const oldest = cache.keys().next().value
-    if (!oldest) break
-    cache.delete(oldest)
-  }
-}
-
-// --- tool factory ---
-
-export interface WebToolsOpts {
-  fetchImpl?: typeof fetch
-  env?: NodeJS.ProcessEnv
-  /** Disable SSRF checks (tests against 127.0.0.1 mock servers). Default false. */
-  allowPrivate?: boolean
-  /** Cache TTL in ms. Default 15 min. */
-  cacheTtlMs?: number
-  /** Max cache entries. Default 100. */
-  cacheMax?: number
-  /** Fetch timeout in ms. Default 20s. */
-  timeoutMs?: number
-}
-
-/**
- * web_search backends (tried in order):
- *  1. Brave Search (env: BRAVE_API_KEY) — free tier, 2000 req/month
- *  2. SearXNG (env: SEARXNG_URL) — self-hosted, unlimited
- *  3. Error with setup instructions if neither is configured
- *
- * web_fetch: guarded fetch (SSRF-blocked private IPs) + Readability + markdown,
- * with per-URL in-memory cache (15 min TTL).
- */
-export function webTools(opts?: WebToolsOpts): ToolHandler[] {
-  const fetchFn = opts?.fetchImpl ?? fetch
-  const env = opts?.env ?? process.env
-  const allowPrivate = opts?.allowPrivate ?? false
-  const cacheTtlMs = opts?.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS
-  const cacheMax = opts?.cacheMax ?? DEFAULT_CACHE_MAX
-  const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS
-  const cache = new Map<string, CacheEntry>()
-
-  return [
-    {
-      def: {
-        name: 'web_search',
-        description:
-          'Search the internet. Returns a list of titles, URLs, and snippets. Requires BRAVE_API_KEY or SEARXNG_URL to be configured.',
-        parameters: {
-          type: 'object',
-          properties: {
-            query: { type: 'string', description: 'Search query' },
-            limit: { type: 'number', description: 'Max results to return (default 5)' },
-          },
-          required: ['query'],
-        },
-      },
-      async invoke(args) {
-        const query = String(args.query ?? '')
-        if (!query) throw new Error('web_search: query is required')
-        const limit = typeof args.limit === 'number' ? args.limit : 5
-
-        const braveKey = env.BRAVE_API_KEY
-        if (braveKey) {
-          const results = await braveSearch(query, limit, braveKey, fetchFn)
-          return results.length === 0 ? 'no results' : formatResults(results)
-        }
-
-        const searxngUrl = env.SEARXNG_URL
-        if (searxngUrl) {
-          const results = await searxngSearch(query, limit, searxngUrl, fetchFn)
-          return results.length === 0 ? 'no results' : formatResults(results)
-        }
-
-        throw new Error(
-          'web_search: no search backend configured. Set BRAVE_API_KEY (free at https://brave.com/search/api/) or SEARXNG_URL.',
-        )
-      },
-    },
-    {
-      def: {
-        name: 'web_fetch',
-        description:
-          'Fetch a URL and return its readable content. HTML is extracted via Readability and converted to markdown. Results are cached for 15 minutes.',
-        parameters: {
-          type: 'object',
-          properties: {
-            url: { type: 'string', description: 'The URL to fetch (http or https)' },
-            max_length: {
-              type: 'number',
-              description: 'Max characters to return (default 20000)',
-            },
-            extract_mode: {
-              type: 'string',
-              enum: ['markdown', 'text'],
-              description: 'Output format for HTML pages. Default "markdown".',
-            },
-          },
-          required: ['url'],
-        },
-      },
-      async invoke(args) {
-        const url = String(args.url ?? '')
-        if (!url) throw new Error('web_fetch: url is required')
-        const maxLen = typeof args.max_length === 'number' ? args.max_length : DEFAULT_MAX_LENGTH
-        const mode: ExtractMode = args.extract_mode === 'text' ? 'text' : 'markdown'
-        const cacheKey = `${mode}|${url}`
-
-        const cached = cacheGet(cache, cacheKey)
-        if (cached) return formatOutput(cached, maxLen)
-
-        let result: GuardedFetchResultShape
-        try {
-          result = await guardedFetch({
-            url,
-            fetchImpl: opts?.fetchImpl,
-            allowPrivate,
-            timeoutMs,
-            init: {
-              headers: {
-                'user-agent': DEFAULT_USER_AGENT,
-                accept: 'text/html,application/xhtml+xml,application/json,text/plain,*/*',
-                'accept-language': 'en-US,en;q=0.9',
-              },
-            },
-          })
-        } catch (err) {
-          if (err instanceof SsrFBlockedError) throw new Error(`web_fetch: ${err.message}`)
-          throw err
-        }
-
-        try {
-          if (!result.response.ok) {
-            throw new Error(`web_fetch: ${result.response.status} ${result.response.statusText}`)
-          }
-          const ct = result.response.headers.get('content-type') ?? ''
-          const body = await result.response.text()
-          let extracted: ExtractResult
-          if (ct.includes('text/html') || ct.includes('xhtml')) {
-            extracted = extractReadable(body, result.finalUrl, mode)
-          } else if (ct.includes('application/json')) {
-            try {
-              extracted = { text: JSON.stringify(JSON.parse(body), null, 2) }
-            } catch {
-              extracted = { text: body }
-            }
-          } else {
-            extracted = { text: body }
-          }
-          cacheSet(cache, cacheKey, extracted, cacheTtlMs, cacheMax)
-          return formatOutput(extracted, maxLen)
-        } finally {
-          await result.release()
-        }
-      },
-    },
-  ]
-}
-
-type GuardedFetchResultShape = Awaited<ReturnType<typeof guardedFetch>>
-
-function formatOutput(r: ExtractResult, maxLen: number): string {
-  const body = r.title ? `# ${r.title}\n\n${r.text}` : r.text
-  if (body.length > maxLen) return `${body.slice(0, maxLen)}\n\n[truncated at ${maxLen} chars]`
-  return body
-}
-
-function formatResults(results: SearchResult[]): string {
-  return results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}\n   ${r.snippet}`).join('\n\n')
-}