@tangle-network/agent-knowledge 4.0.1 → 5.0.0

package/dist/{chunk-WCYW2GDA.js → chunk-MYFM6LKH.js}

@@ -38,7 +38,7 @@ function extractLinks(html, hrefPattern, baseUrl) {
 // src/sources/http.ts
 import { mkdir, readFile, stat, writeFile } from "fs/promises";
 import { dirname, join } from "path";
-var POLITE_USER_AGENT = "agent-knowledge/0.2.0 (+https://github.com/tangle-network/agent-knowledge)";
+var POLITE_USER_AGENT = "agent-knowledge (+https://github.com/tangle-network/agent-knowledge)";
 var MIN_REQUEST_GAP_MS = 1e3;
 var MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
 var hostThrottle = /* @__PURE__ */ new Map();
@@ -548,4 +548,4 @@ export {
   createIrsPublicationsSource,
   createStateSosSource
 };
-//# sourceMappingURL=chunk-WCYW2GDA.js.map
+//# sourceMappingURL=chunk-MYFM6LKH.js.map

package/dist/chunk-MYFM6LKH.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/sources/html.ts","../src/sources/http.ts","../src/sources/cornell-lii.ts","../src/sources/irs-publications.ts","../src/sources/state-sos.ts"],"sourcesContent":["/**\n * Minimal HTML helpers used by the shipped sources.\n *\n * Deliberately not a full DOM parser: every authority we ship against\n * (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered\n * HTML where regex-based extraction is correct and cheap. Bringing in cheerio\n * would add a 1.5MB dependency to a package whose purpose is shipping\n * primitives, not parsing arbitrary web pages.\n *\n * If a future source needs real DOM traversal, it should depend on its own\n * parser locally rather than promoting one into the package-wide deps.\n *\n * @stable\n */\n\n/**\n * Strip HTML tags, collapse whitespace, decode common entities.\n *\n * Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`,\n * `</h*>`) as `\\n` so statute text retains its subsection structure.\n */\nexport function htmlToText(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, '')\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, '')\n    .replace(/<noscript[\\s\\S]*?<\\/noscript>/gi, '')\n    .replace(/<!--([\\s\\S]*?)-->/g, '')\n    .replace(/<\\s*br\\s*\\/?>/gi, '\\n')\n    .replace(/<\\/(p|li|div|tr|h[1-6]|blockquote|section|article)>/gi, '\\n')\n    .replace(/<[^>]+>/g, '')\n    .replace(/&nbsp;/gi, ' ')\n    .replace(/&amp;/gi, '&')\n    .replace(/&lt;/gi, '<')\n    .replace(/&gt;/gi, '>')\n    .replace(/&quot;/gi, '\"')\n    .replace(/&#39;/gi, \"'\")\n    .replace(/&sect;/gi, '§')\n    .replace(/&mdash;/gi, '—')\n    .replace(/&ndash;/gi, '–')\n    .replace(/&#(\\d+);/g, (_, code) => String.fromCodePoint(Number(code)))\n    .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(Number.parseInt(code, 16)))\n    .split('\\n')\n    .map((line) => line.replace(/[\\t  ]+/g, ' ').trim())\n    .filter((line, idx, all) => !(line === '' && all[idx - 1] === ''))\n    .join('\\n')\n    .trim()\n}\n\n/** Extract the first match of a regex's first capture group, or undefined. */\nexport function firstMatch(html: string, pattern: RegExp): string | undefined {\n  return pattern.exec(html)?.[1]?.trim()\n}\n\n/** Extract the inner HTML of the first matching tag with id `id`. */\nexport function innerHtmlById(html: string, id: string): string | undefined {\n  const escaped = id.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  const tagPattern = new RegExp(\n    `<([a-z][a-z0-9]*)\\\\b[^>]*\\\\sid=[\"']${escaped}[\"'][^>]*>([\\\\s\\\\S]*?)<\\\\/\\\\1>`,\n    'i',\n  )\n  return tagPattern.exec(html)?.[2]\n}\n\n/**\n * Extract every (href, text) pair matching the URL regex.\n * Returns absolute URLs by resolving against `baseUrl`.\n */\nexport function extractLinks(\n  html: string,\n  hrefPattern: RegExp,\n  baseUrl: string,\n): { href: string; text: string }[] {\n  const out: { href: string; text: string }[] = []\n  const anchor = /<a\\b[^>]*\\shref=[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi\n  for (const match of html.matchAll(anchor)) {\n    const href = match[1]\n    const inner = match[2]\n    if (!href || !inner) continue\n    if (!hrefPattern.test(href)) continue\n    const text = htmlToText(inner)\n    if (!text) continue\n    try {\n      out.push({ href: new URL(href, baseUrl).toString(), text })\n    } catch {\n      /* skip malformed URL */\n    }\n  }\n  return out\n}\n","import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'\nimport { dirname, join } from 'node:path'\nimport { sha256 } from '../ids'\n\n/**\n * Polite HTTP fetcher shared by remote sources.\n *\n * Independent sources share a per-origin throttle because rate-limited sites\n * may return block pages instead of 429 responses. Responses are cached by URL\n * because many publishers omit reliable ETag and Last-Modified headers. Bodies\n * are checked even after a 2xx response because captcha and block pages often\n * use successful status codes.\n */\n\n/** User-Agent string sent on every outbound request. */\nexport const POLITE_USER_AGENT =\n  'agent-knowledge (+https://github.com/tangle-network/agent-knowledge)'\n\n/** Minimum gap between successive requests to the same origin (ms). */\nexport const MIN_REQUEST_GAP_MS = 1_000\n\n/** Maximum response body we will buffer in memory (bytes). */\nexport const MAX_RESPONSE_BYTES = 8 * 1024 * 1024\n\nconst hostThrottle = new Map<string, Promise<void>>()\n\nexport interface PoliteFetchOptions {\n  signal?: AbortSignal\n  cacheDir?: string\n  /**\n   * Cache age beyond which we re-fetch. Default 1 hour, long enough to\n   * batch a cron sweep across many selectors, short enough that hourly\n   * authoritative-page changes get picked up next tick.\n   */\n  cacheTtlMs?: number\n  /**\n   * Extra request headers. The fetcher always sets `User-Agent` and\n   * `Accept`; callers can add `Accept-Language` etc.\n   */\n  headers?: Record<string, string>\n}\n\nexport interface PoliteFetchResult {\n  url: string\n  status: number\n  /** Decoded UTF-8 body. Truncated to `MAX_RESPONSE_BYTES`. */\n  body: string\n  /**\n   * Best-effort source-attested timestamp. Reads `Last-Modified`,\n   * falling back to `Date`, falling back to fetch time. Always ISO 8601.\n   */\n  sourceUpdatedAt: string\n  fetchedAt: string\n  /** True iff the response was satisfied from disk cache. */\n  fromCache: boolean\n  /**\n   * False on: non-2xx status, captcha/block page heuristic match, or\n   * decoded body below 200 chars from a host known to serve real content\n   * (Cornell, IRS, state SOS). `unverifiableReason` carries the why.\n   */\n  verifiable: boolean\n  unverifiableReason?: string\n}\n\n/**\n * Fetch one URL with per-host throttling, on-disk cache, and block-page\n * detection. Never throws on network/HTTP failure. It returns a result with\n * `verifiable: false` and `unverifiableReason` set so the caller can decide\n * whether to skip, retry, or surface.\n *\n * Throws ONLY on `AbortError` (caller asked to stop) and on cache-write\n * failures that indicate a misconfigured filesystem.\n */\nexport async function politeFetch(\n  url: string,\n  options: PoliteFetchOptions = {},\n): Promise<PoliteFetchResult> {\n  const cacheTtl = options.cacheTtlMs ?? 60 * 60 * 1000\n  const cached = options.cacheDir ? await readCache(options.cacheDir, url, cacheTtl) : undefined\n  if (cached) return cached\n\n  const host = safeHost(url)\n  await throttleHost(host)\n\n  const fetchedAt = new Date().toISOString()\n  let response: Response\n  try {\n    response = await fetch(url, {\n      signal: options.signal,\n      redirect: 'follow',\n      headers: {\n        'User-Agent': POLITE_USER_AGENT,\n        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n        'Accept-Language': 'en-US,en;q=0.9',\n        ...(options.headers ?? {}),\n      },\n    })\n  } catch (error) {\n    if ((error as { name?: string }).name === 'AbortError') throw error\n    const result: PoliteFetchResult = {\n      url,\n      status: 0,\n      body: '',\n      sourceUpdatedAt: fetchedAt,\n      fetchedAt,\n      fromCache: false,\n      verifiable: false,\n      unverifiableReason: `network error: ${(error as Error).message}`,\n    }\n    if (options.cacheDir) await writeCache(options.cacheDir, url, result)\n    return result\n  }\n\n  const text = await readBoundedText(response)\n  const lastModified = response.headers.get('last-modified')\n  const dateHeader = response.headers.get('date')\n  const sourceUpdatedAt = parseHttpDate(lastModified) ?? parseHttpDate(dateHeader) ?? fetchedAt\n\n  const result: PoliteFetchResult = {\n    url,\n    status: response.status,\n    body: text,\n    sourceUpdatedAt,\n    fetchedAt,\n    fromCache: false,\n    verifiable: true,\n  }\n\n  if (response.status < 200 || response.status >= 300) {\n    result.verifiable = false\n    result.unverifiableReason = `non-2xx status: ${response.status}`\n  } else if (looksLikeBlockPage(text)) {\n    result.verifiable = false\n    result.unverifiableReason = 'block-page heuristic matched'\n  } else if (text.length < 200 && knownLargeAuthority(host)) {\n    result.verifiable = false\n    result.unverifiableReason = `body shorter than expected (${text.length} chars)`\n  }\n\n  if (options.cacheDir) await writeCache(options.cacheDir, url, result)\n  return result\n}\n\n/** Reset the in-process throttle map. Test-only. */\nexport function __resetHttpThrottle(): void {\n  hostThrottle.clear()\n}\n\nfunction safeHost(url: string): string {\n  try {\n    return new URL(url).host\n  } catch {\n    return 'unknown'\n  }\n}\n\nasync function throttleHost(host: string): Promise<void> {\n  const prev = hostThrottle.get(host) ?? Promise.resolve()\n  let release: () => void = () => {}\n  const next = new Promise<void>((resolve) => {\n    release = resolve\n  })\n  hostThrottle.set(\n    host,\n    prev.then(() => next),\n  )\n  await prev\n  setTimeout(release, MIN_REQUEST_GAP_MS)\n}\n\nasync function readBoundedText(response: Response): Promise<string> {\n  if (!response.body) return ''\n  const reader = response.body.getReader()\n  const chunks: Uint8Array[] = []\n  let total = 0\n  while (true) {\n    const { done, value } = await reader.read()\n    if (done) break\n    if (!value) continue\n    total += value.length\n    if (total > MAX_RESPONSE_BYTES) {\n      // Stop reading; release the underlying connection.\n      await reader.cancel()\n      break\n    }\n    chunks.push(value)\n  }\n  const merged = new Uint8Array(Math.min(total, MAX_RESPONSE_BYTES))\n  let offset = 0\n  for (const chunk of chunks) {\n    const take = Math.min(chunk.length, merged.length - offset)\n    if (take <= 0) break\n    merged.set(chunk.subarray(0, take), offset)\n    offset += take\n  }\n  return new TextDecoder('utf-8', { fatal: false }).decode(merged)\n}\n\nfunction parseHttpDate(value: string | null): string | undefined {\n  if (!value) return undefined\n  const ms = Date.parse(value)\n  return Number.isFinite(ms) ? new Date(ms).toISOString() : undefined\n}\n\n/** Cheap heuristic that catches CAPTCHA, WAF block pages, and \"Just a moment\" interstitials. */\nexport function looksLikeBlockPage(body: string): boolean {\n  if (!body) return false\n  const lower = body.toLowerCase()\n  const markers = [\n    'verify you are human',\n    'please enable javascript and cookies',\n    'just a moment',\n    'access denied',\n    'request unsuccessful',\n    'cf-error-details',\n    'captcha',\n    'incapsula',\n    'pardon our interruption',\n  ]\n  for (const marker of markers) {\n    if (lower.includes(marker)) return true\n  }\n  return false\n}\n\nfunction knownLargeAuthority(host: string): boolean {\n  return (\n    host.endsWith('law.cornell.edu') ||\n    host.endsWith('irs.gov') ||\n    host.endsWith('sos.ca.gov') ||\n    host.endsWith('sos.state.tx.us') ||\n    host.endsWith('sos.state.us')\n  )\n}\n\nfunction cachePath(cacheDir: string, url: string): string {\n  const key = sha256(url)\n  return join(cacheDir, 'http', `${key.slice(0, 2)}`, `${key}.json`)\n}\n\nasync function readCache(\n  cacheDir: string,\n  url: string,\n  ttlMs: number,\n): Promise<PoliteFetchResult | undefined> {\n  const path = cachePath(cacheDir, url)\n  try {\n    const info = await stat(path)\n    if (Date.now() - info.mtimeMs > ttlMs) return undefined\n    const raw = await readFile(path, 'utf8')\n    const parsed = JSON.parse(raw) as PoliteFetchResult\n    return { ...parsed, fromCache: true }\n  } catch {\n    return undefined\n  }\n}\n\nasync function writeCache(cacheDir: string, url: string, value: PoliteFetchResult): Promise<void> {\n  const path = cachePath(cacheDir, url)\n  await mkdir(dirname(path), { recursive: true })\n  await writeFile(path, JSON.stringify(value), 'utf8')\n}\n","import { sha256 } from '../ids'\nimport { htmlToText, innerHtmlById } from './html'\nimport { politeFetch } from './http'\nimport type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types'\n\n/**\n * Cornell Legal Information Institute (LII) source.\n *\n * Pulls federal US Code sections and Wex encyclopedia entries — the two\n * Cornell LII surfaces an agent typically grounds against. The Wex\n * \"non-compete\" page is the canonical test case for the Ryan-LLC v. FTC\n * vacatur drift the continuous-ingestion story is designed to catch.\n *\n * @stable\n */\n\nconst BASE_URL = 'https://www.law.cornell.edu'\n\nexport interface CornellLiiSelector {\n  /** Either 'uscode' or 'wex'. */\n  kind: 'uscode' | 'wex'\n  /**\n   * For `uscode`: `<title>/<section>` (e.g. `'18/1836'` for DTSA).\n   * For `wex`: the slug (e.g. `'non-compete'`).\n   */\n  path: string\n  /**\n   * Optional pre-declared eval dimensions affected by this section. If\n   * omitted, defaults are chosen from `kind` + path heuristics.\n   */\n  dimensionHints?: string[]\n}\n\nexport interface CornellLiiSourceOptions {\n  /**\n   * Selectors to fetch on each `fetch()` call. The caller (a per-tenant\n   * workspace config, typically) lists exactly the authorities they need\n   * tracked. There is no auto-discovery; that would crawl Cornell at\n   * cron speed, which is what the polite-fetch contract exists to avoid.\n   */\n  selectors: CornellLiiSelector[]\n  /** Source id override; default is `'cornell-lii'`. */\n  id?: string\n}\n\n/**\n * Build a Cornell LII source for the listed selectors.\n *\n * Example: track DTSA + non-compete:\n * ```\n * createCornellLiiSource({\n *   selectors: [\n *     { kind: 'uscode', path: '18/1836' },\n *     { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] },\n *   ],\n * })\n * ```\n */\nexport function createCornellLiiSource(options: CornellLiiSourceOptions): KnowledgeSource {\n  const id = options.id ?? 'cornell-lii'\n  return {\n    id,\n    name: 'Cornell Legal Information Institute',\n    description:\n      'Federal US Code sections (uscode/text/...) and Wex legal encyclopedia entries from law.cornell.edu.',\n    async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> {\n      const limit = opts.limit ?? options.selectors.length\n      const selectors = options.selectors.slice(0, limit)\n      const out: KnowledgeFragment[] = []\n      for (const selector of selectors) {\n        out.push(await fetchOne(id, selector, opts))\n      }\n      return out\n    },\n  }\n}\n\nasync function fetchOne(\n  sourceId: string,\n  selector: CornellLiiSelector,\n  opts: FetchOpts,\n): Promise<KnowledgeFragment> {\n  const path = selector.path.replace(/^\\/+/, '')\n  const url =\n    selector.kind === 'uscode' ? `${BASE_URL}/uscode/text/${path}` : `${BASE_URL}/wex/${path}`\n\n  const response = await politeFetch(url, {\n    signal: opts.signal,\n    cacheDir: opts.cacheDir,\n  })\n\n  const fragmentId = `${selector.kind}:${selector.path}`\n  const dimensionHints = selector.dimensionHints ?? defaultDimensionHints(selector)\n\n  if (!response.verifiable) {\n    return {\n      id: fragmentId,\n      title: `Cornell LII ${selector.kind} ${selector.path}`,\n      body: '',\n      bodyHash: sha256(''),\n      provenance: {\n        url,\n        sourceUpdatedAt: response.sourceUpdatedAt,\n        fetchedAt: response.fetchedAt,\n        jurisdiction: 'US-FED',\n        verifiable: false,\n        unverifiableReason: response.unverifiableReason,\n      },\n      dimensionHints,\n      metadata: { sourceId, status: response.status, fromCache: response.fromCache },\n    }\n  }\n\n  const html = response.body\n  const title = extractTitle(html, selector)\n  const body = extractBody(html, selector)\n  const effective = extractEffectiveDate(html) ?? response.sourceUpdatedAt\n\n  const verifiable = body.length > 50\n  return {\n    id: fragmentId,\n    title,\n    body,\n    bodyHash: sha256(body),\n    provenance: {\n      url,\n      sourceUpdatedAt: effective,\n      fetchedAt: response.fetchedAt,\n      jurisdiction: 'US-FED',\n      verifiable,\n      unverifiableReason: verifiable ? undefined : 'extracted body too short',\n    },\n    dimensionHints,\n    metadata: { sourceId, status: response.status, fromCache: response.fromCache },\n  }\n}\n\nfunction extractTitle(html: string, selector: CornellLiiSelector): string {\n  const h1 = /<h1[^>]*\\bid=[\"']page_title[\"'][^>]*>([\\s\\S]*?)<\\/h1>/i.exec(html)?.[1]\n  if (h1) return htmlToText(h1)\n  const t = /<title>([\\s\\S]*?)<\\/title>/i.exec(html)?.[1]\n  if (t) return htmlToText(t).split(' | ')[0] ?? `Cornell LII ${selector.path}`\n  return `Cornell LII ${selector.kind} ${selector.path}`\n}\n\nfunction extractBody(html: string, selector: CornellLiiSelector): string {\n  if (selector.kind === 'uscode') {\n    // The statute text lives inside a <text><div class=\"text\">…</div></text>\n    // block on US Code section pages. Prefer it; fall back to #tab_default_1\n    // which always contains the section body.\n    const text = /<text>([\\s\\S]*?)<\\/text>/i.exec(html)?.[1]\n    if (text) return htmlToText(text)\n    const tab = innerHtmlById(html, 'tab_default_1')\n    if (tab) return htmlToText(tab)\n  }\n  // Wex pages wrap the encyclopedia entry under <div id=\"main-content\"> (newer\n  // Drupal template) or directly inside <div id=\"extracted-content\"> (older\n  // template). Try both — lazy regex matching against a nested-div container\n  // returns the wrong (shorter) slice, so we anchor on the leaf containers.\n  const mainContent = innerHtmlById(html, 'main-content')\n  if (mainContent) {\n    return htmlToText(mainContent.replace(/<h1[\\s\\S]*?<\\/h1>/i, ''))\n  }\n  const extracted = innerHtmlById(html, 'extracted-content')\n  if (extracted) {\n    return htmlToText(extracted.replace(/<h1[\\s\\S]*?<\\/h1>/i, ''))\n  }\n  return htmlToText(html)\n}\n\nfunction extractEffectiveDate(html: string): string | undefined {\n  // Cornell LII includes \"Editorial Notes\" / \"Amendments\" blocks with\n  // dates; the most reliable machine-readable signal is the last\n  // amendment year embedded near the section text.\n  const amend = /Amendments[\\s\\S]{0,200}?(\\d{4})/i.exec(html)?.[1]\n  if (amend) {\n    const y = Number.parseInt(amend, 10)\n    if (Number.isFinite(y) && y > 1900 && y <= new Date().getUTCFullYear() + 1) {\n      return new Date(Date.UTC(y, 11, 31)).toISOString()\n    }\n  }\n  return undefined\n}\n\nfunction defaultDimensionHints(selector: CornellLiiSelector): string[] {\n  if (selector.kind === 'uscode') return ['jurisdictional_accuracy', 'citation_hygiene']\n  return ['citation_hygiene']\n}\n","import { sha256 } from '../ids'\nimport { htmlToText } from './html'\nimport { politeFetch } from './http'\nimport type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types'\n\n/**\n * IRS publications source.\n *\n * Two surfaces:\n *\n *   1. The publications index at https://www.irs.gov/publications enumerates\n *      every active publication with its revision year — a single fragment\n *      with the full table lets change detection notice when a publication\n *      year flips (e.g. Pub 15 (2025) → Pub 15 (2026)).\n *\n *   2. Individual publication landing pages at /publications/p<N>[<suffix>]\n *      return one fragment per publication with summary text. Callers list\n *      the publications they need tracked via `selectors`.\n *\n * Revenue procedures are fetched under their numbered URLs; the IRS does\n * not maintain a stable HTML index of rev-procs, so the caller passes the\n * specific rev-proc paths they care about.\n *\n * @stable\n */\n\nconst BASE_URL = 'https://www.irs.gov'\nconst INDEX_URL = `${BASE_URL}/publications`\n\nexport interface IrsPublicationsSourceOptions {\n  /**\n   * Specific publication slugs to fetch (e.g. `['p15', 'p17', 'p463']`).\n   * When `includeIndex` is true (default), the publications index page is\n   * also fetched as a single fragment so change detection can notice\n   * year/revision shifts across the whole catalogue.\n   */\n  publications?: string[]\n  /**\n   * Revenue procedure paths to fetch (e.g. `['/irb/2024-31_IRB']`). The\n   * caller passes the exact path; this source does not auto-discover.\n   */\n  revenueProcedures?: string[]\n  includeIndex?: boolean\n  id?: string\n}\n\n/** Default eval dimensions for IRS-sourced fragments. */\nexport const IRS_DIMENSION_HINTS = ['tax_compliance', 'regulatory_currency', 'citation_hygiene']\n\nexport function createIrsPublicationsSource(\n  options: IrsPublicationsSourceOptions = {},\n): KnowledgeSource {\n  const id = options.id ?? 'irs-publications'\n  const includeIndex = options.includeIndex ?? true\n  return {\n    id,\n    name: 'IRS Publications',\n    description:\n      'Internal Revenue Service publications index and individual publication landing pages from irs.gov.',\n    async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> {\n      const out: KnowledgeFragment[] = []\n      const limit = opts.limit ?? Number.POSITIVE_INFINITY\n\n      if (includeIndex && out.length < limit) {\n        out.push(await fetchIndex(id, opts))\n      }\n      for (const slug of options.publications ?? []) {\n        if (out.length >= limit) break\n        out.push(await fetchPublication(id, slug, opts))\n      }\n      for (const path of options.revenueProcedures ?? []) {\n        if (out.length >= limit) break\n        out.push(await fetchRevenueProcedure(id, path, opts))\n      }\n      return out\n    },\n  }\n}\n\nasync function fetchIndex(sourceId: string, opts: FetchOpts): Promise<KnowledgeFragment> {\n  const response = await politeFetch(INDEX_URL, { signal: opts.signal, cacheDir: opts.cacheDir })\n  const tablePattern = /<table[\\s\\S]*?<\\/table>/gi\n  const matches = response.body.match(tablePattern) ?? []\n  // Extract the table that lists current-year publications. IRS publishes\n  // one table per year on the index; the most recent table is always the\n  // first that mentions a year ≥ current.\n  const tables = matches.map((t) => htmlToText(t))\n  const body = tables\n    .filter((t) => /Publication\\s*\\d+/i.test(t))\n    .join('\\n\\n')\n    .slice(0, 200_000)\n\n  const verifiable = response.verifiable && body.length > 200\n  return {\n    id: 'index',\n    title: 'IRS Publications Index',\n    body,\n    bodyHash: sha256(body),\n    provenance: {\n      url: INDEX_URL,\n      sourceUpdatedAt: response.sourceUpdatedAt,\n      fetchedAt: response.fetchedAt,\n      jurisdiction: 'US-FED',\n      verifiable,\n      unverifiableReason:\n        response.unverifiableReason ?? (verifiable ? undefined : 'no publication rows extracted'),\n    },\n    dimensionHints: IRS_DIMENSION_HINTS,\n    metadata: { sourceId, status: response.status, fromCache: response.fromCache, kind: 'index' },\n  }\n}\n\nasync function fetchPublication(\n  sourceId: string,\n  slug: string,\n  opts: FetchOpts,\n): Promise<KnowledgeFragment> {\n  const url = `${BASE_URL}/publications/${slug.replace(/^\\/+/, '')}`\n  const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir })\n\n  const title = extractTitle(response.body, `IRS Publication ${slug}`)\n  const body = extractMainContent(response.body)\n  const verifiable = response.verifiable && body.length > 200\n\n  return {\n    id: `publication:${slug}`,\n    title,\n    body,\n    bodyHash: sha256(body),\n    provenance: {\n      url,\n      sourceUpdatedAt: extractRevisionDate(response.body) ?? response.sourceUpdatedAt,\n      fetchedAt: response.fetchedAt,\n      jurisdiction: 'US-FED',\n      verifiable,\n      unverifiableReason:\n        response.unverifiableReason ?? (verifiable ? undefined : 'no publication body extracted'),\n    },\n    dimensionHints: IRS_DIMENSION_HINTS,\n    metadata: {\n      sourceId,\n      status: response.status,\n      fromCache: response.fromCache,\n      kind: 'publication',\n      slug,\n    },\n  }\n}\n\nasync function fetchRevenueProcedure(\n  sourceId: string,\n  path: string,\n  opts: FetchOpts,\n): Promise<KnowledgeFragment> {\n  const url = `${BASE_URL}${path.startsWith('/') ? path : `/${path}`}`\n  const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir })\n  const body = extractMainContent(response.body)\n  const verifiable = response.verifiable && body.length > 200\n  return {\n    id: `rev-proc:${path}`,\n    title: extractTitle(response.body, `IRS Revenue Procedure ${path}`),\n    body,\n    bodyHash: sha256(body),\n    provenance: {\n      url,\n      sourceUpdatedAt: response.sourceUpdatedAt,\n      fetchedAt: response.fetchedAt,\n      jurisdiction: 'US-FED',\n      verifiable,\n      unverifiableReason:\n        response.unverifiableReason ??\n        (verifiable ? undefined : 'no revenue-procedure body extracted'),\n    },\n    dimensionHints: [...IRS_DIMENSION_HINTS, 'procedural_currency'],\n    metadata: {\n      sourceId,\n      status: response.status,\n      fromCache: response.fromCache,\n      kind: 'rev-proc',\n      path,\n    },\n  }\n}\n\nfunction extractTitle(html: string, fallback: string): string {\n  const og = /<meta\\s+property=[\"']og:title[\"']\\s+content=[\"']([^\"']+)[\"']/i.exec(html)?.[1]\n  if (og) return decodeHtml(og)\n  const title = /<title>([\\s\\S]*?)<\\/title>/i.exec(html)?.[1]\n  if (title) return htmlToText(title).split(' | ')[0] ?? fallback\n  return fallback\n}\n\nfunction extractMainContent(html: string): string {\n  // IRS uses Drupal — the main publication body is inside <main role=\"main\">\n  // or under .field--name-body. We try main first; on miss, body.\n  const main = /<main\\b[\\s\\S]*?<\\/main>/i.exec(html)?.[0]\n  if (main) {\n    const noNav = main\n      .replace(/<nav[\\s\\S]*?<\\/nav>/gi, '')\n      .replace(/<header[\\s\\S]*?<\\/header>/gi, '')\n      .replace(/<footer[\\s\\S]*?<\\/footer>/gi, '')\n    return htmlToText(noNav).slice(0, 200_000)\n  }\n  const body = /<body\\b[\\s\\S]*?<\\/body>/i.exec(html)?.[0]\n  return body ? htmlToText(body).slice(0, 200_000) : htmlToText(html).slice(0, 200_000)\n}\n\nfunction extractRevisionDate(html: string): string | undefined {\n  // IRS publication pages typically show \"Publication X (YYYY)\" in the title;\n  // pulling the year gives a stable revision marker.\n  const m = /Publication\\s+\\S+\\s*\\((\\d{4})\\)/i.exec(html)\n  if (m?.[1]) {\n    const year = Number.parseInt(m[1], 10)\n    if (Number.isFinite(year) && year >= 2000 && year <= new Date().getUTCFullYear() + 1) {\n      return new Date(Date.UTC(year, 0, 1)).toISOString()\n    }\n  }\n  return undefined\n}\n\nfunction decodeHtml(value: string): string {\n  return htmlToText(value)\n}\n","import { sha256 } from '../ids'\nimport { htmlToText } from './html'\nimport { politeFetch } from './http'\nimport type { FetchOpts, KnowledgeFragment, KnowledgeSource } from './types'\n\n/**\n * Generic Secretary-of-State source.\n *\n * Every US state SOS surfaces LLC/Corp formation requirements differently\n * (CA via static forms pages, DE via division of corporations pages, TX\n * via SOSDirect content pages). Rather than baking 50 state-specific\n * parsers into this package, the source takes a config that names the URL\n * pattern + CSS-equivalent selector + jurisdiction tag. Callers supply one\n * config per state they need tracked.\n *\n * The selector is interpreted as a substring/regex of an HTML element id\n * or class — see `StateSosSourceConfig` for the contract. This is\n * intentionally minimal; richer extraction belongs in a state-specific\n * adapter the consumer authors.\n *\n * @experimental Interface will likely grow as we add more state coverage.\n */\n\nexport interface StateSosEntity {\n  /** Stable id for this fragment within the state (e.g. 'llc-formation', 'corp-formation'). */\n  id: string\n  /** Path under the configured `baseUrl` for this entity. */\n  path: string\n  /**\n   * Extraction selector. Choose one:\n   *   - `{ kind: 'id', value: 'main-content' }` — innermost match of element with that id\n   *   - `{ kind: 'class', value: 'field--name-body' }` — innermost match of element with that class\n   *   - `{ kind: 'regex', value: /<article[\\s\\S]*?<\\/article>/i }` — raw regex\n   *   - `{ kind: 'whole' }` — full body, tags stripped (fallback for unstructured pages)\n   */\n  selector:\n    | { kind: 'id'; value: string }\n    | { kind: 'class'; value: string }\n    | { kind: 'regex'; value: RegExp }\n    | { kind: 'whole' }\n  title: string\n  /** Eval dimensions this entity feeds. */\n  dimensionHints?: string[]\n}\n\nexport interface StateSosSourceConfig {\n  /** US state postal code, e.g. 'CA', 'DE', 'TX'. */\n  state: string\n  /** Base URL for the state SOS — e.g. 'https://www.sos.ca.gov'. */\n  baseUrl: string\n  /** Entities this state exposes (LLC, Corp, etc). */\n  entities: StateSosEntity[]\n  /** Source id; default `state-sos:<state>`. */\n  id?: string\n  /** Display name; default `<state> Secretary of State`. */\n  name?: string\n}\n\nexport function createStateSosSource(config: StateSosSourceConfig): KnowledgeSource {\n  const id = config.id ?? `state-sos:${config.state.toLowerCase()}`\n  const name = config.name ?? `${config.state} Secretary of State`\n  return {\n    id,\n    name,\n    description: `${config.state} Secretary of State filings and formation guidance pages.`,\n    async fetch(opts: FetchOpts): Promise<KnowledgeFragment[]> {\n      const limit = opts.limit ?? config.entities.length\n      const entities = config.entities.slice(0, limit)\n      const out: KnowledgeFragment[] = []\n      for (const entity of entities) {\n        out.push(await fetchEntity(id, config, entity, opts))\n      }\n      return out\n    },\n  }\n}\n\nasync function fetchEntity(\n  sourceId: string,\n  config: StateSosSourceConfig,\n  entity: StateSosEntity,\n  opts: FetchOpts,\n): Promise<KnowledgeFragment> {\n  const url = joinUrl(config.baseUrl, entity.path)\n  const response = await politeFetch(url, { signal: opts.signal, cacheDir: opts.cacheDir })\n\n  const body = response.verifiable ? extractBySelector(response.body, entity.selector) : ''\n  const verifiable = response.verifiable && body.length > 100\n\n  return {\n    id: entity.id,\n    title: entity.title,\n    body,\n    bodyHash: sha256(body),\n    provenance: {\n      url,\n      sourceUpdatedAt: response.sourceUpdatedAt,\n      fetchedAt: response.fetchedAt,\n      jurisdiction: `US-${config.state.toUpperCase()}`,\n      verifiable,\n      unverifiableReason:\n        response.unverifiableReason ?? (verifiable ? undefined : 'extracted body too short'),\n    },\n    dimensionHints: entity.dimensionHints ?? [\n      'jurisdictional_accuracy',\n      'corporate_formation',\n      'citation_hygiene',\n    ],\n    metadata: {\n      sourceId,\n      status: response.status,\n      fromCache: response.fromCache,\n      state: config.state,\n    },\n  }\n}\n\nfunction extractBySelector(html: string, selector: StateSosEntity['selector']): string {\n  if (selector.kind === 'whole') {\n    const main = /<main\\b[\\s\\S]*?<\\/main>/i.exec(html)?.[0]\n    return htmlToText(main ?? html).slice(0, 200_000)\n  }\n  if (selector.kind === 'regex') {\n    const m = selector.value.exec(html)?.[0]\n    return m ? htmlToText(m).slice(0, 200_000) : ''\n  }\n  if (selector.kind === 'id') {\n    const escaped = selector.value.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n    const pattern = new RegExp(\n      `<([a-z][a-z0-9]*)\\\\b[^>]*\\\\sid=[\"']${escaped}[\"'][^>]*>([\\\\s\\\\S]*?)<\\\\/\\\\1>`,\n      'i',\n    )\n    const inner = pattern.exec(html)?.[2]\n    return inner ? htmlToText(inner).slice(0, 200_000) : ''\n  }\n  const escaped = selector.value.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')\n  const pattern = new RegExp(\n    `<([a-z][a-z0-9]*)\\\\b[^>]*\\\\sclass=[\"'][^\"']*\\\\b${escaped}\\\\b[^\"']*[\"'][^>]*>([\\\\s\\\\S]*?)<\\\\/\\\\1>`,\n    'i',\n  )\n  const inner = pattern.exec(html)?.[2]\n  return inner ? htmlToText(inner).slice(0, 200_000) : ''\n}\n\nfunction joinUrl(base: string, path: string): string {\n  try {\n    return new URL(path, base.endsWith('/') ? base : `${base}/`).toString()\n  } catch {\n    return `${base.replace(/\\/+$/, '')}/${path.replace(/^\\/+/, '')}`\n  }\n}\n"],"mappings":";;;;;AAqBO,SAAS,WAAW,MAAsB;AAC/C,SAAO,KACJ,QAAQ,+BAA+B,EAAE,EACzC,QAAQ,6BAA6B,EAAE,EACvC,QAAQ,mCAAmC,EAAE,EAC7C,QAAQ,sBAAsB,EAAE,EAChC,QAAQ,mBAAmB,IAAI,EAC/B,QAAQ,yDAAyD,IAAI,EACrE,QAAQ,YAAY,EAAE,EACtB,QAAQ,YAAY,GAAG,EACvB,QAAQ,WAAW,GAAG,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,UAAU,GAAG,EACrB,QAAQ,YAAY,GAAG,EACvB,QAAQ,WAAW,GAAG,EACtB,QAAQ,YAAY,MAAG,EACvB,QAAQ,aAAa,QAAG,EACxB,QAAQ,aAAa,QAAG,EACxB,QAAQ,aAAa,CAAC,GAAG,SAAS,OAAO,cAAc,OAAO,IAAI,CAAC,CAAC,EACpE,QAAQ,qBAAqB,CAAC,GAAG,SAAS,OAAO,cAAc,OAAO,SAAS,MAAM,EAAE,CAAC,CAAC,EACzF,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,QAAQ,YAAY,GAAG,EAAE,KAAK,CAAC,EAClD,OAAO,CAAC,MAAM,KAAK,QAAQ,EAAE,SAAS,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,EAChE,KAAK,IAAI,EACT,KAAK;AACV;AAGO,SAAS,WAAW,MAAc,SAAqC;AAC5E,SAAO,QAAQ,KAAK,IAAI,IAAI,CAAC,GAAG,KAAK;AACvC;AAGO,SAAS,cAAc,MAAc,IAAgC;AAC1E,QAAM,UAAU,GAAG,QAAQ,uBAAuB,MAAM;AACxD,QAAM,aAAa,IAAI;AAAA,IACrB,sCAAsC,OAAO;AAAA,IAC7C;AAAA,EACF;AACA,SAAO,WAAW,KAAK,IAAI,IAAI,CAAC;AAClC;AAMO,SAAS,aACd,MACA,aACA,SACkC;AAClC,QAAM,MAAwC,CAAC;AAC/C,QAAM,SAAS;AACf,aAAW,SAAS,KAAK,SAAS,MAAM,GAAG;AACzC,UAAM,OAAO,MAAM,CAAC;AACpB,UAAM,QAAQ,MAAM,CAAC;AACrB,QAAI,CAAC,QAAQ,CAAC,MAAO;AACrB,QAAI,CAAC,YAAY,KAAK,IAAI,EAAG;AAC7B,UAAM,OAAO,WAAW,KAAK;AAC7B,QAAI,CAAC,KAAM;AACX,QAAI;AACF,UAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,OAAO,EAAE,SAAS,GAAG,KAAK,CAAC;AAAA,IAC5D,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;;;ACxFA,SAAS,OAAO,UAAU,MAAM,iBAAiB;AACjD,SAAS,SAAS,YAAY;AAcvB,IAAM,oBACX;AAGK,IAAM,qBAAqB;AAG3B,IAAM,qBAAqB,IAAI,OAAO;AAE7C,IAAM,eAAe,oBAAI,IAA2B;AAiDpD,eAAsB,YACpB,KACA,UAA8B,CAAC,GACH;AAC5B,QAAM,WAAW,QAAQ,cAAc,KAAK,KAAK;AACjD,QAAM,SAAS,QAAQ,WAAW,MAAM,UAAU,QAAQ,UAAU,KAAK,QAAQ,IAAI;AACrF,MAAI,OAAQ,QAAO;AAEnB,QAAM,OAAO,SAAS,GAAG;AACzB,QAAM,aAAa,IAAI;AAEvB,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AACzC,MAAI;AACJ,MAAI;AACF,eAAW,MAAM,MAAM,KAAK;AAAA,MAC1B,QAAQ,QAAQ;AAAA,MAChB,UAAU;AAAA,MACV,SAAS;AAAA,QACP,cAAc;AAAA,QACd,QAAQ;AAAA,QACR,mBAAmB;AAAA,QACnB,GAAI,QAAQ,WAAW,CAAC;AAAA,MAC1B;AAAA,IACF,CAAC;AAAA,EACH,SAAS,OAAO;AACd,QAAK,MAA4B,SAAS,aAAc,OAAM;AAC9D,UAAMA,UAA4B;AAAA,MAChC;AAAA,MACA,QAAQ;AAAA,MACR,MAAM;AAAA,MACN,iBAAiB;AAAA,MACjB;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,oBAAoB,kBAAmB,MAAgB,OAAO;AAAA,IAChE;AACA,QAAI,QAAQ,SAAU,OAAM,WAAW,QAAQ,UAAU,KAAKA,OAAM;AACpE,WAAOA;AAAA,EACT;AAEA,QAAM,OAAO,MAAM,gBAAgB,QAAQ;AAC3C,QAAM,eAAe,SAAS,QAAQ,IAAI,eAAe;AACzD,QAAM,aAAa,SAAS,QAAQ,IAAI,MAAM;AAC9C,QAAM,kBAAkB,cAAc,YAAY,KAAK,cAAc,UAAU,KAAK;AAEpF,QAAM,SAA4B;AAAA,IAChC;AAAA,IACA,QAAQ,SAAS;AAAA,IACjB,MAAM;AAAA,IACN;AAAA,IACA;AAAA,IACA,WAAW;AAAA,IACX,YAAY;AAAA,EACd;AAEA,MAAI,SAAS,SAAS,OAAO,SAAS,UAAU,KAAK;AACnD,WAAO,aAAa;AACpB,WAAO,qBAAqB,mBAAmB,SAAS,MAAM;AAAA,EAChE,WAAW,mBAAmB,IAAI,GAAG;AACnC,WAAO,aAAa;AACpB,WAAO,qBAAqB;AAAA,EAC9B,WAAW,KAAK,SAAS,OAAO,oBAAoB,IAAI,GAAG;AACzD,WAAO,aAAa;AACpB,WAAO,qBAAqB,+BAA+B,KAAK,MAAM;AAAA,EACxE;AAEA,MAAI,QAAQ,SAAU,OAAM,WAAW,QAAQ,UAAU,KAAK,MAAM;AACpE,SAAO;AACT;AAGO,SAAS,sBAA4B;AAC1C,eAAa,MAAM;AACrB;AAEA,SAAS,SAAS,KAAqB;AACrC,MAAI;AACF,WAAO,IAAI,IAAI,GAAG,EAAE;AAAA,EACtB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAe,aAAa,MAA6B;AACvD,QAAM,OAAO,aAAa,IAAI,IAAI,KAAK,QAAQ,QAAQ;AACvD,MAAI,UAAsB,MAAM;AAAA,EAAC;AACjC,QAAM,OAAO,IAAI,QAAc,CAAC,YAAY;AAC1C,cAAU;AAAA,EACZ,CAAC;AACD,eAAa;AAAA,IACX;AAAA,IACA,KAAK,KAAK,MAAM,IAAI;AAAA,EACtB;AACA,QAAM;AACN,aAAW,SAAS,kBAAkB;AACxC;AAEA,eAAe,gBAAgB,UAAqC;AAClE,MAAI,CAAC,SAAS,KAAM,QAAO;AAC3B,QAAM,SAAS,SAAS,KAAK,UAAU;AACvC,QAAM,SAAuB,CAAC;AAC9B,MAAI,QAAQ;AACZ,SAAO,MAAM;AACX,UAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,QAAI,KAAM;AACV,QAAI,CAAC,MAAO;AACZ,aAAS,MAAM;AACf,QAAI,QAAQ,oBAAoB;AAE9B,YAAM,OAAO,OAAO;AACpB;AAAA,IACF;AACA,WAAO,KAAK,KAAK;AAAA,EACnB;AACA,QAAM,SAAS,IAAI,WAAW,KAAK,IAAI,OAAO,kBAAkB,CAAC;AACjE,MAAI,SAAS;AACb,aAAW,SAAS,QAAQ;AAC1B,UAAM,OAAO,KAAK,IAAI,MAAM,QAAQ,OAAO,SAAS,MAAM;AAC1D,QAAI,QAAQ,EAAG;AACf,WAAO,IAAI,MAAM,SAAS,GAAG,IAAI,GAAG,MAAM;AAC1C,cAAU;AAAA,EACZ;AACA,SAAO,IAAI,YAAY,SAAS,EAAE,OAAO,MAAM,CAAC,EAAE,OAAO,MAAM;AACjE;AAEA,SAAS,cAAc,OAA0C;AAC/D,MAAI,CAAC,MAAO,QAAO;AACnB,QAAM,KAAK,KAAK,MAAM,KAAK;AAC3B,SAAO,OAAO,SAAS,EAAE,IAAI,IAAI,KAAK,EAAE,EAAE,YAAY,IAAI;AAC5D;AAGO,SAAS,mBAAmB,MAAuB;AACxD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,QAAQ,KAAK,YAAY;AAC/B,QAAM,UAAU;AAAA,IACd;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,aAAW,UAAU,SAAS;AAC5B,QAAI,MAAM,SAAS,MAAM,EAAG,QAAO;AAAA,EACrC;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,MAAuB;AAClD,SACE,KAAK,SAAS,iBAAiB,KAC/B,KAAK,SAAS,SAAS,KACvB,KAAK,SAAS,YAAY,KAC1B,KAAK,SAAS,iBAAiB,KAC/B,KAAK,SAAS,cAAc;AAEhC;AAEA,SAAS,UAAU,UAAkB,KAAqB;AACxD,QAAM,MAAM,OAAO,GAAG;AACtB,SAAO,KAAK,UAAU,QAAQ,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,OAAO;AACnE;AAEA,eAAe,UACb,UACA,KACA,OACwC;AACxC,QAAM,OAAO,UAAU,UAAU,GAAG;AACpC,MAAI;AACF,UAAM,OAAO,MAAM,KAAK,IAAI;AAC5B,QAAI,KAAK,IAAI,IAAI,KAAK,UAAU,MAAO,QAAO;AAC9C,UAAM,MAAM,MAAM,SAAS,MAAM,MAAM;AACvC,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,WAAO,EAAE,GAAG,QAAQ,WAAW,KAAK;AAAA,EACtC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAe,WAAW,UAAkB,KAAa,OAAyC;AAChG,QAAM,OAAO,UAAU,UAAU,GAAG;AACpC,QAAM,MAAM,QAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AAC9C,QAAM,UAAU,MAAM,KAAK,UAAU,KAAK,GAAG,MAAM;AACrD;;;ACrPA,IAAM,WAAW;AA0CV,SAAS,uBAAuB,SAAmD;AACxF,QAAM,KAAK,QAAQ,MAAM;AACzB,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,aACE;AAAA,IACF,MAAM,MAAM,MAA+C;AACzD,YAAM,QAAQ,KAAK,SAAS,QAAQ,UAAU;AAC9C,YAAM,YAAY,QAAQ,UAAU,MAAM,GAAG,KAAK;AAClD,YAAM,MAA2B,CAAC;AAClC,iBAAW,YAAY,WAAW;AAChC,YAAI,KAAK,MAAM,SAAS,IAAI,UAAU,IAAI,CAAC;AAAA,MAC7C;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,eAAe,SACb,UACA,UACA,MAC4B;AAC5B,QAAM,OAAO,SAAS,KAAK,QAAQ,QAAQ,EAAE;AAC7C,QAAM,MACJ,SAAS,SAAS,WAAW,GAAG,QAAQ,gBAAgB,IAAI,KAAK,GAAG,QAAQ,QAAQ,IAAI;AAE1F,QAAM,WAAW,MAAM,YAAY,KAAK;AAAA,IACtC,QAAQ,KAAK;AAAA,IACb,UAAU,KAAK;AAAA,EACjB,CAAC;AAED,QAAM,aAAa,GAAG,SAAS,IAAI,IAAI,SAAS,IAAI;AACpD,QAAM,iBAAiB,SAAS,kBAAkB,sBAAsB,QAAQ;AAEhF,MAAI,CAAC,SAAS,YAAY;AACxB,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO,eAAe,SAAS,IAAI,IAAI,SAAS,IAAI;AAAA,MACpD,MAAM;AAAA,MACN,UAAU,OAAO,EAAE;AAAA,MACnB,YAAY;AAAA,QACV;AAAA,QACA,iBAAiB,SAAS;AAAA,QAC1B,WAAW,SAAS;AAAA,QACpB,cAAc;AAAA,QACd,YAAY;AAAA,QACZ,oBAAoB,SAAS;AAAA,MAC/B;AAAA,MACA;AAAA,MACA,UAAU,EAAE,UAAU,QAAQ,SAAS,QAAQ,WAAW,SAAS,UAAU;AAAA,IAC/E;AAAA,EACF;AAEA,QAAM,OAAO,SAAS;AACtB,QAAM,QAAQ,aAAa,MAAM,QAAQ;AACzC,QAAM,OAAO,YAAY,MAAM,QAAQ;AACvC,QAAM,YAAY,qBAAqB,IAAI,KAAK,SAAS;AAEzD,QAAM,aAAa,KAAK,SAAS;AACjC,SAAO;AAAA,IACL,IAAI;AAAA,IACJ;AAAA,IACA;AAAA,IACA,UAAU,OAAO,IAAI;AAAA,IACrB,YAAY;AAAA,MACV;AAAA,MACA,iBAAiB;AAAA,MACjB,WAAW,SAAS;AAAA,MACpB,cAAc;AAAA,MACd;AAAA,MACA,oBAAoB,aAAa,SAAY;AAAA,IAC/C;AAAA,IACA;AAAA,IACA,UAAU,EAAE,UAAU,QAAQ,SAAS,QAAQ,WAAW,SAAS,UAAU;AAAA,EAC/E;AACF;AAEA,SAAS,aAAa,MAAc,UAAsC;AACxE,QAAM,KAAK,yDAAyD,KAAK,IAAI,IAAI,CAAC;AAClF,MAAI,GAAI,QAAO,WAAW,EAAE;AAC5B,QAAM,IAAI,8BAA8B,KAAK,IAAI,IAAI,CAAC;AACtD,MAAI,EAAG,QAAO,WAAW,CAAC,EAAE,MAAM,KAAK,EAAE,CAAC,KAAK,eAAe,SAAS,IAAI;AAC3E,SAAO,eAAe,SAAS,IAAI,IAAI,SAAS,IAAI;AACtD;AAEA,SAAS,YAAY,MAAc,UAAsC;AACvE,MAAI,SAAS,SAAS,UAAU;AAI9B,UAAM,OAAO,4BAA4B,KAAK,IAAI,IAAI,CAAC;AACvD,QAAI,KAAM,QAAO,WAAW,IAAI;AAChC,UAAM,MAAM,cAAc,MAAM,eAAe;AAC/C,QAAI,IAAK,QAAO,WAAW,GAAG;AAAA,EAChC;AAKA,QAAM,cAAc,cAAc,MAAM,cAAc;AACtD,MAAI,aAAa;AACf,WAAO,WAAW,YAAY,QAAQ,sBAAsB,EAAE,CAAC;AAAA,EACjE;AACA,QAAM,YAAY,cAAc,MAAM,mBAAmB;AACzD,MAAI,WAAW;AACb,WAAO,WAAW,UAAU,QAAQ,sBAAsB,EAAE,CAAC;AAAA,EAC/D;AACA,SAAO,WAAW,IAAI;AACxB;AAEA,SAAS,qBAAqB,MAAkC;AAI9D,QAAM,QAAQ,mCAAmC,KAAK,IAAI,IAAI,CAAC;AAC/D,MAAI,OAAO;AACT,UAAM,IAAI,OAAO,SAAS,OAAO,EAAE;AACnC,QAAI,OAAO,SAAS,CAAC,KAAK,IAAI,QAAQ,MAAK,oBAAI,KAAK,GAAE,eAAe,IAAI,GAAG;AAC1E,aAAO,IAAI,KAAK,KAAK,IAAI,GAAG,IAAI,EAAE,CAAC,EAAE,YAAY;AAAA,IACnD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,sBAAsB,UAAwC;AACrE,MAAI,SAAS,SAAS,SAAU,QAAO,CAAC,2BAA2B,kBAAkB;AACrF,SAAO,CAAC,kBAAkB;AAC5B;;;ACjKA,IAAMC,YAAW;AACjB,IAAM,YAAY,GAAGA,SAAQ;AAoBtB,IAAM,sBAAsB,CAAC,kBAAkB,uBAAuB,kBAAkB;AAExF,SAAS,4BACd,UAAwC,CAAC,GACxB;AACjB,QAAM,KAAK,QAAQ,MAAM;AACzB,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,aACE;AAAA,IACF,MAAM,MAAM,MAA+C;AACzD,YAAM,MAA2B,CAAC;AAClC,YAAM,QAAQ,KAAK,SAAS,OAAO;AAEnC,UAAI,gBAAgB,IAAI,SAAS,OAAO;AACtC,YAAI,KAAK,MAAM,WAAW,IAAI,IAAI,CAAC;AAAA,MACrC;AACA,iBAAW,QAAQ,QAAQ,gBAAgB,CAAC,GAAG;AAC7C,YAAI,IAAI,UAAU,MAAO;AACzB,YAAI,KAAK,MAAM,iBAAiB,IAAI,MAAM,IAAI,CAAC;AAAA,MACjD;AACA,iBAAW,QAAQ,QAAQ,qBAAqB,CAAC,GAAG;AAClD,YAAI,IAAI,UAAU,MAAO;AACzB,YAAI,KAAK,MAAM,sBAAsB,IAAI,MAAM,IAAI,CAAC;AAAA,MACtD;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,eAAe,WAAW,UAAkB,MAA6C;AACvF,QAAM,WAAW,MAAM,YAAY,WAAW,EAAE,QAAQ,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC;AAC9F,QAAM,eAAe;AACrB,QAAM,UAAU,SAAS,KAAK,MAAM,YAAY,KAAK,CAAC;AAItD,QAAM,SAAS,QAAQ,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;AAC/C,QAAM,OAAO,OACV,OAAO,CAAC,MAAM,qBAAqB,KAAK,CAAC,CAAC,EAC1C,KAAK,MAAM,EACX,MAAM,GAAG,GAAO;AAEnB,QAAM,aAAa,SAAS,cAAc,KAAK,SAAS;AACxD,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO;AAAA,IACP;AAAA,IACA,UAAU,OAAO,IAAI;AAAA,IACrB,YAAY;AAAA,MACV,KAAK;AAAA,MACL,iBAAiB,SAAS;AAAA,MAC1B,WAAW,SAAS;AAAA,MACpB,cAAc;AAAA,MACd;AAAA,MACA,oBACE,SAAS,uBAAuB,aAAa,SAAY;AAAA,IAC7D;AAAA,IACA,gBAAgB;AAAA,IAChB,UAAU,EAAE,UAAU,QAAQ,SAAS,QAAQ,WAAW,SAAS,WAAW,MAAM,QAAQ;AAAA,EAC9F;AACF;AAEA,eAAe,iBACb,UACA,MACA,MAC4B;AAC5B,QAAM,MAAM,GAAGA,SAAQ,iBAAiB,KAAK,QAAQ,QAAQ,EAAE,CAAC;AAChE,QAAM,WAAW,MAAM,YAAY,KAAK,EAAE,QAAQ,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC;AAExF,QAAM,QAAQC,cAAa,SAAS,MAAM,mBAAmB,IAAI,EAAE;AACnE,QAAM,OAAO,mBAAmB,SAAS,IAAI;AAC7C,QAAM,aAAa,SAAS,cAAc,KAAK,SAAS;AAExD,SAAO;AAAA,IACL,IAAI,eAAe,IAAI;AAAA,IACvB;AAAA,IACA;AAAA,IACA,UAAU,OAAO,IAAI;AAAA,IACrB,YAAY;AAAA,MACV;AAAA,MACA,iBAAiB,oBAAoB,SAAS,IAAI,KAAK,SAAS;AAAA,MAChE,WAAW,SAAS;AAAA,MACpB,cAAc;AAAA,MACd;AAAA,MACA,oBACE,SAAS,uBAAuB,aAAa,SAAY;AAAA,IAC7D;AAAA,IACA,gBAAgB;AAAA,IAChB,UAAU;AAAA,MACR;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,WAAW,SAAS;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,sBACb,UACA,MACA,MAC4B;AAC5B,QAAM,MAAM,GAAGD,SAAQ,GAAG,KAAK,WAAW,GAAG,IAAI,OAAO,IAAI,IAAI,EAAE;AAClE,QAAM,WAAW,MAAM,YAAY,KAAK,EAAE,QAAQ,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC;AACxF,QAAM,OAAO,mBAAmB,SAAS,IAAI;AAC7C,QAAM,aAAa,SAAS,cAAc,KAAK,SAAS;AACxD,SAAO;AAAA,IACL,IAAI,YAAY,IAAI;AAAA,IACpB,OAAOC,cAAa,SAAS,MAAM,yBAAyB,IAAI,EAAE;AAAA,IAClE;AAAA,IACA,UAAU,OAAO,IAAI;AAAA,IACrB,YAAY;AAAA,MACV;AAAA,MACA,iBAAiB,SAAS;AAAA,MAC1B,WAAW,SAAS;AAAA,MACpB,cAAc;AAAA,MACd;AAAA,MACA,oBACE,SAAS,uBACR,aAAa,SAAY;AAAA,IAC9B;AAAA,IACA,gBAAgB,CAAC,GAAG,qBAAqB,qBAAqB;AAAA,IAC9D,UAAU;AAAA,MACR;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,WAAW,SAAS;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAASA,cAAa,MAAc,UAA0B;AAC5D,QAAM,KAAK,gEAAgE,KAAK,IAAI,IAAI,CAAC;AACzF,MAAI,GAAI,QAAO,WAAW,EAAE;AAC5B,QAAM,QAAQ,8BAA8B,KAAK,IAAI,IAAI,CAAC;AAC1D,MAAI,MAAO,QAAO,WAAW,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC,KAAK;AACvD,SAAO;AACT;AAEA,SAAS,mBAAmB,MAAsB;AAGhD,QAAM,OAAO,2BAA2B,KAAK,IAAI,IAAI,CAAC;AACtD,MAAI,MAAM;AACR,UAAM,QAAQ,KACX,QAAQ,yBAAyB,EAAE,EACnC,QAAQ,+BAA+B,EAAE,EACzC,QAAQ,+BAA+B,EAAE;AAC5C,WAAO,WAAW,KAAK,EAAE,MAAM,GAAG,GAAO;AAAA,EAC3C;AACA,QAAM,OAAO,2BAA2B,KAAK,IAAI,IAAI,CAAC;AACtD,SAAO,OAAO,WAAW,IAAI,EAAE,MAAM,GAAG,GAAO,IAAI,WAAW,IAAI,EAAE,MAAM,GAAG,GAAO;AACtF;AAEA,SAAS,oBAAoB,MAAkC;AAG7D,QAAM,IAAI,mCAAmC,KAAK,IAAI;AACtD,MAAI,IAAI,CAAC,GAAG;AACV,UAAM,OAAO,OAAO,SAAS,EAAE,CAAC,GAAG,EAAE;AACrC,QAAI,OAAO,SAAS,IAAI,KAAK,QAAQ,OAAQ,SAAQ,oBAAI,KAAK,GAAE,eAAe,IAAI,GAAG;AACpF,aAAO,IAAI,KAAK,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,EAAE,YAAY;AAAA,IACpD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,WAAW,OAAuB;AACzC,SAAO,WAAW,KAAK;AACzB;;;ACpKO,SAAS,qBAAqB,QAA+C;AAClF,QAAM,KAAK,OAAO,MAAM,aAAa,OAAO,MAAM,YAAY,CAAC;AAC/D,QAAM,OAAO,OAAO,QAAQ,GAAG,OAAO,KAAK;AAC3C,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,aAAa,GAAG,OAAO,KAAK;AAAA,IAC5B,MAAM,MAAM,MAA+C;AACzD,YAAM,QAAQ,KAAK,SAAS,OAAO,SAAS;AAC5C,YAAM,WAAW,OAAO,SAAS,MAAM,GAAG,KAAK;AAC/C,YAAM,MAA2B,CAAC;AAClC,iBAAW,UAAU,UAAU;AAC7B,YAAI,KAAK,MAAM,YAAY,IAAI,QAAQ,QAAQ,IAAI,CAAC;AAAA,MACtD;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEA,eAAe,YACb,UACA,QACA,QACA,MAC4B;AAC5B,QAAM,MAAM,QAAQ,OAAO,SAAS,OAAO,IAAI;AAC/C,QAAM,WAAW,MAAM,YAAY,KAAK,EAAE,QAAQ,KAAK,QAAQ,UAAU,KAAK,SAAS,CAAC;AAExF,QAAM,OAAO,SAAS,aAAa,kBAAkB,SAAS,MAAM,OAAO,QAAQ,IAAI;AACvF,QAAM,aAAa,SAAS,cAAc,KAAK,SAAS;AAExD,SAAO;AAAA,IACL,IAAI,OAAO;AAAA,IACX,OAAO,OAAO;AAAA,IACd;AAAA,IACA,UAAU,OAAO,IAAI;AAAA,IACrB,YAAY;AAAA,MACV;AAAA,MACA,iBAAiB,SAAS;AAAA,MAC1B,WAAW,SAAS;AAAA,MACpB,cAAc,MAAM,OAAO,MAAM,YAAY,CAAC;AAAA,MAC9C;AAAA,MACA,oBACE,SAAS,uBAAuB,aAAa,SAAY;AAAA,IAC7D;AAAA,IACA,gBAAgB,OAAO,kBAAkB;AAAA,MACvC;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,IACA,UAAU;AAAA,MACR;AAAA,MACA,QAAQ,SAAS;AAAA,MACjB,WAAW,SAAS;AAAA,MACpB,OAAO,OAAO;AAAA,IAChB;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,MAAc,UAA8C;AACrF,MAAI,SAAS,SAAS,SAAS;AAC7B,UAAM,OAAO,2BAA2B,KAAK,IAAI,IAAI,CAAC;AACtD,WAAO,WAAW,QAAQ,IAAI,EAAE,MAAM,GAAG,GAAO;AAAA,EAClD;AACA,MAAI,SAAS,SAAS,SAAS;AAC7B,UAAM,IAAI,SAAS,MAAM,KAAK,IAAI,IAAI,CAAC;AACvC,WAAO,IAAI,WAAW,CAAC,EAAE,MAAM,GAAG,GAAO,IAAI;AAAA,EAC/C;AACA,MAAI,SAAS,SAAS,MAAM;AAC1B,UAAMC,WAAU,SAAS,MAAM,QAAQ,uBAAuB,MAAM;AACpE,UAAMC,WAAU,IAAI;AAAA,MAClB,sCAAsCD,QAAO;AAAA,MAC7C;AAAA,IACF;AACA,UAAME,SAAQD,SAAQ,KAAK,IAAI,IAAI,CAAC;AACpC,WAAOC,SAAQ,WAAWA,MAAK,EAAE,MAAM,GAAG,GAAO,IAAI;AAAA,EACvD;AACA,QAAM,UAAU,SAAS,MAAM,QAAQ,uBAAuB,MAAM;AACpE,QAAM,UAAU,IAAI;AAAA,IAClB,kDAAkD,OAAO;AAAA,IACzD;AAAA,EACF;AACA,QAAM,QAAQ,QAAQ,KAAK,IAAI,IAAI,CAAC;AACpC,SAAO,QAAQ,WAAW,KAAK,EAAE,MAAM,GAAG,GAAO,IAAI;AACvD;AAEA,SAAS,QAAQ,MAAc,MAAsB;AACnD,MAAI;AACF,WAAO,IAAI,IAAI,MAAM,KAAK,SAAS,GAAG,IAAI,OAAO,GAAG,IAAI,GAAG,EAAE,SAAS;AAAA,EACxE,QAAQ;AACN,WAAO,GAAG,KAAK,QAAQ,QAAQ,EAAE,CAAC,IAAI,KAAK,QAAQ,QAAQ,EAAE,CAAC;AAAA,EAChE;AACF;","names":["result","BASE_URL","extractTitle","escaped","pattern","inner"]}

package/dist/cli.js

@@ -10,7 +10,7 @@ import {
   loadSourceRegistry,
   validateKnowledgeIndex,
   writeKnowledgeIndex
-} from "./chunk-EUAXSBMH.js";
+} from "./chunk-AKYJG2MR.js";
 import {
   detectKnowledgeGaps,
   findSurprisingConnections,