@tangle-network/agent-knowledge 5.0.4 → 6.1.0

package/dist/sources/index.js

@@ -1,34 +1,615 @@
-import {
-  IRS_DIMENSION_HINTS,
-  MAX_RESPONSE_BYTES,
-  MIN_REQUEST_GAP_MS,
-  POLITE_USER_AGENT,
-  __resetHttpThrottle,
-  createCornellLiiSource,
-  createIrsPublicationsSource,
-  createStateSosSource,
-  extractLinks,
-  firstMatch,
-  htmlToText,
-  innerHtmlById,
-  looksLikeBlockPage,
-  politeFetch
-} from "../chunk-MYFM6LKH.js";
-import "../chunk-YMKHCTS2.js";
-export {
-  IRS_DIMENSION_HINTS,
-  MAX_RESPONSE_BYTES,
-  MIN_REQUEST_GAP_MS,
-  POLITE_USER_AGENT,
-  __resetHttpThrottle,
-  createCornellLiiSource,
-  createIrsPublicationsSource,
-  createStateSosSource,
-  extractLinks,
-  firstMatch,
-  htmlToText,
-  innerHtmlById,
-  looksLikeBlockPage,
-  politeFetch
-};
+import { t as sha256 } from "../ids-DRqPZ42_.js";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+//#region src/sources/html.ts
+/**
+* Minimal HTML helpers used by the shipped sources.
+*
+* Deliberately not a full DOM parser: every authority we ship against
+* (Cornell LII, IRS.gov, state SOS portals) has well-behaved server-rendered
+* HTML where regex-based extraction is correct and cheap. Bringing in cheerio
+* would add a 1.5MB dependency to a package whose purpose is shipping
+* primitives, not parsing arbitrary web pages.
+*
+* If a future source needs real DOM traversal, it should depend on its own
+* parser locally rather than promoting one into the package-wide deps.
+*
+* @stable
+*/
+/**
+* Strip HTML tags, collapse whitespace, decode common entities.
+*
+* Preserves paragraph and line breaks (`</p>`, `<br>`, `</li>`, `</div>`,
+* `</h*>`) as `\n` so statute text retains its subsection structure.
+*/
+function htmlToText(html) {
+	return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<noscript[\s\S]*?<\/noscript>/gi, "").replace(/<!--([\s\S]*?)-->/g, "").replace(/<\s*br\s*\/?>/gi, "\n").replace(/<\/(p|li|div|tr|h[1-6]|blockquote|section|article)>/gi, "\n").replace(/<[^>]+>/g, "").replace(/&nbsp;/gi, " ").replace(/&amp;/gi, "&").replace(/&lt;/gi, "<").replace(/&gt;/gi, ">").replace(/&quot;/gi, "\"").replace(/&#39;/gi, "'").replace(/&sect;/gi, "§").replace(/&mdash;/gi, "—").replace(/&ndash;/gi, "–").replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))).replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(Number.parseInt(code, 16))).split("\n").map((line) => line.replace(/[\t  ]+/g, " ").trim()).filter((line, idx, all) => !(line === "" && all[idx - 1] === "")).join("\n").trim();
+}
+/** Extract the first match of a regex's first capture group, or undefined. */
+function firstMatch(html, pattern) {
+	return pattern.exec(html)?.[1]?.trim();
+}
+/** Extract the inner HTML of the first matching tag with id `id`. */
+function innerHtmlById(html, id) {
+	const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, "i").exec(html)?.[2];
+}
+/**
+* Extract every (href, text) pair matching the URL regex.
+* Returns absolute URLs by resolving against `baseUrl`.
+*/
+function extractLinks(html, hrefPattern, baseUrl) {
+	const out = [];
+	for (const match of html.matchAll(/<a\b[^>]*\shref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi)) {
+		const href = match[1];
+		const inner = match[2];
+		if (!href || !inner) continue;
+		if (!hrefPattern.test(href)) continue;
+		const text = htmlToText(inner);
+		if (!text) continue;
+		try {
+			out.push({
+				href: new URL(href, baseUrl).toString(),
+				text
+			});
+		} catch {}
+	}
+	return out;
+}
+//#endregion
+//#region src/sources/http.ts
+/**
+* Polite HTTP fetcher shared by remote sources.
+*
+* Independent sources share a per-origin throttle because rate-limited sites
+* may return block pages instead of 429 responses. Responses are cached by URL
+* because many publishers omit reliable ETag and Last-Modified headers. Bodies
+* are checked even after a 2xx response because captcha and block pages often
+* use successful status codes.
+*/
+/** User-Agent string sent on every outbound request. */
+const POLITE_USER_AGENT = "agent-knowledge (+https://github.com/tangle-network/agent-knowledge)";
+/** Minimum gap between successive requests to the same origin (ms). */
+const MIN_REQUEST_GAP_MS = 1e3;
+/** Maximum response body we will buffer in memory (bytes). */
+const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+const hostThrottle = /* @__PURE__ */ new Map();
+/**
+* Fetch one URL with per-host throttling, on-disk cache, and block-page
+* detection. Never throws on network/HTTP failure. It returns a result with
+* `verifiable: false` and `unverifiableReason` set so the caller can decide
+* whether to skip, retry, or surface.
+*
+* Throws ONLY on `AbortError` (caller asked to stop) and on cache-write
+* failures that indicate a misconfigured filesystem.
+*/
+async function politeFetch(url, options = {}) {
+	const cacheTtl = options.cacheTtlMs ?? 3600 * 1e3;
+	const cached = options.cacheDir ? await readCache(options.cacheDir, url, cacheTtl) : void 0;
+	if (cached) return cached;
+	const host = safeHost(url);
+	await throttleHost(host);
+	const fetchedAt = (/* @__PURE__ */ new Date()).toISOString();
+	let response;
+	try {
+		response = await fetch(url, {
+			signal: options.signal,
+			redirect: "follow",
+			headers: {
+				"User-Agent": POLITE_USER_AGENT,
+				Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				"Accept-Language": "en-US,en;q=0.9",
+				...options.headers ?? {}
+			}
+		});
+	} catch (error) {
+		if (error.name === "AbortError") throw error;
+		const result = {
+			url,
+			status: 0,
+			body: "",
+			sourceUpdatedAt: fetchedAt,
+			fetchedAt,
+			fromCache: false,
+			verifiable: false,
+			unverifiableReason: `network error: ${error.message}`
+		};
+		if (options.cacheDir) await writeCache(options.cacheDir, url, result);
+		return result;
+	}
+	const text = await readBoundedText(response);
+	const lastModified = response.headers.get("last-modified");
+	const dateHeader = response.headers.get("date");
+	const sourceUpdatedAt = parseHttpDate(lastModified) ?? parseHttpDate(dateHeader) ?? fetchedAt;
+	const result = {
+		url,
+		status: response.status,
+		body: text,
+		sourceUpdatedAt,
+		fetchedAt,
+		fromCache: false,
+		verifiable: true
+	};
+	if (response.status < 200 || response.status >= 300) {
+		result.verifiable = false;
+		result.unverifiableReason = `non-2xx status: ${response.status}`;
+	} else if (looksLikeBlockPage(text)) {
+		result.verifiable = false;
+		result.unverifiableReason = "block-page heuristic matched";
+	} else if (text.length < 200 && knownLargeAuthority(host)) {
+		result.verifiable = false;
+		result.unverifiableReason = `body shorter than expected (${text.length} chars)`;
+	}
+	if (options.cacheDir) await writeCache(options.cacheDir, url, result);
+	return result;
+}
+/** Reset the in-process throttle map. Test-only. */
+function __resetHttpThrottle() {
+	hostThrottle.clear();
+}
+function safeHost(url) {
+	try {
+		return new URL(url).host;
+	} catch {
+		return "unknown";
+	}
+}
+async function throttleHost(host) {
+	const prev = hostThrottle.get(host) ?? Promise.resolve();
+	let release = () => {};
+	const next = new Promise((resolve) => {
+		release = resolve;
+	});
+	hostThrottle.set(host, prev.then(() => next));
+	await prev;
+	setTimeout(release, MIN_REQUEST_GAP_MS);
+}
+async function readBoundedText(response) {
+	if (!response.body) return "";
+	const reader = response.body.getReader();
+	const chunks = [];
+	let total = 0;
+	while (true) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		if (!value) continue;
+		total += value.length;
+		if (total > 8388608) {
+			await reader.cancel();
+			break;
+		}
+		chunks.push(value);
+	}
+	const merged = new Uint8Array(Math.min(total, MAX_RESPONSE_BYTES));
+	let offset = 0;
+	for (const chunk of chunks) {
+		const take = Math.min(chunk.length, merged.length - offset);
+		if (take <= 0) break;
+		merged.set(chunk.subarray(0, take), offset);
+		offset += take;
+	}
+	return new TextDecoder("utf-8", { fatal: false }).decode(merged);
+}
+function parseHttpDate(value) {
+	if (!value) return void 0;
+	const ms = Date.parse(value);
+	return Number.isFinite(ms) ? new Date(ms).toISOString() : void 0;
+}
+/** Cheap heuristic that catches CAPTCHA, WAF block pages, and "Just a moment" interstitials. */
+function looksLikeBlockPage(body) {
+	if (!body) return false;
+	const lower = body.toLowerCase();
+	for (const marker of [
+		"verify you are human",
+		"please enable javascript and cookies",
+		"just a moment",
+		"access denied",
+		"request unsuccessful",
+		"cf-error-details",
+		"captcha",
+		"incapsula",
+		"pardon our interruption"
+	]) if (lower.includes(marker)) return true;
+	return false;
+}
+function knownLargeAuthority(host) {
+	return host.endsWith("law.cornell.edu") || host.endsWith("irs.gov") || host.endsWith("sos.ca.gov") || host.endsWith("sos.state.tx.us") || host.endsWith("sos.state.us");
+}
+function cachePath(cacheDir, url) {
+	const key = sha256(url);
+	return join(cacheDir, "http", `${key.slice(0, 2)}`, `${key}.json`);
+}
+async function readCache(cacheDir, url, ttlMs) {
+	const path = cachePath(cacheDir, url);
+	try {
+		const info = await stat(path);
+		if (Date.now() - info.mtimeMs > ttlMs) return void 0;
+		const raw = await readFile(path, "utf8");
+		return {
+			...JSON.parse(raw),
+			fromCache: true
+		};
+	} catch {
+		return;
+	}
+}
+async function writeCache(cacheDir, url, value) {
+	const path = cachePath(cacheDir, url);
+	await mkdir(dirname(path), { recursive: true });
+	await writeFile(path, JSON.stringify(value), "utf8");
+}
+//#endregion
+//#region src/sources/cornell-lii.ts
+/**
+* Cornell Legal Information Institute (LII) source.
+*
+* Pulls federal US Code sections and Wex encyclopedia entries — the two
+* Cornell LII surfaces an agent typically grounds against. The Wex
+* "non-compete" page is the canonical test case for the Ryan-LLC v. FTC
+* vacatur drift the continuous-ingestion story is designed to catch.
+*
+* @stable
+*/
+const BASE_URL$1 = "https://www.law.cornell.edu";
+/**
+* Build a Cornell LII source for the listed selectors.
+*
+* Example: track DTSA + non-compete:
+* ```
+* createCornellLiiSource({
+*   selectors: [
+*     { kind: 'uscode', path: '18/1836' },
+*     { kind: 'wex', path: 'non-compete', dimensionHints: ['jurisdictional_accuracy'] },
+*   ],
+* })
+* ```
+*/
+function createCornellLiiSource(options) {
+	const id = options.id ?? "cornell-lii";
+	return {
+		id,
+		name: "Cornell Legal Information Institute",
+		description: "Federal US Code sections (uscode/text/...) and Wex legal encyclopedia entries from law.cornell.edu.",
+		async fetch(opts) {
+			const limit = opts.limit ?? options.selectors.length;
+			const selectors = options.selectors.slice(0, limit);
+			const out = [];
+			for (const selector of selectors) out.push(await fetchOne(id, selector, opts));
+			return out;
+		}
+	};
+}
+async function fetchOne(sourceId, selector, opts) {
+	const path = selector.path.replace(/^\/+/, "");
+	const url = selector.kind === "uscode" ? `${BASE_URL$1}/uscode/text/${path}` : `${BASE_URL$1}/wex/${path}`;
+	const response = await politeFetch(url, {
+		signal: opts.signal,
+		cacheDir: opts.cacheDir
+	});
+	const fragmentId = `${selector.kind}:${selector.path}`;
+	const dimensionHints = selector.dimensionHints ?? defaultDimensionHints(selector);
+	if (!response.verifiable) return {
+		id: fragmentId,
+		title: `Cornell LII ${selector.kind} ${selector.path}`,
+		body: "",
+		bodyHash: sha256(""),
+		provenance: {
+			url,
+			sourceUpdatedAt: response.sourceUpdatedAt,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: "US-FED",
+			verifiable: false,
+			unverifiableReason: response.unverifiableReason
+		},
+		dimensionHints,
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache
+		}
+	};
+	const html = response.body;
+	const title = extractTitle$1(html, selector);
+	const body = extractBody(html, selector);
+	const effective = extractEffectiveDate(html) ?? response.sourceUpdatedAt;
+	const verifiable = body.length > 50;
+	return {
+		id: fragmentId,
+		title,
+		body,
+		bodyHash: sha256(body),
+		provenance: {
+			url,
+			sourceUpdatedAt: effective,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: "US-FED",
+			verifiable,
+			unverifiableReason: verifiable ? void 0 : "extracted body too short"
+		},
+		dimensionHints,
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache
+		}
+	};
+}
+function extractTitle$1(html, selector) {
+	const h1 = /<h1[^>]*\bid=["']page_title["'][^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1];
+	if (h1) return htmlToText(h1);
+	const t = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1];
+	if (t) return htmlToText(t).split(" | ")[0] ?? `Cornell LII ${selector.path}`;
+	return `Cornell LII ${selector.kind} ${selector.path}`;
+}
+function extractBody(html, selector) {
+	if (selector.kind === "uscode") {
+		const text = /<text>([\s\S]*?)<\/text>/i.exec(html)?.[1];
+		if (text) return htmlToText(text);
+		const tab = innerHtmlById(html, "tab_default_1");
+		if (tab) return htmlToText(tab);
+	}
+	const mainContent = innerHtmlById(html, "main-content");
+	if (mainContent) return htmlToText(mainContent.replace(/<h1[\s\S]*?<\/h1>/i, ""));
+	const extracted = innerHtmlById(html, "extracted-content");
+	if (extracted) return htmlToText(extracted.replace(/<h1[\s\S]*?<\/h1>/i, ""));
+	return htmlToText(html);
+}
+function extractEffectiveDate(html) {
+	const amend = /Amendments[\s\S]{0,200}?(\d{4})/i.exec(html)?.[1];
+	if (amend) {
+		const y = Number.parseInt(amend, 10);
+		if (Number.isFinite(y) && y > 1900 && y <= (/* @__PURE__ */ new Date()).getUTCFullYear() + 1) return new Date(Date.UTC(y, 11, 31)).toISOString();
+	}
+}
+function defaultDimensionHints(selector) {
+	if (selector.kind === "uscode") return ["jurisdictional_accuracy", "citation_hygiene"];
+	return ["citation_hygiene"];
+}
+//#endregion
+//#region src/sources/irs-publications.ts
+/**
+* IRS publications source.
+*
+* Two surfaces:
+*
+*   1. The publications index at https://www.irs.gov/publications enumerates
+*      every active publication with its revision year — a single fragment
+*      with the full table lets change detection notice when a publication
+*      year flips (e.g. Pub 15 (2025) → Pub 15 (2026)).
+*
+*   2. Individual publication landing pages at /publications/p<N>[<suffix>]
+*      return one fragment per publication with summary text. Callers list
+*      the publications they need tracked via `selectors`.
+*
+* Revenue procedures are fetched under their numbered URLs; the IRS does
+* not maintain a stable HTML index of rev-procs, so the caller passes the
+* specific rev-proc paths they care about.
+*
+* @stable
+*/
+const BASE_URL = "https://www.irs.gov";
+const INDEX_URL = `${BASE_URL}/publications`;
+/** Default eval dimensions for IRS-sourced fragments. */
+const IRS_DIMENSION_HINTS = [
+	"tax_compliance",
+	"regulatory_currency",
+	"citation_hygiene"
+];
+function createIrsPublicationsSource(options = {}) {
+	const id = options.id ?? "irs-publications";
+	const includeIndex = options.includeIndex ?? true;
+	return {
+		id,
+		name: "IRS Publications",
+		description: "Internal Revenue Service publications index and individual publication landing pages from irs.gov.",
+		async fetch(opts) {
+			const out = [];
+			const limit = opts.limit ?? Number.POSITIVE_INFINITY;
+			if (includeIndex && out.length < limit) out.push(await fetchIndex(id, opts));
+			for (const slug of options.publications ?? []) {
+				if (out.length >= limit) break;
+				out.push(await fetchPublication(id, slug, opts));
+			}
+			for (const path of options.revenueProcedures ?? []) {
+				if (out.length >= limit) break;
+				out.push(await fetchRevenueProcedure(id, path, opts));
+			}
+			return out;
+		}
+	};
+}
+async function fetchIndex(sourceId, opts) {
+	const response = await politeFetch(INDEX_URL, {
+		signal: opts.signal,
+		cacheDir: opts.cacheDir
+	});
+	const body = (response.body.match(/<table[\s\S]*?<\/table>/gi) ?? []).map((t) => htmlToText(t)).filter((t) => /Publication\s*\d+/i.test(t)).join("\n\n").slice(0, 2e5);
+	const verifiable = response.verifiable && body.length > 200;
+	return {
+		id: "index",
+		title: "IRS Publications Index",
+		body,
+		bodyHash: sha256(body),
+		provenance: {
+			url: INDEX_URL,
+			sourceUpdatedAt: response.sourceUpdatedAt,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: "US-FED",
+			verifiable,
+			unverifiableReason: response.unverifiableReason ?? (verifiable ? void 0 : "no publication rows extracted")
+		},
+		dimensionHints: IRS_DIMENSION_HINTS,
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache,
+			kind: "index"
+		}
+	};
+}
+async function fetchPublication(sourceId, slug, opts) {
+	const url = `${BASE_URL}/publications/${slug.replace(/^\/+/, "")}`;
+	const response = await politeFetch(url, {
+		signal: opts.signal,
+		cacheDir: opts.cacheDir
+	});
+	const title = extractTitle(response.body, `IRS Publication ${slug}`);
+	const body = extractMainContent(response.body);
+	const verifiable = response.verifiable && body.length > 200;
+	return {
+		id: `publication:${slug}`,
+		title,
+		body,
+		bodyHash: sha256(body),
+		provenance: {
+			url,
+			sourceUpdatedAt: extractRevisionDate(response.body) ?? response.sourceUpdatedAt,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: "US-FED",
+			verifiable,
+			unverifiableReason: response.unverifiableReason ?? (verifiable ? void 0 : "no publication body extracted")
+		},
+		dimensionHints: IRS_DIMENSION_HINTS,
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache,
+			kind: "publication",
+			slug
+		}
+	};
+}
+async function fetchRevenueProcedure(sourceId, path, opts) {
+	const url = `${BASE_URL}${path.startsWith("/") ? path : `/${path}`}`;
+	const response = await politeFetch(url, {
+		signal: opts.signal,
+		cacheDir: opts.cacheDir
+	});
+	const body = extractMainContent(response.body);
+	const verifiable = response.verifiable && body.length > 200;
+	return {
+		id: `rev-proc:${path}`,
+		title: extractTitle(response.body, `IRS Revenue Procedure ${path}`),
+		body,
+		bodyHash: sha256(body),
+		provenance: {
+			url,
+			sourceUpdatedAt: response.sourceUpdatedAt,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: "US-FED",
+			verifiable,
+			unverifiableReason: response.unverifiableReason ?? (verifiable ? void 0 : "no revenue-procedure body extracted")
+		},
+		dimensionHints: [...IRS_DIMENSION_HINTS, "procedural_currency"],
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache,
+			kind: "rev-proc",
+			path
+		}
+	};
+}
+function extractTitle(html, fallback) {
+	const og = /<meta\s+property=["']og:title["']\s+content=["']([^"']+)["']/i.exec(html)?.[1];
+	if (og) return decodeHtml(og);
+	const title = /<title>([\s\S]*?)<\/title>/i.exec(html)?.[1];
+	if (title) return htmlToText(title).split(" | ")[0] ?? fallback;
+	return fallback;
+}
+function extractMainContent(html) {
+	const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0];
+	if (main) return htmlToText(main.replace(/<nav[\s\S]*?<\/nav>/gi, "").replace(/<header[\s\S]*?<\/header>/gi, "").replace(/<footer[\s\S]*?<\/footer>/gi, "")).slice(0, 2e5);
+	const body = /<body\b[\s\S]*?<\/body>/i.exec(html)?.[0];
+	return body ? htmlToText(body).slice(0, 2e5) : htmlToText(html).slice(0, 2e5);
+}
+function extractRevisionDate(html) {
+	const m = /Publication\s+\S+\s*\((\d{4})\)/i.exec(html);
+	if (m?.[1]) {
+		const year = Number.parseInt(m[1], 10);
+		if (Number.isFinite(year) && year >= 2e3 && year <= (/* @__PURE__ */ new Date()).getUTCFullYear() + 1) return new Date(Date.UTC(year, 0, 1)).toISOString();
+	}
+}
+function decodeHtml(value) {
+	return htmlToText(value);
+}
+//#endregion
+//#region src/sources/state-sos.ts
+function createStateSosSource(config) {
+	const id = config.id ?? `state-sos:${config.state.toLowerCase()}`;
+	return {
+		id,
+		name: config.name ?? `${config.state} Secretary of State`,
+		description: `${config.state} Secretary of State filings and formation guidance pages.`,
+		async fetch(opts) {
+			const limit = opts.limit ?? config.entities.length;
+			const entities = config.entities.slice(0, limit);
+			const out = [];
+			for (const entity of entities) out.push(await fetchEntity(id, config, entity, opts));
+			return out;
+		}
+	};
+}
+async function fetchEntity(sourceId, config, entity, opts) {
+	const url = joinUrl(config.baseUrl, entity.path);
+	const response = await politeFetch(url, {
+		signal: opts.signal,
+		cacheDir: opts.cacheDir
+	});
+	const body = response.verifiable ? extractBySelector(response.body, entity.selector) : "";
+	const verifiable = response.verifiable && body.length > 100;
+	return {
+		id: entity.id,
+		title: entity.title,
+		body,
+		bodyHash: sha256(body),
+		provenance: {
+			url,
+			sourceUpdatedAt: response.sourceUpdatedAt,
+			fetchedAt: response.fetchedAt,
+			jurisdiction: `US-${config.state.toUpperCase()}`,
+			verifiable,
+			unverifiableReason: response.unverifiableReason ?? (verifiable ? void 0 : "extracted body too short")
+		},
+		dimensionHints: entity.dimensionHints ?? [
+			"jurisdictional_accuracy",
+			"corporate_formation",
+			"citation_hygiene"
+		],
+		metadata: {
+			sourceId,
+			status: response.status,
+			fromCache: response.fromCache,
+			state: config.state
+		}
+	};
+}
+function extractBySelector(html, selector) {
+	if (selector.kind === "whole") {
+		const main = /<main\b[\s\S]*?<\/main>/i.exec(html)?.[0];
+		return htmlToText(main ?? html).slice(0, 2e5);
+	}
+	if (selector.kind === "regex") {
+		const m = selector.value.exec(html)?.[0];
+		return m ? htmlToText(m).slice(0, 2e5) : "";
+	}
+	if (selector.kind === "id") {
+		const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+		const inner = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\sid=["']${escaped}["'][^>]*>([\\s\\S]*?)<\\/\\1>`, "i").exec(html)?.[2];
+		return inner ? htmlToText(inner).slice(0, 2e5) : "";
+	}
+	const escaped = selector.value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	const inner = new RegExp(`<([a-z][a-z0-9]*)\\b[^>]*\\sclass=["'][^"']*\\b${escaped}\\b[^"']*["'][^>]*>([\\s\\S]*?)<\\/\\1>`, "i").exec(html)?.[2];
+	return inner ? htmlToText(inner).slice(0, 2e5) : "";
+}
+function joinUrl(base, path) {
+	try {
+		return new URL(path, base.endsWith("/") ? base : `${base}/`).toString();
+	} catch {
+		return `${base.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
+	}
+}
+//#endregion
+export { IRS_DIMENSION_HINTS, MAX_RESPONSE_BYTES, MIN_REQUEST_GAP_MS, POLITE_USER_AGENT, __resetHttpThrottle, createCornellLiiSource, createIrsPublicationsSource, createStateSosSource, extractLinks, firstMatch, htmlToText, innerHtmlById, looksLikeBlockPage, politeFetch };
+
 //# sourceMappingURL=index.js.map