@agwab/pi-workflow 0.1.1 → 0.1.2

package/dist/workflow-web-source.js

@@ -0,0 +1,897 @@
+import { createHash } from "node:crypto";
+import { appendFile, mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
+import { isIP } from "node:net";
+import { dirname, resolve } from "node:path";
+export const WORKFLOW_WEB_SOURCE_CACHE_SCHEMA = "workflow-web-source-cache-v1";
+export const WORKFLOW_WEB_SOURCE_INDEX_SCHEMA = "workflow-web-source-index-v1";
+export const WORKFLOW_WEB_SOURCE_INDEX_EVENT_SCHEMA = "workflow-web-source-index-event-v1";
+export const WORKFLOW_WEB_SOURCE_EVENT_SCHEMA = "workflow-web-source-event-v1";
+export const WORKFLOW_WEB_SOURCE_TOOLS = [
+    "workflow_web_search",
+    "workflow_web_fetch_source",
+    "workflow_web_source_read",
+];
+export const DEFAULT_WORKFLOW_WEB_SOURCE_POLICY = {
+    previewChars: 800,
+    duplicatePreviewChars: 160,
+    sourceReadMaxChars: 1_200,
+    searchSnippetChars: 240,
+    perTaskVisibleCharBudget: 12_000,
+};
+export const DEFAULT_WORKFLOW_WEB_SECURITY_POLICY = {
+    allowPrivateHosts: false,
+    cacheRawProviderPayloads: false,
+};
+const SENSITIVE_QUERY_PARAM_PATTERN = /(^|[-_])(access[-_]?token|auth|code|credential|key|password|secret|session|signature|sig|token)([-_]|$)/i;
+const PRIVATE_HOST_PATTERNS = [
+    /^localhost$/i,
+    /^127\./,
+    /^0\./,
+    /^10\./,
+    /^192\.168\./,
+    /^169\.254\./,
+    /^metadata\.google\.internal$/i,
+];
+export function normalizeWorkflowWebSourcePolicy(policy) {
+    return {
+        ...DEFAULT_WORKFLOW_WEB_SOURCE_POLICY,
+        ...(policy ?? {}),
+    };
+}
+export function normalizeWorkflowWebSecurityPolicy(policy) {
+    return {
+        ...DEFAULT_WORKFLOW_WEB_SECURITY_POLICY,
+        ...(policy ?? {}),
+    };
+}
+export function isWorkflowWebSourceTool(tool) {
+    return WORKFLOW_WEB_SOURCE_TOOLS.includes(tool);
+}
+export function createWorkflowWebVisibleBudget(limit) {
+    return { limit: Math.max(0, Math.floor(limit)), used: 0 };
+}
+export function consumeWorkflowWebVisibleBudget(budget, text, maxChars) {
+    const remainingBefore = Math.max(0, budget.limit - budget.used);
+    const allowed = Math.max(0, Math.min(maxChars, remainingBefore));
+    const truncated = text.length > allowed;
+    const visible = text.slice(0, allowed);
+    budget.used += visible.length;
+    return {
+        text: visible,
+        truncated,
+        remaining: Math.max(0, budget.limit - budget.used),
+        used: budget.used,
+    };
+}
+export function validateWorkflowWebUrl(url, security = DEFAULT_WORKFLOW_WEB_SECURITY_POLICY) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return { ok: false, reason: "invalid_url" };
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+        return { ok: false, reason: "unsafe_scheme" };
+    }
+    const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, "");
+    if (!security.allowPrivateHosts && isPrivateHostname(host)) {
+        return { ok: false, reason: "private_host_blocked" };
+    }
+    return { ok: true, normalizedUrl: parsed.href, domain: host };
+}
+export function sanitizeUrlForModel(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return redactInlineSecrets(url);
+    }
+    return sanitizeParsedUrlForModel(parsed);
+}
+function sanitizeParsedUrlForModel(parsed) {
+    parsed.username = "";
+    parsed.password = "";
+    for (const key of [...parsed.searchParams.keys()]) {
+        if (SENSITIVE_QUERY_PARAM_PATTERN.test(key)) {
+            parsed.searchParams.set(key, "REDACTED");
+        }
+    }
+    parsed.hash = redactUrlFragment(parsed.hash);
+    return redactInlineSecretsNoUrls(parsed.href);
+}
+export function sourceRefFor(url, text) {
+    return `wsrc_${hashString(`${sourceUrlCacheKey(url)}\0${text}`).slice(0, 32)}`;
+}
+export function sourceUrlCacheKey(url) {
+    return `urlkey_${hashString(canonicalUrlForCache(url)).slice(0, 32)}`;
+}
+function sourceUrlDisplayCacheKey(url) {
+    let parsed;
+    try {
+        parsed = new URL(sanitizeUrlForModel(url));
+    }
+    catch {
+        return sanitizeUrlForModel(url).trim();
+    }
+    parsed.hash = shouldKeepFragmentForCache(parsed.hash) ? parsed.hash : "";
+    parsed.hostname = parsed.hostname.toLowerCase();
+    if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+        parsed.pathname = parsed.pathname.slice(0, -1);
+    }
+    const sortedParams = [...parsed.searchParams.entries()].sort(([left], [right]) => left.localeCompare(right));
+    parsed.search = "";
+    for (const [key, value] of sortedParams) {
+        parsed.searchParams.append(key, value);
+    }
+    return parsed.href;
+}
+function canonicalUrlForCache(url) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return url.trim();
+    }
+    parsed.hostname = parsed.hostname.toLowerCase();
+    parsed.hash = shouldKeepFragmentForCache(parsed.hash) ? parsed.hash : "";
+    if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+        parsed.pathname = parsed.pathname.slice(0, -1);
+    }
+    const sortedParams = [...parsed.searchParams.entries()].sort(([left], [right]) => left.localeCompare(right));
+    parsed.search = "";
+    for (const [key, value] of sortedParams) {
+        parsed.searchParams.append(key, value);
+    }
+    return parsed.href;
+}
+export function createWorkflowWebSource(options) {
+    const checked = validateWorkflowWebUrl(options.url, {
+        ...DEFAULT_WORKFLOW_WEB_SECURITY_POLICY,
+        allowPrivateHosts: true,
+    });
+    const domain = checked.ok ? checked.domain : "unknown";
+    const redactedUrl = sanitizeUrlForModel(options.url);
+    const contentHash = hashString(options.text);
+    return {
+        schema: WORKFLOW_WEB_SOURCE_CACHE_SCHEMA,
+        sourceRef: sourceRefFor(options.url, options.text),
+        createdAt: new Date().toISOString(),
+        runId: options.config.runId,
+        taskId: options.config.taskId,
+        url: redactedUrl,
+        redactedUrl,
+        urlKey: sourceUrlCacheKey(options.url),
+        domain,
+        ...(options.title ? { title: options.title } : {}),
+        ...(options.provider ? { provider: options.provider } : {}),
+        contentHash,
+        text: options.text,
+        textChars: options.text.length,
+        ...(options.extractionLossy !== undefined
+            ? { extractionLossy: options.extractionLossy }
+            : {}),
+        ...(options.metadata ? { metadata: options.metadata } : {}),
+    };
+}
+export async function writeWorkflowWebSource(config, source) {
+    await mkdir(resolve(config.cacheDir, "sources"), { recursive: true });
+    await writeJsonAtomic(sourceObjectPath(config, source.sourceRef), source);
+    const entry = sourceToIndexEntry(source);
+    await appendWorkflowWebSourceIndexEvent(config, entry);
+    const index = await readWorkflowWebSourceIndex(config);
+    const withoutExisting = index.sources.filter((indexEntry) => indexEntry.sourceRef !== source.sourceRef);
+    withoutExisting.push(entry);
+    await writeJsonAtomic(indexPath(config), {
+        ...index,
+        updatedAt: new Date().toISOString(),
+        sources: mergeSourceIndexEntries(withoutExisting),
+    });
+}
+export async function readWorkflowWebSource(config, sourceRef) {
+    if (!isWorkflowWebSourceRef(sourceRef))
+        return undefined;
+    try {
+        const parsed = JSON.parse(await readFile(sourceObjectPath(config, sourceRef), "utf8"));
+        if (!isRecord(parsed))
+            return undefined;
+        if (parsed.schema !== WORKFLOW_WEB_SOURCE_CACHE_SCHEMA)
+            return undefined;
+        if (parsed.sourceRef !== sourceRef)
+            return undefined;
+        if (typeof parsed.text !== "string")
+            return undefined;
+        return parsed;
+    }
+    catch {
+        return undefined;
+    }
+}
+export async function readWorkflowWebSourceIndex(config) {
+    const base = await readWorkflowWebSourceIndexFile(config);
+    const ledgerEntries = await readWorkflowWebSourceIndexLedger(config);
+    if (ledgerEntries.length === 0)
+        return base;
+    return {
+        ...base,
+        updatedAt: new Date().toISOString(),
+        sources: mergeSourceIndexEntries([...base.sources, ...ledgerEntries]),
+    };
+}
+export async function findWorkflowWebSourceByUrl(config, url) {
+    const redactedUrl = sanitizeUrlForModel(url);
+    const targetKey = sourceUrlCacheKey(url);
+    const targetDisplayKey = sourceUrlDisplayCacheKey(redactedUrl);
+    const index = await readWorkflowWebSourceIndex(config);
+    const existing = [...index.sources].reverse().find((entry) => {
+        return sourceIndexEntryMatchesUrl(entry, url, redactedUrl, targetKey, targetDisplayKey);
+    });
+    if (existing) {
+        const source = await readWorkflowWebSource(config, existing.sourceRef);
+        if (source)
+            return source;
+    }
+    return findWorkflowWebSourceByUrlFromSources(config, url, redactedUrl, targetKey, targetDisplayKey);
+}
+function sourceIndexEntryMatchesUrl(entry, url, redactedUrl, targetKey, targetDisplayKey) {
+    if (entry.urlKey)
+        return entry.urlKey === targetKey;
+    if (redactedUrlIdentityUnsafe(redactedUrl) || redactedUrlIdentityUnsafe(entry.redactedUrl) || redactedUrlIdentityUnsafe(entry.url)) {
+        return false;
+    }
+    return (entry.redactedUrl === redactedUrl ||
+        entry.url === url ||
+        sourceUrlDisplayCacheKey(entry.redactedUrl) === targetDisplayKey ||
+        sourceUrlDisplayCacheKey(entry.url) === targetDisplayKey);
+}
+function redactedUrlIdentityUnsafe(url) {
+    return /REDACTED/.test(url) || /[?&#][^=]*(?:token|secret|password|signature|sig|key|auth|session|credential)[^=]*=/i.test(url);
+}
+async function findWorkflowWebSourceByUrlFromSources(config, url, redactedUrl, targetKey, targetDisplayKey) {
+    let entries;
+    try {
+        entries = await readdir(resolve(config.cacheDir, "sources"));
+    }
+    catch {
+        return undefined;
+    }
+    for (const entry of entries.reverse()) {
+        if (!entry.endsWith(".json"))
+            continue;
+        const sourceRef = entry.slice(0, -".json".length);
+        const source = await readWorkflowWebSource(config, sourceRef);
+        if (!source)
+            continue;
+        if (source.urlKey) {
+            if (source.urlKey === targetKey)
+                return source;
+            continue;
+        }
+        if (redactedUrlIdentityUnsafe(redactedUrl) || redactedUrlIdentityUnsafe(source.redactedUrl) || redactedUrlIdentityUnsafe(source.url)) {
+            continue;
+        }
+        if (source.redactedUrl === redactedUrl ||
+            source.url === url ||
+            sourceUrlDisplayCacheKey(source.redactedUrl) === targetDisplayKey ||
+            sourceUrlDisplayCacheKey(source.url) === targetDisplayKey) {
+            return source;
+        }
+    }
+    return undefined;
+}
+export async function recordWorkflowWebSourceEvent(config, event, data = {}) {
+    await mkdir(resolve(config.cacheDir), { recursive: true });
+    await appendFile(resolve(config.cacheDir, "events.jsonl"), `${JSON.stringify({
+        schema: WORKFLOW_WEB_SOURCE_EVENT_SCHEMA,
+        at: new Date().toISOString(),
+        runId: config.runId,
+        taskId: config.taskId,
+        event,
+        ...redactRecordForModel(data),
+    })}\n`, "utf8");
+}
+export function buildWorkflowWebSourceCard(options) {
+    const previewLimit = options.duplicate
+        ? options.policy.duplicatePreviewChars
+        : options.policy.previewChars;
+    const preview = consumeWorkflowWebVisibleBudget(options.budget, redactInlineSecrets(options.source.text), previewLimit);
+    return {
+        sourceRef: options.source.sourceRef,
+        url: options.source.redactedUrl,
+        domain: options.source.domain,
+        ...(options.source.title ? { title: options.source.title } : {}),
+        preview: preview.text,
+        textChars: options.source.textChars,
+        fullContentCached: true,
+        duplicate: Boolean(options.duplicate),
+        budget: {
+            limit: options.budget.limit,
+            used: preview.used,
+            remaining: preview.remaining,
+            truncated: preview.truncated,
+        },
+        next: `Use workflow_web_source_read with sourceRef=${options.source.sourceRef} and an exact query for one quote, queries:[...] or reads:[...] to batch several quotes, or claim+terms when the exact quote is unknown. Do not read workflow cache files directly.`,
+    };
+}
+export function readWorkflowWebSourceSnippet(options) {
+    return readWorkflowWebSourceSnippets({
+        source: options.source,
+        requests: [
+            {
+                query: options.query,
+                claim: options.claim,
+                terms: options.terms,
+                maxChars: options.maxChars,
+            },
+        ],
+        maxChars: options.maxChars,
+        budget: options.budget,
+    })[0] ?? { status: "not_found", visibleChars: 0 };
+}
+export function readWorkflowWebSourceSnippets(options) {
+    let normalizedSource;
+    const getNormalizedSource = () => {
+        normalizedSource ??= normalizeForSearch(options.source.text);
+        return normalizedSource;
+    };
+    return options.requests.map((request) => readWorkflowWebSourceSnippetWithCache({
+        source: options.source,
+        request,
+        maxChars: request.maxChars ?? options.maxChars,
+        budget: options.budget,
+        getNormalizedSource,
+    }));
+}
+export function extractTextFromToolResult(result) {
+    if (!isRecord(result))
+        return "";
+    const content = result.content;
+    if (!Array.isArray(content))
+        return "";
+    return content
+        .map((entry) => {
+        if (!isRecord(entry))
+            return "";
+        const text = entry.text;
+        return typeof text === "string" ? text : "";
+    })
+        .filter(Boolean)
+        .join("\n\n");
+}
+export function extractTitleFromToolResult(result) {
+    if (!isRecord(result))
+        return undefined;
+    const details = result.details;
+    if (isRecord(details) && typeof details.title === "string")
+        return details.title;
+    const text = extractTextFromToolResult(result);
+    const heading = text.match(/^#\s+(.+)$/m)?.[1]?.trim();
+    return heading ? heading.slice(0, 200) : undefined;
+}
+export function extractSearchCandidates(result, policy = DEFAULT_WORKFLOW_WEB_SOURCE_POLICY) {
+    const text = extractTextFromToolResult(result);
+    if (!text.trim())
+        return [];
+    const urls = [...text.matchAll(/https?:\/\/[^\s)\]>"']+/g)].map((match) => match[0]);
+    if (urls.length === 0) {
+        return [
+            {
+                snippet: redactInlineSecrets(text.trim().slice(0, policy.searchSnippetChars)),
+            },
+        ];
+    }
+    return [...new Set(urls)].slice(0, 10).map((url) => {
+        const checked = validateWorkflowWebUrl(url, {
+            ...DEFAULT_WORKFLOW_WEB_SECURITY_POLICY,
+            allowPrivateHosts: true,
+        });
+        return {
+            url: sanitizeUrlForModel(url),
+            domain: checked.ok ? checked.domain : undefined,
+            snippet: redactInlineSecrets(nearbySnippet(text, url, policy.searchSnippetChars)),
+        };
+    });
+}
+export function toolResultFromJson(value) {
+    return {
+        content: [{ type: "text", text: `${JSON.stringify(value)}\n` }],
+        details: { workflowWebSource: true },
+    };
+}
+export function errorToolResult(code, message, extra = {}) {
+    return toolResultFromJson({ status: "blocked", code, message, ...extra });
+}
+function redactUrlFragment(hash) {
+    if (!hash)
+        return "";
+    const raw = hash.startsWith("#") ? hash.slice(1) : hash;
+    if (!raw)
+        return "";
+    try {
+        const params = new URLSearchParams(raw);
+        let changed = false;
+        for (const key of [...params.keys()]) {
+            if (SENSITIVE_QUERY_PARAM_PATTERN.test(key)) {
+                params.set(key, "REDACTED");
+                changed = true;
+            }
+        }
+        if (changed)
+            return `#${params.toString()}`;
+    }
+    catch {
+        // Fall through to inline redaction.
+    }
+    const redacted = redactInlineSecrets(raw);
+    return redacted ? `#${redacted}` : "";
+}
+function shouldKeepFragmentForCache(hash) {
+    if (!hash)
+        return false;
+    const raw = hash.startsWith("#") ? hash.slice(1) : hash;
+    return raw.startsWith("/") || raw.startsWith("!") || raw.includes("?");
+}
+function sourceToIndexEntry(source) {
+    return {
+        sourceRef: source.sourceRef,
+        createdAt: source.createdAt,
+        url: source.url,
+        redactedUrl: source.redactedUrl,
+        ...(source.urlKey ? { urlKey: source.urlKey } : {}),
+        domain: source.domain,
+        ...(source.title ? { title: source.title } : {}),
+        contentHash: source.contentHash,
+        textChars: source.textChars,
+        ...(source.provider ? { provider: source.provider } : {}),
+    };
+}
+function readWorkflowWebSourceSnippetWithCache(options) {
+    const query = options.request.query?.trim() ?? "";
+    if (query) {
+        const exactIndex = options.source.text.indexOf(query);
+        if (exactIndex >= 0) {
+            return snippetForMatch({
+                text: options.source.text,
+                start: exactIndex,
+                end: exactIndex + query.length,
+                matchType: "exact",
+                maxChars: options.maxChars,
+                budget: options.budget,
+            });
+        }
+        const sourceNorm = options.getNormalizedSource();
+        const queryNorm = normalizeForSearch(query);
+        const normalizedIndex = sourceNorm.normalized.indexOf(queryNorm.normalized);
+        if (normalizedIndex >= 0) {
+            const start = sourceNorm.map[normalizedIndex] ?? 0;
+            const endMapIndex = Math.min(sourceNorm.map.length - 1, normalizedIndex + Math.max(0, queryNorm.normalized.length - 1));
+            const end = (sourceNorm.map[endMapIndex] ?? start) + 1;
+            return snippetForMatch({
+                text: options.source.text,
+                start,
+                end,
+                matchType: "normalized",
+                maxChars: options.maxChars,
+                budget: options.budget,
+            });
+        }
+    }
+    const termNeedles = prepareTermNeedles(options.request.terms, options.request.claim);
+    if (termNeedles.length === 0)
+        return { status: "not_found", visibleChars: 0 };
+    return snippetForTerms({
+        text: options.source.text,
+        normalizedSource: options.getNormalizedSource(),
+        terms: termNeedles,
+        maxChars: options.maxChars,
+        budget: options.budget,
+    });
+}
+function snippetForTerms(options) {
+    const needles = options.terms
+        .map((term) => ({ raw: term, normalized: normalizeForSearch(term).normalized }))
+        .filter((term) => term.normalized.length > 0);
+    if (needles.length === 0)
+        return { status: "not_found", visibleChars: 0 };
+    const candidates = [];
+    for (const needle of needles) {
+        let fromIndex = 0;
+        let occurrenceCount = 0;
+        while (occurrenceCount < 20) {
+            const normalizedIndex = options.normalizedSource.normalized.indexOf(needle.normalized, fromIndex);
+            if (normalizedIndex < 0)
+                break;
+            const start = options.normalizedSource.map[normalizedIndex] ?? 0;
+            const endMapIndex = Math.min(options.normalizedSource.map.length - 1, normalizedIndex + Math.max(0, needle.normalized.length - 1));
+            const end = (options.normalizedSource.map[endMapIndex] ?? start) + 1;
+            candidates.push(scoreTermWindow(options.text, start, end, options.maxChars, needles));
+            fromIndex = normalizedIndex + Math.max(1, needle.normalized.length);
+            occurrenceCount += 1;
+        }
+    }
+    if (candidates.length === 0)
+        return { status: "not_found", visibleChars: 0 };
+    const best = candidates.sort((left, right) => {
+        if (right.score !== left.score)
+            return right.score - left.score;
+        return right.matchedTerms.length - left.matchedTerms.length;
+    })[0];
+    const raw = redactInlineSecrets(options.text.slice(best.start, best.end));
+    const consumed = consumeWorkflowWebVisibleBudget(options.budget, raw, options.maxChars);
+    return {
+        status: "matched",
+        matchType: "terms",
+        quote: consumed.text,
+        startOffset: best.start,
+        endOffset: best.end,
+        visibleChars: consumed.text.length,
+        matchedTerms: best.matchedTerms,
+        missingTerms: best.missingTerms,
+        coverageRatio: best.matchedTerms.length / Math.max(1, needles.length),
+        candidateOnly: true,
+    };
+}
+function scoreTermWindow(text, matchStart, matchEnd, maxChars, terms) {
+    const center = Math.floor((matchStart + matchEnd) / 2);
+    const start = Math.max(0, center - Math.floor(maxChars / 2));
+    const end = Math.min(text.length, start + maxChars);
+    const windowNorm = normalizeForSearch(text.slice(start, end)).normalized;
+    const matchedTerms = terms
+        .filter((term) => windowNorm.includes(term.normalized))
+        .map((term) => term.raw);
+    const missingTerms = terms
+        .filter((term) => !windowNorm.includes(term.normalized))
+        .map((term) => term.raw);
+    const occurrenceScore = terms.reduce((score, term) => {
+        return score + (windowNorm.includes(term.normalized) ? term.normalized.length : 0);
+    }, 0);
+    return {
+        start,
+        end,
+        matchedTerms,
+        missingTerms,
+        score: matchedTerms.length * 1_000 + occurrenceScore,
+    };
+}
+function prepareTermNeedles(terms, claim) {
+    const explicitTerms = dedupeStrings((terms ?? []).map((term) => term.trim()).filter(Boolean));
+    if (explicitTerms.length > 0)
+        return explicitTerms.slice(0, 16);
+    if (!claim?.trim())
+        return [];
+    return extractClaimTerms(claim).slice(0, 16);
+}
+function extractClaimTerms(claim) {
+    const tokens = claim
+        .match(/[\p{L}\p{N}][\p{L}\p{N}._/-]{2,}/gu)
+        ?.map((token) => token.toLowerCase()) ?? [];
+    const filtered = tokens.filter((token) => !SOURCE_READ_STOPWORDS.has(token));
+    return dedupeStrings(filtered).sort((left, right) => right.length - left.length);
+}
+function dedupeStrings(values) {
+    const seen = new Set();
+    const deduped = [];
+    for (const value of values) {
+        const key = normalizeForSearch(value).normalized;
+        if (!key || seen.has(key))
+            continue;
+        seen.add(key);
+        deduped.push(value);
+    }
+    return deduped;
+}
+const SOURCE_READ_STOPWORDS = new Set([
+    "about",
+    "across",
+    "after",
+    "against",
+    "also",
+    "because",
+    "before",
+    "between",
+    "claim",
+    "claims",
+    "could",
+    "does",
+    "from",
+    "have",
+    "into",
+    "more",
+    "must",
+    "only",
+    "other",
+    "over",
+    "should",
+    "source",
+    "sources",
+    "than",
+    "that",
+    "their",
+    "there",
+    "these",
+    "this",
+    "through",
+    "under",
+    "using",
+    "when",
+    "where",
+    "which",
+    "with",
+    "without",
+]);
+function snippetForMatch(options) {
+    const matchLength = Math.max(0, options.end - options.start);
+    const slack = Math.max(0, options.maxChars - matchLength);
+    const before = Math.floor(slack / 2);
+    const snippetStart = Math.max(0, options.start - before);
+    const snippetEnd = Math.min(options.text.length, snippetStart + options.maxChars);
+    const raw = redactInlineSecrets(options.text.slice(snippetStart, snippetEnd));
+    const consumed = consumeWorkflowWebVisibleBudget(options.budget, raw, options.maxChars);
+    return {
+        status: "matched",
+        matchType: options.matchType,
+        quote: consumed.text,
+        startOffset: options.start,
+        endOffset: options.end,
+        visibleChars: consumed.text.length,
+    };
+}
+function normalizeForSearch(text) {
+    let normalized = "";
+    const map = [];
+    let previousWhitespace = false;
+    for (let index = 0; index < text.length; index += 1) {
+        const raw = text[index];
+        let folded = raw.normalize("NFKC").toLowerCase();
+        folded = folded
+            .replace(/[\u2018\u2019\u201A\u201B]/g, "'")
+            .replace(/[\u201C\u201D\u201E\u201F]/g, '"')
+            .replace(/[\u2010-\u2015\u2212]/g, "-");
+        if (/\s/.test(folded)) {
+            if (!previousWhitespace) {
+                normalized += " ";
+                map.push(index);
+            }
+            previousWhitespace = true;
+            continue;
+        }
+        previousWhitespace = false;
+        for (const char of folded) {
+            normalized += char;
+            map.push(index);
+        }
+    }
+    return { normalized: normalized.trim(), map };
+}
+function nearbySnippet(text, needle, maxChars) {
+    const index = text.indexOf(needle);
+    if (index < 0)
+        return text.trim().slice(0, maxChars);
+    const start = Math.max(0, index - Math.floor(maxChars / 2));
+    return text.slice(start, start + maxChars).trim();
+}
+async function readWorkflowWebSourceIndexFile(config) {
+    try {
+        const parsed = JSON.parse(await readFile(indexPath(config), "utf8"));
+        if (!isRecord(parsed) || parsed.schema !== WORKFLOW_WEB_SOURCE_INDEX_SCHEMA) {
+            throw new Error("invalid index");
+        }
+        const sources = Array.isArray(parsed.sources)
+            ? parsed.sources.flatMap((entry) => {
+                const normalized = sourceIndexEntryFromUnknown(entry);
+                return normalized ? [normalized] : [];
+            })
+            : [];
+        return {
+            schema: WORKFLOW_WEB_SOURCE_INDEX_SCHEMA,
+            updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : new Date().toISOString(),
+            runId: typeof parsed.runId === "string" ? parsed.runId : config.runId,
+            sources: mergeSourceIndexEntries(sources),
+        };
+    }
+    catch {
+        return emptyWorkflowWebSourceIndex(config);
+    }
+}
+async function appendWorkflowWebSourceIndexEvent(config, entry) {
+    await mkdir(resolve(config.cacheDir), { recursive: true });
+    await appendFile(indexEventsPath(config), `${JSON.stringify({
+        schema: WORKFLOW_WEB_SOURCE_INDEX_EVENT_SCHEMA,
+        at: new Date().toISOString(),
+        runId: config.runId,
+        taskId: config.taskId,
+        entry,
+    })}\n`, "utf8");
+}
+async function readWorkflowWebSourceIndexLedger(config) {
+    let text;
+    try {
+        text = await readFile(indexEventsPath(config), "utf8");
+    }
+    catch {
+        return [];
+    }
+    const entries = [];
+    for (const line of text.split(/\r?\n/)) {
+        if (!line.trim())
+            continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (!isRecord(parsed) || parsed.schema !== WORKFLOW_WEB_SOURCE_INDEX_EVENT_SCHEMA)
+                continue;
+            const entry = sourceIndexEntryFromUnknown(parsed.entry);
+            if (entry)
+                entries.push(entry);
+        }
+        catch {
+            // Ignore torn or corrupt ledger lines; source file scan still provides a final fallback.
+        }
+    }
+    return entries;
+}
+function sourceIndexEntryFromUnknown(value) {
+    if (!isRecord(value))
+        return undefined;
+    if (typeof value.sourceRef !== "string" || !isWorkflowWebSourceRef(value.sourceRef))
+        return undefined;
+    if (typeof value.createdAt !== "string")
+        return undefined;
+    if (typeof value.url !== "string")
+        return undefined;
+    if (typeof value.redactedUrl !== "string")
+        return undefined;
+    if (typeof value.domain !== "string")
+        return undefined;
+    if (typeof value.contentHash !== "string")
+        return undefined;
+    if (!Number.isFinite(Number(value.textChars)))
+        return undefined;
+    return {
+        sourceRef: value.sourceRef,
+        createdAt: value.createdAt,
+        url: value.url,
+        redactedUrl: value.redactedUrl,
+        ...(typeof value.urlKey === "string" ? { urlKey: value.urlKey } : {}),
+        domain: value.domain,
+        ...(typeof value.title === "string" ? { title: value.title } : {}),
+        contentHash: value.contentHash,
+        textChars: Number(value.textChars),
+        ...(typeof value.provider === "string" ? { provider: value.provider } : {}),
+    };
+}
+function mergeSourceIndexEntries(entries) {
+    const bySourceRef = new Map();
+    for (const entry of entries)
+        bySourceRef.set(entry.sourceRef, entry);
+    return [...bySourceRef.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+}
+function emptyWorkflowWebSourceIndex(config) {
+    return {
+        schema: WORKFLOW_WEB_SOURCE_INDEX_SCHEMA,
+        updatedAt: new Date().toISOString(),
+        runId: config.runId,
+        sources: [],
+    };
+}
+function indexPath(config) {
+    return resolve(config.cacheDir, "index.json");
+}
+function indexEventsPath(config) {
+    return resolve(config.cacheDir, "index-events.jsonl");
+}
+function sourceObjectPath(config, sourceRef) {
+    if (!isWorkflowWebSourceRef(sourceRef)) {
+        throw new Error("invalid workflow web sourceRef");
+    }
+    const sourcesDir = resolve(config.cacheDir, "sources");
+    const path = resolve(sourcesDir, `${sourceRef}.json`);
+    if (!path.startsWith(`${sourcesDir}/`)) {
+        throw new Error("workflow web sourceRef escaped source cache");
+    }
+    return path;
+}
+function isWorkflowWebSourceRef(sourceRef) {
+    return /^wsrc_[a-f0-9]{32}$/.test(sourceRef);
+}
+async function writeJsonAtomic(path, value) {
+    await mkdir(dirname(path), { recursive: true });
+    const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
+    await writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+    await rename(tmp, path);
+}
+function hashString(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+function isPrivateHostname(host) {
+    if (PRIVATE_HOST_PATTERNS.some((pattern) => pattern.test(host)))
+        return true;
+    return nonPublicIpReason(host) !== undefined;
+}
+function nonPublicIpReason(address) {
+    const lower = address.toLowerCase().replace(/^\[|\]$/g, "");
+    const mappedIpv4 = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/)?.[1];
+    if (mappedIpv4)
+        return nonPublicIpReason(mappedIpv4);
+    const hexMapped = lower.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
+    if (hexMapped) {
+        const high = Number.parseInt(hexMapped[1], 16);
+        const low = Number.parseInt(hexMapped[2], 16);
+        return nonPublicIpReason(`${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`);
+    }
+    if (isIP(lower) === 4) {
+        const parts = lower.split(".").map((part) => Number(part));
+        if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
+            return "non_public_ip_blocked";
+        const [a, b, c, d] = parts;
+        if (a === 0 || a === 10 || a === 127 || a >= 224)
+            return "non_public_ip_blocked";
+        if (a === 100 && b >= 64 && b <= 127)
+            return "non_public_ip_blocked";
+        if (a === 169 && b === 254)
+            return "non_public_ip_blocked";
+        if (a === 172 && b >= 16 && b <= 31)
+            return "non_public_ip_blocked";
+        if (a === 192 && b === 168)
+            return "non_public_ip_blocked";
+        if (a === 192 && b === 0 && (c === 0 || c === 2))
+            return "non_public_ip_blocked";
+        if (a === 198 && (b === 18 || b === 19))
+            return "non_public_ip_blocked";
+        if (a === 198 && b === 51 && c === 100)
+            return "non_public_ip_blocked";
+        if (a === 203 && b === 0 && c === 113)
+            return "non_public_ip_blocked";
+        if (a === 255 && b === 255 && c === 255 && d === 255)
+            return "non_public_ip_blocked";
+    }
+    if (isIP(lower) === 6) {
+        if (lower === "::" || lower === "::1")
+            return "non_public_ip_blocked";
+        if (lower.startsWith("fc") || lower.startsWith("fd"))
+            return "non_public_ip_blocked";
+        if (lower.startsWith("fe80") || lower.startsWith("ff"))
+            return "non_public_ip_blocked";
+        if (lower.startsWith("2001:db8"))
+            return "non_public_ip_blocked";
+    }
+    return undefined;
+}
+function redactRecordForModel(value) {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, redactValueForModel(item)]));
+}
+function redactValueForModel(value) {
+    if (typeof value === "string")
+        return redactInlineSecrets(sanitizeUrlMaybe(value));
+    if (Array.isArray(value))
+        return value.map((item) => redactValueForModel(item));
+    if (!isRecord(value))
+        return value;
+    return redactRecordForModel(value);
+}
+function sanitizeUrlMaybe(value) {
+    return /^https?:\/\//i.test(value) ? sanitizeUrlForModel(value) : value;
+}
+function redactInlineSecrets(value) {
+    const withSanitizedUrls = value.replace(/https?:\/\/[^\s)\]}>"']+/gi, (match) => {
+        const trailing = match.match(/[.,;:!?]+$/)?.[0] ?? "";
+        const core = trailing ? match.slice(0, -trailing.length) : match;
+        try {
+            return `${sanitizeParsedUrlForModel(new URL(core))}${trailing}`;
+        }
+        catch {
+            return match;
+        }
+    });
+    return redactInlineSecretsNoUrls(withSanitizedUrls);
+}
+function redactInlineSecretsNoUrls(value) {
+    return value
+        .replace(/(authorization|cookie|set-cookie):\s*[^\n\r]+/gi, "$1: REDACTED")
+        .replace(/(token|secret|password|api[-_]?key)=([^\s&]+)/gi, "$1=REDACTED")
+        .replace(/\/Users\/[^\s:'")]+/g, "/Users/REDACTED");
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}