@m4ike1/ion-coding-agent 0.1.0

package/dist/core/webfetch/robots.js

@@ -0,0 +1,153 @@
+/**
+ * robots.txt parsing for the polite crawler.
+ *
+ * Follows RFC 9309 where it matters: the group whose user-agent matches ours
+ * (or `*` when none does) supplies the rules, the longest matching pattern wins,
+ * and an Allow beats a Disallow of the same length. A missing or unreadable
+ * robots.txt means "allowed", which is what the RFC requires and what a crawler
+ * must assume when the file is absent.
+ *
+ * The file is fetched through the shared guardrails, so a hostile robots.txt
+ * cannot redirect the crawler somewhere it was already barred from going.
+ */
+import { assertBytesWithinBudget, guardedRequest, WEBFETCH_MAX_BYTES, WEBFETCH_REQUEST_TIMEOUT_MS, } from "./guardrails.js";
+export const ROBOTS_USER_AGENT = "ion";
+function patternToRegex(pattern) {
+    if (pattern.length === 0)
+        return undefined;
+    const anchored = pattern.endsWith("$");
+    const body = anchored ? pattern.slice(0, -1) : pattern;
+    let source = "";
+    for (const character of body) {
+        if (character === "*") {
+            source += ".*";
+        }
+        else {
+            source += character.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+        }
+    }
+    return { regex: new RegExp(`^${source}${anchored ? "$" : ""}`), anchored };
+}
+/** Parse a robots.txt body into a policy for {@link ROBOTS_USER_AGENT}. */
+export function parseRobotsTxt(text, userAgent = ROBOTS_USER_AGENT) {
+    const agent = userAgent.toLowerCase();
+    const groups = [];
+    let current;
+    let expectingAgents = false;
+    for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, "").trim();
+        if (line.length === 0)
+            continue;
+        const separator = line.indexOf(":");
+        if (separator === -1)
+            continue;
+        const field = line.slice(0, separator).trim().toLowerCase();
+        const value = line.slice(separator + 1).trim();
+        if (field === "user-agent") {
+            if (!current || !expectingAgents) {
+                current = { agents: [], lines: [] };
+                groups.push(current);
+                expectingAgents = true;
+            }
+            current.agents.push(value.toLowerCase());
+            continue;
+        }
+        // Any non-user-agent field ends the agent block, per RFC 9309.
+        expectingAgents = false;
+        if (!current)
+            continue;
+        current.lines.push(`${field}:${value}`);
+    }
+    const matches = (group) => group.agents.some((candidate) => candidate !== "*" && agent.includes(candidate));
+    const specific = groups.find(matches);
+    const wildcard = groups.find((group) => group.agents.includes("*"));
+    const selected = specific ?? wildcard;
+    const policy = { rules: [], sitemaps: [] };
+    if (!selected)
+        return policy;
+    for (const entry of selected.lines) {
+        const separator = entry.indexOf(":");
+        const field = entry.slice(0, separator);
+        const value = entry.slice(separator + 1);
+        if (field === "sitemap") {
+            if (value.length > 0)
+                policy.sitemaps.push(value);
+            continue;
+        }
+        if (field === "crawl-delay") {
+            const seconds = Number.parseFloat(value);
+            if (Number.isFinite(seconds) && seconds >= 0)
+                policy.crawlDelaySeconds = seconds;
+            continue;
+        }
+        if (field !== "allow" && field !== "disallow")
+            continue;
+        const compiled = patternToRegex(value);
+        // `Disallow:` with an empty value means allow everything, so it adds no rule.
+        if (!compiled)
+            continue;
+        policy.rules.push({
+            allow: field === "allow",
+            pattern: value,
+            regex: compiled.regex,
+            length: value.length,
+        });
+    }
+    return policy;
+}
+/** Longest matching pattern wins; an Allow wins a tie. No rules means allowed. */
+export function isPathAllowed(policy, pathWithQuery) {
+    let best;
+    for (const rule of policy.rules) {
+        if (!rule.regex.test(pathWithQuery))
+            continue;
+        if (!best || rule.length > best.length || (rule.length === best.length && rule.allow && !best.allow)) {
+            best = rule;
+        }
+    }
+    return best ? best.allow : true;
+}
+export function isUrlAllowed(policy, url) {
+    return isPathAllowed(policy, `${url.pathname}${url.search}`);
+}
+/**
+ * Fetch and parse a site's robots.txt. Returns an allow-everything policy when
+ * the file is missing, unreadable, or not text: a crawler must not refuse to
+ * read a site merely because the site failed to describe its own rules, and it
+ * must not guess rules it was not given.
+ */
+export async function fetchRobotsPolicy(originUrl, options = {}) {
+    const allowAll = { rules: [], sitemaps: [] };
+    const robotsUrl = new URL("/robots.txt", originUrl.origin);
+    try {
+        const response = await guardedRequest(robotsUrl.href, {
+            headers: { "User-Agent": options.userAgent ?? ROBOTS_USER_AGENT },
+            signal: options.signal,
+            lookup: options.lookup,
+            request: options.request,
+            maxBytes: options.maxBytes ?? WEBFETCH_MAX_BYTES,
+            timeoutMs: WEBFETCH_REQUEST_TIMEOUT_MS,
+        });
+        // 4xx means no published rules; only a served body is authoritative.
+        if (response.statusCode >= 400)
+            return allowAll;
+        const chunks = [];
+        let totalBytes = 0;
+        for await (const chunk of response.body) {
+            totalBytes += chunk.length;
+            assertBytesWithinBudget(totalBytes, options.maxBytes ?? WEBFETCH_MAX_BYTES);
+            chunks.push(chunk);
+        }
+        const merged = new Uint8Array(totalBytes);
+        let offset = 0;
+        for (const chunk of chunks) {
+            merged.set(chunk, offset);
+            offset += chunk.length;
+        }
+        return parseRobotsTxt(new TextDecoder().decode(merged), options.userAgent ?? ROBOTS_USER_AGENT);
+    }
+    catch {
+        return allowAll;
+    }
+}
+//# sourceMappingURL=robots.js.map

package/dist/core/webfetch/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../../src/core/webfetch/robots.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EACN,uBAAuB,EAEvB,cAAc,EAEd,kBAAkB,EAClB,2BAA2B,GAC3B,MAAM,iBAAiB,CAAC;AAEzB,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAgBvC,SAAS,cAAc,CAAC,OAAe,EAAoD;IAC1F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACvD,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,IAAI,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,SAAS,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;QAC3D,CAAC;IACF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,IAAI,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;AAAA,CAC3E;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,SAAS,GAAW,iBAAiB,EAAgB;IACjG,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,MAAM,GAA4C,EAAE,CAAC;IAC3D,IAAI,OAA0D,CAAC;IAC/D,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,SAAS;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE/C,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC;gBAClC,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,eAAe,GAAG,IAAI,CAAC;YACxB,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,SAAS;QACV,CAAC;QAED,+DAA+D;QAC/D,eAAe,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,KAA2B,EAAE,EAAE,CAC/C,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IAClF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,QAAQ,IAAI,QAAQ,CAAC;IAEtC,MAAM,MAAM,GAAiB,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACzD,IAAI,CAAC,QAAQ;QAAE,OAAO,MAAM,CAAC;IAE7B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACzC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClD,SAAS;QACV,CAAC;QACD,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;gBAAE,MAAM,CAAC,iBAAiB,GAAG,OAAO,CAAC;YACjF,SAAS;QACV,CAAC;QACD,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU;YAAE,SAAS;QACxD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACvC,8EAA8E;QAC9E,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YACjB,KAAK,EAAE,KAAK,KAAK,OAAO;YACxB,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAAA,CACd;AAED,kFAAkF;AAClF,MAAM,UAAU,aAAa,CAAC,MAAoB,EAAE,aAAqB,EAAW;IACnF,IAAI,IAA4B,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC;YAAE,SAAS;QAC9C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACtG,IAAI,GAAG,IAAI,CAAC;QACb,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,YAAY,CAAC,MAAoB,EAAE,GAAQ,EAAW;IACrE,OAAO,aAAa,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAAA,CAC7D;AAUD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,SAAc,EAAE,OAAO,GAAuB,EAAE,EAAyB;IAChH,MAAM,QAAQ,GAAiB,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC3D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE;YACrD,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,iBAAiB,EAAE;YACjE,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,kBAAkB;YAChD,SAAS,EAAE,2BAA2B;SACtC,CAAC,CAAC;QACH,qEAAqE;QACrE,IAAI,QAAQ,CAAC,UAAU,IAAI,GAAG;YAAE,OAAO,QAAQ,CAAC;QAChD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,uBAAuB,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,IAAI,kBAAkB,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACxB,CAAC;QACD,OAAO,cAAc,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,iBAAiB,CAAC,CAAC;IACjG,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,QAAQ,CAAC;IACjB,CAAC;AAAA,CACD","sourcesContent":["/**\n * robots.txt parsing for the polite crawler.\n *\n * Follows RFC 9309 where it matters: the group whose user-agent matches ours\n * (or `*` when none does) supplies the rules, the longest matching pattern wins,\n * and an Allow beats a Disallow of the same length. A missing or unreadable\n * robots.txt means \"allowed\", which is what the RFC requires and what a crawler\n * must assume when the file is absent.\n *\n * The file is fetched through the shared guardrails, so a hostile robots.txt\n * cannot redirect the crawler somewhere it was already barred from going.\n */\n\nimport {\n\tassertBytesWithinBudget,\n\ttype DnsLookup,\n\tguardedRequest,\n\ttype LowLevelRequestFn,\n\tWEBFETCH_MAX_BYTES,\n\tWEBFETCH_REQUEST_TIMEOUT_MS,\n} from \"./guardrails.ts\";\n\nexport const ROBOTS_USER_AGENT = \"ion\";\n\nexport interface RobotsRule {\n\tallow: boolean;\n\tpattern: string;\n\tregex: RegExp;\n\tlength: number;\n}\n\nexport interface RobotsPolicy {\n\trules: RobotsRule[];\n\t/** Seconds requested by Crawl-delay, when the file declares one. */\n\tcrawlDelaySeconds?: number;\n\tsitemaps: string[];\n}\n\nfunction patternToRegex(pattern: string): { regex: RegExp; anchored: boolean } | undefined {\n\tif (pattern.length === 0) return undefined;\n\tconst anchored = pattern.endsWith(\"$\");\n\tconst body = anchored ? pattern.slice(0, -1) : pattern;\n\tlet source = \"\";\n\tfor (const character of body) {\n\t\tif (character === \"*\") {\n\t\t\tsource += \".*\";\n\t\t} else {\n\t\t\tsource += character.replace(/[.+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n\t\t}\n\t}\n\treturn { regex: new RegExp(`^${source}${anchored ? \"$\" : \"\"}`), anchored };\n}\n\n/** Parse a robots.txt body into a policy for {@link ROBOTS_USER_AGENT}. */\nexport function parseRobotsTxt(text: string, userAgent: string = ROBOTS_USER_AGENT): RobotsPolicy {\n\tconst agent = userAgent.toLowerCase();\n\tconst groups: { agents: string[]; lines: string[] }[] = [];\n\tlet current: { agents: string[]; lines: string[] } | undefined;\n\tlet expectingAgents = false;\n\n\tfor (const rawLine of text.split(/\\r?\\n/)) {\n\t\tconst line = rawLine.replace(/#.*$/, \"\").trim();\n\t\tif (line.length === 0) continue;\n\t\tconst separator = line.indexOf(\":\");\n\t\tif (separator === -1) continue;\n\t\tconst field = line.slice(0, separator).trim().toLowerCase();\n\t\tconst value = line.slice(separator + 1).trim();\n\n\t\tif (field === \"user-agent\") {\n\t\t\tif (!current || !expectingAgents) {\n\t\t\t\tcurrent = { agents: [], lines: [] };\n\t\t\t\tgroups.push(current);\n\t\t\t\texpectingAgents = true;\n\t\t\t}\n\t\t\tcurrent.agents.push(value.toLowerCase());\n\t\t\tcontinue;\n\t\t}\n\n\t\t// Any non-user-agent field ends the agent block, per RFC 9309.\n\t\texpectingAgents = false;\n\t\tif (!current) continue;\n\t\tcurrent.lines.push(`${field}:${value}`);\n\t}\n\n\tconst matches = (group: { agents: string[] }) =>\n\t\tgroup.agents.some((candidate) => candidate !== \"*\" && agent.includes(candidate));\n\tconst specific = groups.find(matches);\n\tconst wildcard = groups.find((group) => group.agents.includes(\"*\"));\n\tconst selected = specific ?? wildcard;\n\n\tconst policy: RobotsPolicy = { rules: [], sitemaps: [] };\n\tif (!selected) return policy;\n\n\tfor (const entry of selected.lines) {\n\t\tconst separator = entry.indexOf(\":\");\n\t\tconst field = entry.slice(0, separator);\n\t\tconst value = entry.slice(separator + 1);\n\t\tif (field === \"sitemap\") {\n\t\t\tif (value.length > 0) policy.sitemaps.push(value);\n\t\t\tcontinue;\n\t\t}\n\t\tif (field === \"crawl-delay\") {\n\t\t\tconst seconds = Number.parseFloat(value);\n\t\t\tif (Number.isFinite(seconds) && seconds >= 0) policy.crawlDelaySeconds = seconds;\n\t\t\tcontinue;\n\t\t}\n\t\tif (field !== \"allow\" && field !== \"disallow\") continue;\n\t\tconst compiled = patternToRegex(value);\n\t\t// `Disallow:` with an empty value means allow everything, so it adds no rule.\n\t\tif (!compiled) continue;\n\t\tpolicy.rules.push({\n\t\t\tallow: field === \"allow\",\n\t\t\tpattern: value,\n\t\t\tregex: compiled.regex,\n\t\t\tlength: value.length,\n\t\t});\n\t}\n\treturn policy;\n}\n\n/** Longest matching pattern wins; an Allow wins a tie. No rules means allowed. */\nexport function isPathAllowed(policy: RobotsPolicy, pathWithQuery: string): boolean {\n\tlet best: RobotsRule | undefined;\n\tfor (const rule of policy.rules) {\n\t\tif (!rule.regex.test(pathWithQuery)) continue;\n\t\tif (!best || rule.length > best.length || (rule.length === best.length && rule.allow && !best.allow)) {\n\t\t\tbest = rule;\n\t\t}\n\t}\n\treturn best ? best.allow : true;\n}\n\nexport function isUrlAllowed(policy: RobotsPolicy, url: URL): boolean {\n\treturn isPathAllowed(policy, `${url.pathname}${url.search}`);\n}\n\nexport interface RobotsFetchOptions {\n\tuserAgent?: string;\n\tsignal?: AbortSignal;\n\tlookup?: DnsLookup;\n\trequest?: LowLevelRequestFn;\n\tmaxBytes?: number;\n}\n\n/**\n * Fetch and parse a site's robots.txt. Returns an allow-everything policy when\n * the file is missing, unreadable, or not text: a crawler must not refuse to\n * read a site merely because the site failed to describe its own rules, and it\n * must not guess rules it was not given.\n */\nexport async function fetchRobotsPolicy(originUrl: URL, options: RobotsFetchOptions = {}): Promise<RobotsPolicy> {\n\tconst allowAll: RobotsPolicy = { rules: [], sitemaps: [] };\n\tconst robotsUrl = new URL(\"/robots.txt\", originUrl.origin);\n\ttry {\n\t\tconst response = await guardedRequest(robotsUrl.href, {\n\t\t\theaders: { \"User-Agent\": options.userAgent ?? ROBOTS_USER_AGENT },\n\t\t\tsignal: options.signal,\n\t\t\tlookup: options.lookup,\n\t\t\trequest: options.request,\n\t\t\tmaxBytes: options.maxBytes ?? WEBFETCH_MAX_BYTES,\n\t\t\ttimeoutMs: WEBFETCH_REQUEST_TIMEOUT_MS,\n\t\t});\n\t\t// 4xx means no published rules; only a served body is authoritative.\n\t\tif (response.statusCode >= 400) return allowAll;\n\t\tconst chunks: Uint8Array[] = [];\n\t\tlet totalBytes = 0;\n\t\tfor await (const chunk of response.body) {\n\t\t\ttotalBytes += chunk.length;\n\t\t\tassertBytesWithinBudget(totalBytes, options.maxBytes ?? WEBFETCH_MAX_BYTES);\n\t\t\tchunks.push(chunk);\n\t\t}\n\t\tconst merged = new Uint8Array(totalBytes);\n\t\tlet offset = 0;\n\t\tfor (const chunk of chunks) {\n\t\t\tmerged.set(chunk, offset);\n\t\t\toffset += chunk.length;\n\t\t}\n\t\treturn parseRobotsTxt(new TextDecoder().decode(merged), options.userAgent ?? ROBOTS_USER_AGENT);\n\t} catch {\n\t\treturn allowAll;\n\t}\n}\n"]}

package/dist/core/webfetch/search-intent.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Search intent: the harness-side description of what a search should return.
+ *
+ * One shape serves every backend, so `webfetch` can ask for images or news
+ * without caring which provider answers. Cardinality belongs to the modality
+ * (`results: {web: 5, images: 3}`) because a flat `sources + limit` is
+ * ambiguous: 5 total, or 5 per source?
+ *
+ * Only fields a backend can actually honor live here. Deliberately absent until
+ * something can serve them: per-source purpose labels, query-rewrite toggles,
+ * and search budgets/planning. `strategy` is reserved for those.
+ *
+ * Tag sugar (`[images=3] [news=5] [recent=7d] [domains=openai.com] query`)
+ * compiles into this type in the tool layer; backends never see tags.
+ */
+/** Per-modality result counts. Absent means "do not request this source". */
+export interface SearchResultCounts {
+    web?: number;
+    images?: number;
+    news?: number;
+}
+export interface SearchFilters {
+    /** Hostnames or domain prefixes to restrict results to. */
+    domains?: string[];
+    /** Maximum age of a result, e.g. `7d`, `24h`, `1w`, `1m`. Provider-mapped. */
+    recency?: string;
+    /** Two-letter language/country hint where the backend supports one. */
+    language?: string;
+}
+export interface SearchIntent {
+    query: string;
+    results?: SearchResultCounts;
+    filters?: SearchFilters;
+    /** Reserved for v2 search planning (depth budgets, fan-out, dedup). */
+    strategy?: never;
+}
+export declare const SEARCH_MAX_PER_SOURCE = 100;
+/** Default per-source count when the caller gives no `results` block. */
+export declare const SEARCH_DEFAULT_WEB_RESULTS = 5;
+export declare const SEARCH_DEFAULT_WINDOW_MS = 30000;
+export declare class SearchIntentError extends Error {
+    constructor(message: string);
+}
+/** Normalized intent: counts are clamped and unknown keys dropped. `web` is
+ * present unless the caller named other sources explicitly. */
+export interface ResolvedSearchIntent {
+    query: string;
+    results: SearchResultCounts;
+    filters: {
+        domains?: string[];
+        recencyMs?: number;
+        language?: string;
+    };
+}
+export declare function resolveSearchIntent(intent: SearchIntent): ResolvedSearchIntent;
+export declare function parseRecency(value: string): number;
+/** Total results requested across every modality, for budget reporting. */
+export declare function totalRequestedResults(intent: ResolvedSearchIntent): number;
+//# sourceMappingURL=search-intent.d.ts.map

package/dist/core/webfetch/search-intent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-intent.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/search-intent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,6EAA6E;AAC7E,MAAM,WAAW,kBAAkB;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,aAAa;IAC7B,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,8EAA8E;IAC9E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,kBAAkB,CAAC;IAC7B,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,KAAK,CAAC;CACjB;AAED,eAAO,MAAM,qBAAqB,MAA+B,CAAC;AAClE,yEAAyE;AACzE,eAAO,MAAM,0BAA0B,IAAI,CAAC;AAC5C,eAAO,MAAM,wBAAwB,QAA8B,CAAC;AAYpE,qBAAa,iBAAkB,SAAQ,KAAK;IAC3C,YAAY,OAAO,EAAE,MAAM,EAG1B;CACD;AAUD;+DAC+D;AAC/D,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,kBAAkB,CAAC;IAC5B,OAAO,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACvE;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,oBAAoB,CA2B9E;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAUlD;AAED,2EAA2E;AAC3E,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,oBAAoB,GAAG,MAAM,CAE1E","sourcesContent":["/**\n * Search intent: the harness-side description of what a search should return.\n *\n * One shape serves every backend, so `webfetch` can ask for images or news\n * without caring which provider answers. Cardinality belongs to the modality\n * (`results: {web: 5, images: 3}`) because a flat `sources + limit` is\n * ambiguous: 5 total, or 5 per source?\n *\n * Only fields a backend can actually honor live here. Deliberately absent until\n * something can serve them: per-source purpose labels, query-rewrite toggles,\n * and search budgets/planning. `strategy` is reserved for those.\n *\n * Tag sugar (`[images=3] [news=5] [recent=7d] [domains=openai.com] query`)\n * compiles into this type in the tool layer; backends never see tags.\n */\n\nimport { WEBFETCH_FIRECRAWL_MAX_LIMIT, WEBFETCH_REQUEST_TIMEOUT_MS } from \"./guardrails.ts\";\n\n/** Per-modality result counts. Absent means \"do not request this source\". */\nexport interface SearchResultCounts {\n\tweb?: number;\n\timages?: number;\n\tnews?: number;\n}\n\nexport interface SearchFilters {\n\t/** Hostnames or domain prefixes to restrict results to. */\n\tdomains?: string[];\n\t/** Maximum age of a result, e.g. `7d`, `24h`, `1w`, `1m`. Provider-mapped. */\n\trecency?: string;\n\t/** Two-letter language/country hint where the backend supports one. */\n\tlanguage?: string;\n}\n\nexport interface SearchIntent {\n\tquery: string;\n\tresults?: SearchResultCounts;\n\tfilters?: SearchFilters;\n\t/** Reserved for v2 search planning (depth budgets, fan-out, dedup). */\n\tstrategy?: never;\n}\n\nexport const SEARCH_MAX_PER_SOURCE = WEBFETCH_FIRECRAWL_MAX_LIMIT;\n/** Default per-source count when the caller gives no `results` block. */\nexport const SEARCH_DEFAULT_WEB_RESULTS = 5;\nexport const SEARCH_DEFAULT_WINDOW_MS = WEBFETCH_REQUEST_TIMEOUT_MS;\n\nconst DURATION_PATTERN = /^(\\d+)\\s*(h|d|w|m|y)$/;\n\nconst DURATION_MS: Record<string, number> = {\n\th: 60 * 60 * 1000,\n\td: 24 * 60 * 60 * 1000,\n\tw: 7 * 24 * 60 * 60 * 1000,\n\tm: 30 * 24 * 60 * 60 * 1000,\n\ty: 365 * 24 * 60 * 60 * 1000,\n};\n\nexport class SearchIntentError extends Error {\n\tconstructor(message: string) {\n\t\tsuper(message);\n\t\tthis.name = \"SearchIntentError\";\n\t}\n}\n\n/** Clamp one count into 1..SEARCH_MAX_PER_SOURCE, rejecting non-integers. */\nfunction clampCount(value: number, source: string): number {\n\tif (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) {\n\t\tthrow new SearchIntentError(`results.${source} must be a positive integer`);\n\t}\n\treturn Math.min(value, SEARCH_MAX_PER_SOURCE);\n}\n\n/** Normalized intent: counts are clamped and unknown keys dropped. `web` is\n * present unless the caller named other sources explicitly. */\nexport interface ResolvedSearchIntent {\n\tquery: string;\n\tresults: SearchResultCounts;\n\tfilters: { domains?: string[]; recencyMs?: number; language?: string };\n}\n\nexport function resolveSearchIntent(intent: SearchIntent): ResolvedSearchIntent {\n\t// A named source replaces the default web-only set rather than adding to it.\n\tconst query = intent.query.trim();\n\tif (!query) {\n\t\tthrow new SearchIntentError(\"Search query must not be empty\");\n\t}\n\tconst requested = intent.results ?? {};\n\t// The web default applies only when no source was requested at all. Once the\n\t// caller names a source (say `[images=3]` or `{images: 3}`), the requested set\n\t// is exactly what they asked for: asking for images must not also return pages.\n\tconst namesSources = Object.keys(requested).length > 0;\n\tconst results: ResolvedSearchIntent[\"results\"] = {};\n\tif (!namesSources || requested.web !== undefined) {\n\t\tresults.web = requested.web === undefined ? SEARCH_DEFAULT_WEB_RESULTS : clampCount(requested.web, \"web\");\n\t}\n\tif (requested.images !== undefined) results.images = clampCount(requested.images, \"images\");\n\tif (requested.news !== undefined) results.news = clampCount(requested.news, \"news\");\n\n\tconst filters: ResolvedSearchIntent[\"filters\"] = {};\n\tconst domains = intent.filters?.domains\n\t\t?.map((domain) => domain.trim().toLowerCase())\n\t\t.filter((domain) => domain.length > 0);\n\tif (domains && domains.length > 0) filters.domains = [...new Set(domains)];\n\tconst language = intent.filters?.language?.trim();\n\tif (language) filters.language = language;\n\tif (intent.filters?.recency !== undefined) filters.recencyMs = parseRecency(intent.filters.recency);\n\treturn { query, results, filters };\n}\n\nexport function parseRecency(value: string): number {\n\tconst match = DURATION_PATTERN.exec(value.trim().toLowerCase());\n\tif (!match) {\n\t\tthrow new SearchIntentError(`filters.recency must look like 24h, 7d, 2w, 6m, or 1y (got \"${value}\")`);\n\t}\n\tconst amount = Number.parseInt(match[1], 10);\n\tif (!Number.isFinite(amount) || amount < 1) {\n\t\tthrow new SearchIntentError(`filters.recency must be a positive duration (got \"${value}\")`);\n\t}\n\treturn amount * DURATION_MS[match[2]];\n}\n\n/** Total results requested across every modality, for budget reporting. */\nexport function totalRequestedResults(intent: ResolvedSearchIntent): number {\n\treturn Object.values(intent.results).reduce<number>((sum, count) => sum + (count ?? 0), 0);\n}\n"]}

package/dist/core/webfetch/search-intent.js

@@ -0,0 +1,89 @@
+/**
+ * Search intent: the harness-side description of what a search should return.
+ *
+ * One shape serves every backend, so `webfetch` can ask for images or news
+ * without caring which provider answers. Cardinality belongs to the modality
+ * (`results: {web: 5, images: 3}`) because a flat `sources + limit` is
+ * ambiguous: 5 total, or 5 per source?
+ *
+ * Only fields a backend can actually honor live here. Deliberately absent until
+ * something can serve them: per-source purpose labels, query-rewrite toggles,
+ * and search budgets/planning. `strategy` is reserved for those.
+ *
+ * Tag sugar (`[images=3] [news=5] [recent=7d] [domains=openai.com] query`)
+ * compiles into this type in the tool layer; backends never see tags.
+ */
+import { WEBFETCH_FIRECRAWL_MAX_LIMIT, WEBFETCH_REQUEST_TIMEOUT_MS } from "./guardrails.js";
+export const SEARCH_MAX_PER_SOURCE = WEBFETCH_FIRECRAWL_MAX_LIMIT;
+/** Default per-source count when the caller gives no `results` block. */
+export const SEARCH_DEFAULT_WEB_RESULTS = 5;
+export const SEARCH_DEFAULT_WINDOW_MS = WEBFETCH_REQUEST_TIMEOUT_MS;
+const DURATION_PATTERN = /^(\d+)\s*(h|d|w|m|y)$/;
+const DURATION_MS = {
+    h: 60 * 60 * 1000,
+    d: 24 * 60 * 60 * 1000,
+    w: 7 * 24 * 60 * 60 * 1000,
+    m: 30 * 24 * 60 * 60 * 1000,
+    y: 365 * 24 * 60 * 60 * 1000,
+};
+export class SearchIntentError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "SearchIntentError";
+    }
+}
+/** Clamp one count into 1..SEARCH_MAX_PER_SOURCE, rejecting non-integers. */
+function clampCount(value, source) {
+    if (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) {
+        throw new SearchIntentError(`results.${source} must be a positive integer`);
+    }
+    return Math.min(value, SEARCH_MAX_PER_SOURCE);
+}
+export function resolveSearchIntent(intent) {
+    // A named source replaces the default web-only set rather than adding to it.
+    const query = intent.query.trim();
+    if (!query) {
+        throw new SearchIntentError("Search query must not be empty");
+    }
+    const requested = intent.results ?? {};
+    // The web default applies only when no source was requested at all. Once the
+    // caller names a source (say `[images=3]` or `{images: 3}`), the requested set
+    // is exactly what they asked for: asking for images must not also return pages.
+    const namesSources = Object.keys(requested).length > 0;
+    const results = {};
+    if (!namesSources || requested.web !== undefined) {
+        results.web = requested.web === undefined ? SEARCH_DEFAULT_WEB_RESULTS : clampCount(requested.web, "web");
+    }
+    if (requested.images !== undefined)
+        results.images = clampCount(requested.images, "images");
+    if (requested.news !== undefined)
+        results.news = clampCount(requested.news, "news");
+    const filters = {};
+    const domains = intent.filters?.domains
+        ?.map((domain) => domain.trim().toLowerCase())
+        .filter((domain) => domain.length > 0);
+    if (domains && domains.length > 0)
+        filters.domains = [...new Set(domains)];
+    const language = intent.filters?.language?.trim();
+    if (language)
+        filters.language = language;
+    if (intent.filters?.recency !== undefined)
+        filters.recencyMs = parseRecency(intent.filters.recency);
+    return { query, results, filters };
+}
+export function parseRecency(value) {
+    const match = DURATION_PATTERN.exec(value.trim().toLowerCase());
+    if (!match) {
+        throw new SearchIntentError(`filters.recency must look like 24h, 7d, 2w, 6m, or 1y (got "${value}")`);
+    }
+    const amount = Number.parseInt(match[1], 10);
+    if (!Number.isFinite(amount) || amount < 1) {
+        throw new SearchIntentError(`filters.recency must be a positive duration (got "${value}")`);
+    }
+    return amount * DURATION_MS[match[2]];
+}
+/** Total results requested across every modality, for budget reporting. */
+export function totalRequestedResults(intent) {
+    return Object.values(intent.results).reduce((sum, count) => sum + (count ?? 0), 0);
+}
+//# sourceMappingURL=search-intent.js.map

package/dist/core/webfetch/search-intent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-intent.js","sourceRoot":"","sources":["../../../src/core/webfetch/search-intent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,4BAA4B,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AA0B5F,MAAM,CAAC,MAAM,qBAAqB,GAAG,4BAA4B,CAAC;AAClE,yEAAyE;AACzE,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC;AAC5C,MAAM,CAAC,MAAM,wBAAwB,GAAG,2BAA2B,CAAC;AAEpE,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;AAEjD,MAAM,WAAW,GAA2B;IAC3C,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;IACjB,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;IACtB,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;IAC1B,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;IAC3B,CAAC,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;CAC5B,CAAC;AAEF,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC3C,YAAY,OAAe,EAAE;QAC5B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAAA,CAChC;CACD;AAED,6EAA6E;AAC7E,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc,EAAU;IAC1D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,iBAAiB,CAAC,WAAW,MAAM,6BAA6B,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;AAAA,CAC9C;AAUD,MAAM,UAAU,mBAAmB,CAAC,MAAoB,EAAwB;IAC/E,6EAA6E;IAC7E,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,IAAI,iBAAiB,CAAC,gCAAgC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACvC,6EAA6E;IAC7E,+EAA+E;IAC/E,gFAAgF;IAChF,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvD,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAClD,OAAO,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3G,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5F,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,CAAC,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAEpF,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO;QACtC,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxC,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAClD,IAAI,QAAQ;QAAE,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC1C,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,SAAS;QAAE,OAAO,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpG,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAAA,CACnC;AAED,MAAM,UAAU,YAAY,CAAC,KAAa,EAAU;IACnD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAChE,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,IAAI,iBAAiB,CAAC,+DAA+D,KAAK,IAAI,CAAC,CAAC;IACvG,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,iBAAiB,CAAC,qDAAqD,KAAK,IAAI,CAAC,CAAC;IAC7F,CAAC;IACD,OAAO,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAAA,CACtC;AAED,2EAA2E;AAC3E,MAAM,UAAU,qBAAqB,CAAC,MAA4B,EAAU;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAAA,CAC3F","sourcesContent":["/**\n * Search intent: the harness-side description of what a search should return.\n *\n * One shape serves every backend, so `webfetch` can ask for images or news\n * without caring which provider answers. Cardinality belongs to the modality\n * (`results: {web: 5, images: 3}`) because a flat `sources + limit` is\n * ambiguous: 5 total, or 5 per source?\n *\n * Only fields a backend can actually honor live here. Deliberately absent until\n * something can serve them: per-source purpose labels, query-rewrite toggles,\n * and search budgets/planning. `strategy` is reserved for those.\n *\n * Tag sugar (`[images=3] [news=5] [recent=7d] [domains=openai.com] query`)\n * compiles into this type in the tool layer; backends never see tags.\n */\n\nimport { WEBFETCH_FIRECRAWL_MAX_LIMIT, WEBFETCH_REQUEST_TIMEOUT_MS } from \"./guardrails.ts\";\n\n/** Per-modality result counts. Absent means \"do not request this source\". */\nexport interface SearchResultCounts {\n\tweb?: number;\n\timages?: number;\n\tnews?: number;\n}\n\nexport interface SearchFilters {\n\t/** Hostnames or domain prefixes to restrict results to. */\n\tdomains?: string[];\n\t/** Maximum age of a result, e.g. `7d`, `24h`, `1w`, `1m`. Provider-mapped. */\n\trecency?: string;\n\t/** Two-letter language/country hint where the backend supports one. */\n\tlanguage?: string;\n}\n\nexport interface SearchIntent {\n\tquery: string;\n\tresults?: SearchResultCounts;\n\tfilters?: SearchFilters;\n\t/** Reserved for v2 search planning (depth budgets, fan-out, dedup). */\n\tstrategy?: never;\n}\n\nexport const SEARCH_MAX_PER_SOURCE = WEBFETCH_FIRECRAWL_MAX_LIMIT;\n/** Default per-source count when the caller gives no `results` block. */\nexport const SEARCH_DEFAULT_WEB_RESULTS = 5;\nexport const SEARCH_DEFAULT_WINDOW_MS = WEBFETCH_REQUEST_TIMEOUT_MS;\n\nconst DURATION_PATTERN = /^(\\d+)\\s*(h|d|w|m|y)$/;\n\nconst DURATION_MS: Record<string, number> = {\n\th: 60 * 60 * 1000,\n\td: 24 * 60 * 60 * 1000,\n\tw: 7 * 24 * 60 * 60 * 1000,\n\tm: 30 * 24 * 60 * 60 * 1000,\n\ty: 365 * 24 * 60 * 60 * 1000,\n};\n\nexport class SearchIntentError extends Error {\n\tconstructor(message: string) {\n\t\tsuper(message);\n\t\tthis.name = \"SearchIntentError\";\n\t}\n}\n\n/** Clamp one count into 1..SEARCH_MAX_PER_SOURCE, rejecting non-integers. */\nfunction clampCount(value: number, source: string): number {\n\tif (!Number.isFinite(value) || !Number.isInteger(value) || value < 1) {\n\t\tthrow new SearchIntentError(`results.${source} must be a positive integer`);\n\t}\n\treturn Math.min(value, SEARCH_MAX_PER_SOURCE);\n}\n\n/** Normalized intent: counts are clamped and unknown keys dropped. `web` is\n * present unless the caller named other sources explicitly. */\nexport interface ResolvedSearchIntent {\n\tquery: string;\n\tresults: SearchResultCounts;\n\tfilters: { domains?: string[]; recencyMs?: number; language?: string };\n}\n\nexport function resolveSearchIntent(intent: SearchIntent): ResolvedSearchIntent {\n\t// A named source replaces the default web-only set rather than adding to it.\n\tconst query = intent.query.trim();\n\tif (!query) {\n\t\tthrow new SearchIntentError(\"Search query must not be empty\");\n\t}\n\tconst requested = intent.results ?? {};\n\t// The web default applies only when no source was requested at all. Once the\n\t// caller names a source (say `[images=3]` or `{images: 3}`), the requested set\n\t// is exactly what they asked for: asking for images must not also return pages.\n\tconst namesSources = Object.keys(requested).length > 0;\n\tconst results: ResolvedSearchIntent[\"results\"] = {};\n\tif (!namesSources || requested.web !== undefined) {\n\t\tresults.web = requested.web === undefined ? SEARCH_DEFAULT_WEB_RESULTS : clampCount(requested.web, \"web\");\n\t}\n\tif (requested.images !== undefined) results.images = clampCount(requested.images, \"images\");\n\tif (requested.news !== undefined) results.news = clampCount(requested.news, \"news\");\n\n\tconst filters: ResolvedSearchIntent[\"filters\"] = {};\n\tconst domains = intent.filters?.domains\n\t\t?.map((domain) => domain.trim().toLowerCase())\n\t\t.filter((domain) => domain.length > 0);\n\tif (domains && domains.length > 0) filters.domains = [...new Set(domains)];\n\tconst language = intent.filters?.language?.trim();\n\tif (language) filters.language = language;\n\tif (intent.filters?.recency !== undefined) filters.recencyMs = parseRecency(intent.filters.recency);\n\treturn { query, results, filters };\n}\n\nexport function parseRecency(value: string): number {\n\tconst match = DURATION_PATTERN.exec(value.trim().toLowerCase());\n\tif (!match) {\n\t\tthrow new SearchIntentError(`filters.recency must look like 24h, 7d, 2w, 6m, or 1y (got \"${value}\")`);\n\t}\n\tconst amount = Number.parseInt(match[1], 10);\n\tif (!Number.isFinite(amount) || amount < 1) {\n\t\tthrow new SearchIntentError(`filters.recency must be a positive duration (got \"${value}\")`);\n\t}\n\treturn amount * DURATION_MS[match[2]];\n}\n\n/** Total results requested across every modality, for budget reporting. */\nexport function totalRequestedResults(intent: ResolvedSearchIntent): number {\n\treturn Object.values(intent.results).reduce<number>((sum, count) => sum + (count ?? 0), 0);\n}\n"]}

package/dist/core/webfetch/search-tags.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Optional tag sugar for `webfetch`: `[images=3] [news=5] [recent=7d]
+ * [domains=openai.com] nigerian tech startups`.
+ *
+ * A deliberately tiny grammar — `[key=value]` pairs plus `[key]` shorthands —
+ * that compiles into the same `SearchIntent` the structured parameters produce.
+ * Backends never see tags. Bracketed text that is not a recognized key stays in
+ * the query, so searching for the literal string "[img3]" still works.
+ *
+ * The tool layer applies this only when the structured `results`/`filters` args
+ * are absent, so an explicit call always wins over sugar in the query string.
+ */
+import { type SearchIntent } from "./search-intent.ts";
+export interface ParsedSearchQuery {
+    intent: SearchIntent;
+    /** Tags that were recognized and applied. */
+    appliedTags: string[];
+    /** Recognized tags that could not be parsed; the tool reports these. */
+    warnings: string[];
+}
+/** Strip recognized tags from the query and fold them into an intent. */
+export declare function parseSearchQueryTags(input: string): ParsedSearchQuery;
+//# sourceMappingURL=search-tags.d.ts.map

package/dist/core/webfetch/search-tags.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-tags.d.ts","sourceRoot":"","sources":["../../../src/core/webfetch/search-tags.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAoC,KAAK,YAAY,EAA2B,MAAM,oBAAoB,CAAC;AAelH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,YAAY,CAAC;IACrB,6CAA6C;IAC7C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,yEAAyE;AACzE,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA2FrE","sourcesContent":["/**\n * Optional tag sugar for `webfetch`: `[images=3] [news=5] [recent=7d]\n * [domains=openai.com] nigerian tech startups`.\n *\n * A deliberately tiny grammar — `[key=value]` pairs plus `[key]` shorthands —\n * that compiles into the same `SearchIntent` the structured parameters produce.\n * Backends never see tags. Bracketed text that is not a recognized key stays in\n * the query, so searching for the literal string \"[img3]\" still works.\n *\n * The tool layer applies this only when the structured `results`/`filters` args\n * are absent, so an explicit call always wins over sugar in the query string.\n */\n\nimport { parseRecency, type SearchFilters, type SearchIntent, type SearchResultCounts } from \"./search-intent.ts\";\n\n/** `[key]` shorthands for the most common request. */\nconst FLAG_SHORTHANDS: Record<string, keyof SearchResultCounts> = {\n\timg: \"images\",\n\timages: \"images\",\n\tnews: \"news\",\n\tweb: \"web\",\n};\n\n/** Keys that take a value inside brackets. */\nconst VALUE_KEYS = new Set([\"images\", \"news\", \"web\", \"recent\", \"domain\", \"domains\", \"lang\"]);\n\nconst TAG_PATTERN = /\\[([a-z]+)(?:=([^\\]]*))?\\]/gi;\n\nexport interface ParsedSearchQuery {\n\tintent: SearchIntent;\n\t/** Tags that were recognized and applied. */\n\tappliedTags: string[];\n\t/** Recognized tags that could not be parsed; the tool reports these. */\n\twarnings: string[];\n}\n\n/** Strip recognized tags from the query and fold them into an intent. */\nexport function parseSearchQueryTags(input: string): ParsedSearchQuery {\n\tconst counts: SearchResultCounts = {};\n\tconst filters: SearchFilters = {};\n\tconst appliedTags: string[] = [];\n\tconst warnings: string[] = [];\n\t/** Tags that produced a specific warning, so it is not reported twice. */\n\tconst warned = new Set<string>();\n\tlet query = input;\n\n\tconst replace = (match: string, key: string, value: string | undefined): string => {\n\t\tconst normalizedKey = key.toLowerCase();\n\t\tif (value === undefined) {\n\t\t\tconst source = FLAG_SHORTHANDS[normalizedKey];\n\t\t\tif (!source) return match;\n\t\t\tcounts[source] = (counts[source] ?? 0) + 1;\n\t\t\tappliedTags.push(match);\n\t\t\treturn \"\";\n\t\t}\n\t\tconst trimmed = value.trim();\n\t\tswitch (normalizedKey) {\n\t\t\tcase \"web\":\n\t\t\tcase \"images\":\n\t\t\tcase \"news\": {\n\t\t\t\tconst count = Number.parseInt(trimmed, 10);\n\t\t\t\tif (!Number.isInteger(count) || count < 1) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a positive integer count`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tcounts[normalizedKey] = count;\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tcase \"recent\": {\n\t\t\t\ttry {\n\t\t\t\t\tparseRecency(trimmed);\n\t\t\t\t\tfilters.recency = trimmed;\n\t\t\t\t\tappliedTags.push(match);\n\t\t\t\t\treturn \"\";\n\t\t\t\t} catch {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a duration like 24h, 7d, or 1m`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t}\n\t\t\tcase \"domain\":\n\t\t\tcase \"domains\": {\n\t\t\t\tconst domains = trimmed\n\t\t\t\t\t.split(/[,\\s]+/)\n\t\t\t\t\t.map((domain) => domain.trim())\n\t\t\t\t\t.filter(Boolean);\n\t\t\t\tif (domains.length === 0) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected one or more domains`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tfilters.domains = [...(filters.domains ?? []), ...domains];\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tcase \"lang\": {\n\t\t\t\tif (!/^[a-z]{2}$/i.test(trimmed)) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a two-letter language code`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tfilters.language = trimmed.toLowerCase();\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tdefault:\n\t\t\t\treturn match;\n\t\t}\n\t};\n\n\tquery = query.replace(TAG_PATTERN, (match, key: string, value: string | undefined) => replace(match, key, value));\n\n\t// A tag the replacer already rejected (bad count, bad duration) keeps its\n\t// specific warning; only report keys it never understood at all.\n\tfor (const match of query.match(TAG_PATTERN) ?? []) {\n\t\tif (warned.has(match)) continue;\n\t\tconst key = /\\[([a-z]+)(?:=([^\\]]*))?\\]/i.exec(match)?.[1]?.toLowerCase() ?? \"\";\n\t\tif (VALUE_KEYS.has(key) || key in FLAG_SHORTHANDS) {\n\t\t\twarnings.push(`Ignored ${match}: value was not understood`);\n\t\t}\n\t}\n\n\tconst intent: SearchIntent = { query: query.replace(/\\s{2,}/g, \" \").trim() };\n\tif (Object.keys(counts).length > 0) intent.results = counts;\n\tif (Object.keys(filters).length > 0) intent.filters = filters;\n\treturn { intent, appliedTags, warnings };\n}\n"]}

package/dist/core/webfetch/search-tags.js

@@ -0,0 +1,118 @@
+/**
+ * Optional tag sugar for `webfetch`: `[images=3] [news=5] [recent=7d]
+ * [domains=openai.com] nigerian tech startups`.
+ *
+ * A deliberately tiny grammar — `[key=value]` pairs plus `[key]` shorthands —
+ * that compiles into the same `SearchIntent` the structured parameters produce.
+ * Backends never see tags. Bracketed text that is not a recognized key stays in
+ * the query, so searching for the literal string "[img3]" still works.
+ *
+ * The tool layer applies this only when the structured `results`/`filters` args
+ * are absent, so an explicit call always wins over sugar in the query string.
+ */
+import { parseRecency } from "./search-intent.js";
+/** `[key]` shorthands for the most common request. */
+const FLAG_SHORTHANDS = {
+    img: "images",
+    images: "images",
+    news: "news",
+    web: "web",
+};
+/** Keys that take a value inside brackets. */
+const VALUE_KEYS = new Set(["images", "news", "web", "recent", "domain", "domains", "lang"]);
+const TAG_PATTERN = /\[([a-z]+)(?:=([^\]]*))?\]/gi;
+/** Strip recognized tags from the query and fold them into an intent. */
+export function parseSearchQueryTags(input) {
+    const counts = {};
+    const filters = {};
+    const appliedTags = [];
+    const warnings = [];
+    /** Tags that produced a specific warning, so it is not reported twice. */
+    const warned = new Set();
+    let query = input;
+    const replace = (match, key, value) => {
+        const normalizedKey = key.toLowerCase();
+        if (value === undefined) {
+            const source = FLAG_SHORTHANDS[normalizedKey];
+            if (!source)
+                return match;
+            counts[source] = (counts[source] ?? 0) + 1;
+            appliedTags.push(match);
+            return "";
+        }
+        const trimmed = value.trim();
+        switch (normalizedKey) {
+            case "web":
+            case "images":
+            case "news": {
+                const count = Number.parseInt(trimmed, 10);
+                if (!Number.isInteger(count) || count < 1) {
+                    warned.add(match);
+                    warnings.push(`Ignored ${match}: expected a positive integer count`);
+                    return match;
+                }
+                counts[normalizedKey] = count;
+                appliedTags.push(match);
+                return "";
+            }
+            case "recent": {
+                try {
+                    parseRecency(trimmed);
+                    filters.recency = trimmed;
+                    appliedTags.push(match);
+                    return "";
+                }
+                catch {
+                    warned.add(match);
+                    warnings.push(`Ignored ${match}: expected a duration like 24h, 7d, or 1m`);
+                    return match;
+                }
+            }
+            case "domain":
+            case "domains": {
+                const domains = trimmed
+                    .split(/[,\s]+/)
+                    .map((domain) => domain.trim())
+                    .filter(Boolean);
+                if (domains.length === 0) {
+                    warned.add(match);
+                    warnings.push(`Ignored ${match}: expected one or more domains`);
+                    return match;
+                }
+                filters.domains = [...(filters.domains ?? []), ...domains];
+                appliedTags.push(match);
+                return "";
+            }
+            case "lang": {
+                if (!/^[a-z]{2}$/i.test(trimmed)) {
+                    warned.add(match);
+                    warnings.push(`Ignored ${match}: expected a two-letter language code`);
+                    return match;
+                }
+                filters.language = trimmed.toLowerCase();
+                appliedTags.push(match);
+                return "";
+            }
+            default:
+                return match;
+        }
+    };
+    query = query.replace(TAG_PATTERN, (match, key, value) => replace(match, key, value));
+    // A tag the replacer already rejected (bad count, bad duration) keeps its
+    // specific warning; only report keys it never understood at all.
+    for (const match of query.match(TAG_PATTERN) ?? []) {
+        if (warned.has(match))
+            continue;
+        const key = /\[([a-z]+)(?:=([^\]]*))?\]/i.exec(match)?.[1]?.toLowerCase() ?? "";
+        if (VALUE_KEYS.has(key) || key in FLAG_SHORTHANDS) {
+            warnings.push(`Ignored ${match}: value was not understood`);
+        }
+    }
+    const intent = { query: query.replace(/\s{2,}/g, " ").trim() };
+    if (Object.keys(counts).length > 0)
+        intent.results = counts;
+    if (Object.keys(filters).length > 0)
+        intent.filters = filters;
+    return { intent, appliedTags, warnings };
+}
+//# sourceMappingURL=search-tags.js.map

package/dist/core/webfetch/search-tags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-tags.js","sourceRoot":"","sources":["../../../src/core/webfetch/search-tags.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAkE,MAAM,oBAAoB,CAAC;AAElH,sDAAsD;AACtD,MAAM,eAAe,GAA6C;IACjE,GAAG,EAAE,QAAQ;IACb,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;CACV,CAAC;AAEF,8CAA8C;AAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAE7F,MAAM,WAAW,GAAG,8BAA8B,CAAC;AAUnD,yEAAyE;AACzE,MAAM,UAAU,oBAAoB,CAAC,KAAa,EAAqB;IACtE,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,0EAA0E;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,KAAyB,EAAU,EAAE,CAAC;QAClF,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,eAAe,CAAC,aAAa,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAC3C,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO,EAAE,CAAC;QACX,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7B,QAAQ,aAAa,EAAE,CAAC;YACvB,KAAK,KAAK,CAAC;YACX,KAAK,QAAQ,CAAC;YACd,KAAK,MAAM,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC3C,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,qCAAqC,CAAC,CAAC;oBACrE,OAAO,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,CAAC,aAAa,CAAC,GAAG,KAAK,CAAC;gBAC9B,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,EAAE,CAAC;YACX,CAAC;YACD,KAAK,QAAQ,EAAE,CAAC;gBACf,IAAI,CAAC;oBACJ,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;oBAC1B,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACxB,OAAO,EAAE,CAAC;gBACX,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,2CAA2C,CAAC,CAAC;oBAC3E,OAAO,KAAK,CAAC;gBACd,CAAC;YACF,CAAC;YACD,KAAK,QAAQ,CAAC;YACd,KAAK,SAAS,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,OAAO;qBACrB,KAAK,CAAC,QAAQ,CAAC;qBACf,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;qBAC9B,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,gCAAgC,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO,CAAC,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,CAAC;gBAC3D,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,EAAE,CAAC;YACX,CAAC;YACD,KAAK,MAAM,EAAE,CAAC;gBACb,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAClB,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,uCAAuC,CAAC,CAAC;oBACvE,OAAO,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;gBACzC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,EAAE,CAAC;YACX,CAAC;YACD;gBACC,OAAO,KAAK,CAAC;QACf,CAAC;IAAA,CACD,CAAC;IAEF,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,KAAyB,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IAElH,0EAA0E;IAC1E,iEAAiE;IACjE,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;QACpD,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAChC,MAAM,GAAG,GAAG,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAChF,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;YACnD,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,4BAA4B,CAAC,CAAC;QAC7D,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IAC7E,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC;IAC5D,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IAC9D,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AAAA,CACzC","sourcesContent":["/**\n * Optional tag sugar for `webfetch`: `[images=3] [news=5] [recent=7d]\n * [domains=openai.com] nigerian tech startups`.\n *\n * A deliberately tiny grammar — `[key=value]` pairs plus `[key]` shorthands —\n * that compiles into the same `SearchIntent` the structured parameters produce.\n * Backends never see tags. Bracketed text that is not a recognized key stays in\n * the query, so searching for the literal string \"[img3]\" still works.\n *\n * The tool layer applies this only when the structured `results`/`filters` args\n * are absent, so an explicit call always wins over sugar in the query string.\n */\n\nimport { parseRecency, type SearchFilters, type SearchIntent, type SearchResultCounts } from \"./search-intent.ts\";\n\n/** `[key]` shorthands for the most common request. */\nconst FLAG_SHORTHANDS: Record<string, keyof SearchResultCounts> = {\n\timg: \"images\",\n\timages: \"images\",\n\tnews: \"news\",\n\tweb: \"web\",\n};\n\n/** Keys that take a value inside brackets. */\nconst VALUE_KEYS = new Set([\"images\", \"news\", \"web\", \"recent\", \"domain\", \"domains\", \"lang\"]);\n\nconst TAG_PATTERN = /\\[([a-z]+)(?:=([^\\]]*))?\\]/gi;\n\nexport interface ParsedSearchQuery {\n\tintent: SearchIntent;\n\t/** Tags that were recognized and applied. */\n\tappliedTags: string[];\n\t/** Recognized tags that could not be parsed; the tool reports these. */\n\twarnings: string[];\n}\n\n/** Strip recognized tags from the query and fold them into an intent. */\nexport function parseSearchQueryTags(input: string): ParsedSearchQuery {\n\tconst counts: SearchResultCounts = {};\n\tconst filters: SearchFilters = {};\n\tconst appliedTags: string[] = [];\n\tconst warnings: string[] = [];\n\t/** Tags that produced a specific warning, so it is not reported twice. */\n\tconst warned = new Set<string>();\n\tlet query = input;\n\n\tconst replace = (match: string, key: string, value: string | undefined): string => {\n\t\tconst normalizedKey = key.toLowerCase();\n\t\tif (value === undefined) {\n\t\t\tconst source = FLAG_SHORTHANDS[normalizedKey];\n\t\t\tif (!source) return match;\n\t\t\tcounts[source] = (counts[source] ?? 0) + 1;\n\t\t\tappliedTags.push(match);\n\t\t\treturn \"\";\n\t\t}\n\t\tconst trimmed = value.trim();\n\t\tswitch (normalizedKey) {\n\t\t\tcase \"web\":\n\t\t\tcase \"images\":\n\t\t\tcase \"news\": {\n\t\t\t\tconst count = Number.parseInt(trimmed, 10);\n\t\t\t\tif (!Number.isInteger(count) || count < 1) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a positive integer count`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tcounts[normalizedKey] = count;\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tcase \"recent\": {\n\t\t\t\ttry {\n\t\t\t\t\tparseRecency(trimmed);\n\t\t\t\t\tfilters.recency = trimmed;\n\t\t\t\t\tappliedTags.push(match);\n\t\t\t\t\treturn \"\";\n\t\t\t\t} catch {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a duration like 24h, 7d, or 1m`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t}\n\t\t\tcase \"domain\":\n\t\t\tcase \"domains\": {\n\t\t\t\tconst domains = trimmed\n\t\t\t\t\t.split(/[,\\s]+/)\n\t\t\t\t\t.map((domain) => domain.trim())\n\t\t\t\t\t.filter(Boolean);\n\t\t\t\tif (domains.length === 0) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected one or more domains`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tfilters.domains = [...(filters.domains ?? []), ...domains];\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tcase \"lang\": {\n\t\t\t\tif (!/^[a-z]{2}$/i.test(trimmed)) {\n\t\t\t\t\twarned.add(match);\n\t\t\t\t\twarnings.push(`Ignored ${match}: expected a two-letter language code`);\n\t\t\t\t\treturn match;\n\t\t\t\t}\n\t\t\t\tfilters.language = trimmed.toLowerCase();\n\t\t\t\tappliedTags.push(match);\n\t\t\t\treturn \"\";\n\t\t\t}\n\t\t\tdefault:\n\t\t\t\treturn match;\n\t\t}\n\t};\n\n\tquery = query.replace(TAG_PATTERN, (match, key: string, value: string | undefined) => replace(match, key, value));\n\n\t// A tag the replacer already rejected (bad count, bad duration) keeps its\n\t// specific warning; only report keys it never understood at all.\n\tfor (const match of query.match(TAG_PATTERN) ?? []) {\n\t\tif (warned.has(match)) continue;\n\t\tconst key = /\\[([a-z]+)(?:=([^\\]]*))?\\]/i.exec(match)?.[1]?.toLowerCase() ?? \"\";\n\t\tif (VALUE_KEYS.has(key) || key in FLAG_SHORTHANDS) {\n\t\t\twarnings.push(`Ignored ${match}: value was not understood`);\n\t\t}\n\t}\n\n\tconst intent: SearchIntent = { query: query.replace(/\\s{2,}/g, \" \").trim() };\n\tif (Object.keys(counts).length > 0) intent.results = counts;\n\tif (Object.keys(filters).length > 0) intent.filters = filters;\n\treturn { intent, appliedTags, warnings };\n}\n"]}

package/dist/env-compat.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=env-compat.d.ts.map

package/dist/env-compat.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env-compat.d.ts","sourceRoot":"","sources":["../src/env-compat.ts"],"names":[],"mappings":"","sourcesContent":["/**\n * One-release fallback: if ION_* env is not set but PI_* is set, copy PI_ to ION_.\n * This runs at import time before other modules read env vars.\n */\nfor (const [k, v] of Object.entries(process.env)) {\n\tif (k.startsWith(\"PI_\") && v !== undefined) {\n\t\tconst ionKey = k.replace(/^PI_/, \"ION_\");\n\t\tif (process.env[ionKey] === undefined) {\n\t\t\tprocess.env[ionKey] = v;\n\t\t}\n\t}\n}\n// Also handle PI_ fallback for XDG? Not needed, XDG not prefixed.\n// Handle legacy PI_* env that should also be readable as ION_* for one release.\n// This file intentionally has side-effects on import.\n"]}

package/dist/env-compat.js

@@ -0,0 +1,17 @@
+/**
+ * One-release fallback: if ION_* env is not set but PI_* is set, copy PI_ to ION_.
+ * This runs at import time before other modules read env vars.
+ */
+for (const [k, v] of Object.entries(process.env)) {
+    if (k.startsWith("PI_") && v !== undefined) {
+        const ionKey = k.replace(/^PI_/, "ION_");
+        if (process.env[ionKey] === undefined) {
+            process.env[ionKey] = v;
+        }
+    }
+}
+export {};
+// Also handle PI_ fallback for XDG? Not needed, XDG not prefixed.
+// Handle legacy PI_* env that should also be readable as ION_* for one release.
+// This file intentionally has side-effects on import.
+//# sourceMappingURL=env-compat.js.map

package/dist/env-compat.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"env-compat.js","sourceRoot":"","sources":["../src/env-compat.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;IAClD,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;AACF,CAAC;;AACD,kEAAkE;AAClE,gFAAgF;AAChF,sDAAsD","sourcesContent":["/**\n * One-release fallback: if ION_* env is not set but PI_* is set, copy PI_ to ION_.\n * This runs at import time before other modules read env vars.\n */\nfor (const [k, v] of Object.entries(process.env)) {\n\tif (k.startsWith(\"PI_\") && v !== undefined) {\n\t\tconst ionKey = k.replace(/^PI_/, \"ION_\");\n\t\tif (process.env[ionKey] === undefined) {\n\t\t\tprocess.env[ionKey] = v;\n\t\t}\n\t}\n}\n// Also handle PI_ fallback for XDG? Not needed, XDG not prefixed.\n// Handle legacy PI_* env that should also be readable as ION_* for one release.\n// This file intentionally has side-effects on import.\n"]}