@agent-native/core 0.52.0 → 0.54.0

package/dist/eval/types.d.ts

@@ -0,0 +1,162 @@
+/**
+ * Types for the first-class evals primitive.
+ *
+ * This is a *test-case* eval system (define a prompt + expected behavior,
+ * actually run the agent, score the output) — distinct from the post-hoc
+ * run-scoring engine in `../observability/evals.ts`, which scores already-
+ * completed production runs. The two are complementary:
+ *
+ *   - `observability/evals.ts` — "how did this real run do?" (passive,
+ *     sampled, lives next to traces).
+ *   - `eval/*` (this module) — "does the agent do the right thing on this
+ *     fixed input?" (active, deterministic CI gate, run via the CLI).
+ *
+ * The pipeline shape (preprocess → analyze → generateScore → generateReason)
+ * is borrowed from Mastra's scorer design: each scorer is a small, composable
+ * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM
+ * judge while still producing one normalized 0..1 number plus a reason.
+ */
+import type { AgentEngine } from "../agent/engine/types.js";
+/**
+ * The result of actually running the agent loop for one eval input. Scorers
+ * receive this as the thing under test. It is intentionally small and
+ * transport-agnostic so a scorer never reaches into framework internals.
+ */
+export interface AgentRunOutput {
+    /** Concatenated assistant text emitted across the run. */
+    readonly text: string;
+    /** Names of tools/actions the agent invoked, in call order. */
+    readonly toolCalls: readonly string[];
+    /** Whether the run completed without a terminal error event. */
+    readonly ok: boolean;
+    /** Terminal error message, if the run errored. */
+    readonly error?: string;
+    /** Synthetic run id, useful for writing eval rows to the observability store. */
+    readonly runId: string;
+    /** Wall-clock duration of the run in milliseconds. */
+    readonly durationMs: number;
+}
+/**
+ * Context handed to a scorer's analyze step when it needs an LLM judge. The
+ * engine/model are resolved by the runner from the existing engine registry —
+ * a scorer NEVER hardcodes a provider or model, keeping evals provider-
+ * agnostic. `judge()` is a convenience that streams a single judging turn and
+ * returns the raw model text.
+ */
+export interface ScorerAnalyzeContext {
+    /** The resolved, provider-agnostic engine for LLM-judge scorers. */
+    readonly engine: AgentEngine;
+    /** The resolved model string for the engine. */
+    readonly model: string;
+    /**
+     * Run a single LLM judging turn. Returns the model's raw text output. Used
+     * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —
+     * the engine is whatever the app/CLI resolved.
+     */
+    judge(opts: {
+        systemPrompt?: string;
+        prompt: string;
+        maxOutputTokens?: number;
+        signal?: AbortSignal;
+    }): Promise<string>;
+}
+/**
+ * A 4-step scoring pipeline (Mastra-style):
+ *
+ *   preprocess(run)      → x        (transform the run/output; optional)
+ *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)
+ *   generateScore(a)     → 0..1     (REQUIRED, normalized)
+ *   generateReason(...)  → string   (human-readable why; optional)
+ *
+ * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a
+ * single scorer is fully typed end-to-end.
+ */
+export interface Scorer<Pre = AgentRunOutput, Ana = Pre> {
+    readonly name: string;
+    preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;
+    analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;
+    /** REQUIRED. Returns a normalized score in [0, 1]. */
+    generateScore(analysis: Ana): number | Promise<number>;
+    generateReason?(args: {
+        run: AgentRunOutput;
+        analysis: Ana;
+        score: number;
+    }): string | Promise<string>;
+}
+/** Definition object passed to `createScorer`. */
+export interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {
+    name: string;
+    preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;
+    analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;
+    generateScore(analysis: Ana): number | Promise<number>;
+    generateReason?(args: {
+        run: AgentRunOutput;
+        analysis: Ana;
+        score: number;
+    }): string | Promise<string>;
+}
+/** The prompt + optional setup that drives one eval case. */
+export interface EvalInput {
+    /** The user prompt / message sent to the agent. */
+    prompt: string;
+    /**
+     * Optional prior conversation turns to seed before `prompt`. Each is a
+     * plain { role, text } pair; the runner converts them to engine messages.
+     */
+    history?: Array<{
+        role: "user" | "assistant";
+        text: string;
+    }>;
+}
+/** Context passed to an eval's optional `run` override. */
+export interface EvalRunContext {
+    readonly input: EvalInput;
+    /** The default agent runner — invoke it to run the agent loop as a caller. */
+    runAgent(input: EvalInput): Promise<AgentRunOutput>;
+}
+/**
+ * A named eval = one test case. `scorers` produce per-scorer rows; the case
+ * passes when EVERY scorer meets `threshold` (default 0.5, overridable per
+ * eval and globally from the CLI).
+ */
+export interface Eval {
+    name: string;
+    input: EvalInput;
+    /**
+     * Optional override for how the agent is run for this case. Defaults to the
+     * runner's headless `runAgent`. Use this to do custom setup (seed data,
+     * multi-turn) before/after the agent call.
+     */
+    run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;
+    scorers: Scorer<any, any>[];
+    /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */
+    threshold?: number;
+}
+/** One result row per (eval × scorer). Stores both the number AND the reason. */
+export interface ScorerResult {
+    scorer: string;
+    score: number;
+    reason?: string;
+    passed: boolean;
+}
+/** Aggregated result for a single eval (all of its scorers). */
+export interface EvalResultRow {
+    eval: string;
+    threshold: number;
+    scores: ScorerResult[];
+    /** True only when every scorer passed. */
+    passed: boolean;
+    /** Mean of the scorer scores, for at-a-glance ranking. */
+    avgScore: number;
+    durationMs: number;
+    /** Terminal error if the agent run itself failed. */
+    error?: string;
+}
+/** The full runner report. */
+export interface EvalRunReport {
+    total: number;
+    passed: number;
+    failed: number;
+    results: EvalResultRow[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}

package/dist/eval/types.js

@@ -0,0 +1,20 @@
+/**
+ * Types for the first-class evals primitive.
+ *
+ * This is a *test-case* eval system (define a prompt + expected behavior,
+ * actually run the agent, score the output) — distinct from the post-hoc
+ * run-scoring engine in `../observability/evals.ts`, which scores already-
+ * completed production runs. The two are complementary:
+ *
+ *   - `observability/evals.ts` — "how did this real run do?" (passive,
+ *     sampled, lives next to traces).
+ *   - `eval/*` (this module) — "does the agent do the right thing on this
+ *     fixed input?" (active, deterministic CI gate, run via the CLI).
+ *
+ * The pipeline shape (preprocess → analyze → generateScore → generateReason)
+ * is borrowed from Mastra's scorer design: each scorer is a small, composable
+ * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM
+ * judge while still producing one normalized 0..1 number plus a reason.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/eval/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/**\n * Types for the first-class evals primitive.\n *\n * This is a *test-case* eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n *\n *   - `observability/evals.ts` — \"how did this real run do?\" (passive,\n *     sampled, lives next to traces).\n *   - `eval/*` (this module) — \"does the agent do the right thing on this\n *     fixed input?\" (active, deterministic CI gate, run via the CLI).\n *\n * The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n */\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/**\n * The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n */\nexport interface AgentRunOutput {\n  /** Concatenated assistant text emitted across the run. */\n  readonly text: string;\n  /** Names of tools/actions the agent invoked, in call order. */\n  readonly toolCalls: readonly string[];\n  /** Whether the run completed without a terminal error event. */\n  readonly ok: boolean;\n  /** Terminal error message, if the run errored. */\n  readonly error?: string;\n  /** Synthetic run id, useful for writing eval rows to the observability store. */\n  readonly runId: string;\n  /** Wall-clock duration of the run in milliseconds. */\n  readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/**\n * Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n */\nexport interface ScorerAnalyzeContext {\n  /** The resolved, provider-agnostic engine for LLM-judge scorers. */\n  readonly engine: AgentEngine;\n  /** The resolved model string for the engine. */\n  readonly model: string;\n  /**\n   * Run a single LLM judging turn. Returns the model's raw text output. Used\n   * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n   * the engine is whatever the app/CLI resolved.\n   */\n  judge(opts: {\n    systemPrompt?: string;\n    prompt: string;\n    maxOutputTokens?: number;\n    signal?: AbortSignal;\n  }): Promise<string>;\n}\n\n/**\n * A 4-step scoring pipeline (Mastra-style):\n *\n *   preprocess(run)      → x        (transform the run/output; optional)\n *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)\n *   generateScore(a)     → 0..1     (REQUIRED, normalized)\n *   generateReason(...)  → string   (human-readable why; optional)\n *\n * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n */\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n  readonly name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  /** REQUIRED. Returns a normalized score in [0, 1]. */\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n/** Definition object passed to `createScorer`. */\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n  name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/** The prompt + optional setup that drives one eval case. */\nexport interface EvalInput {\n  /** The user prompt / message sent to the agent. */\n  prompt: string;\n  /**\n   * Optional prior conversation turns to seed before `prompt`. Each is a\n   * plain { role, text } pair; the runner converts them to engine messages.\n   */\n  history?: Array<{ role: \"user\" | \"assistant\"; text: string }>;\n}\n\n/** Context passed to an eval's optional `run` override. */\nexport interface EvalRunContext {\n  readonly input: EvalInput;\n  /** The default agent runner — invoke it to run the agent loop as a caller. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/**\n * A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n */\nexport interface Eval {\n  name: string;\n  input: EvalInput;\n  /**\n   * Optional override for how the agent is run for this case. Defaults to the\n   * runner's headless `runAgent`. Use this to do custom setup (seed data,\n   * multi-turn) before/after the agent call.\n   */\n  run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;\n  scorers: Scorer<any, any>[];\n  /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */\n  threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/** One result row per (eval × scorer). Stores both the number AND the reason. */\nexport interface ScorerResult {\n  scorer: string;\n  score: number;\n  reason?: string;\n  passed: boolean;\n}\n\n/** Aggregated result for a single eval (all of its scorers). */\nexport interface EvalResultRow {\n  eval: string;\n  threshold: number;\n  scores: ScorerResult[];\n  /** True only when every scorer passed. */\n  passed: boolean;\n  /** Mean of the scorer scores, for at-a-glance ranking. */\n  avgScore: number;\n  durationMs: number;\n  /** Terminal error if the agent run itself failed. */\n  error?: string;\n}\n\n/** The full runner report. */\nexport interface EvalRunReport {\n  total: number;\n  passed: number;\n  failed: number;\n  results: EvalResultRow[];\n}\n"]}

package/dist/extensions/fetch-tool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-tool.d.ts","sourceRoot":"","sources":["../../src/extensions/fetch-tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAwDhE,MAAM,WAAW,gBAAgB;IAC/B,6EAA6E;IAC7E,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;QACtC,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;KACzB,CAAC,CAAC;IACH,+CAA+C;IAC/C,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,GAAE,gBAAqB,GAC1B,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CA+Q7B"}
+{"version":3,"file":"fetch-tool.d.ts","sourceRoot":"","sources":["../../src/extensions/fetch-tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AA6DhE,MAAM,WAAW,gBAAgB;IAC/B,6EAA6E;IAC7E,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC;QACtC,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;KACzB,CAAC,CAAC;IACH,+CAA+C;IAC/C,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACrE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,GAAE,gBAAqB,GAC1B,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CA+U7B"}

package/dist/extensions/fetch-tool.js

@@ -11,6 +11,7 @@
  */
 import { collectSecretValues, MAX_EXTENSION_PROXY_RESPONSE_SIZE, normalizeExtensionProxyMethod, readResponseTextWithLimit, redactSecrets, redactString, sanitizeOutboundHeaders, } from "./proxy-security.js";
 import { createSsrfSafeDispatcher, isBlockedExtensionUrlWithDns, } from "./url-safety.js";
+import { formatWebContentResult, parseWebContentSearchOptions, processWebContent, } from "./web-content.js";
 const DEFAULT_TIMEOUT_MS = 15_000;
 /**
  * Headers that mimic a current Chrome on macOS so anti-bot middleware (Cloudflare,
@@ -81,6 +82,43 @@ export function createFetchToolEntry(opts = {}) {
                             type: "number",
                             description: "Maximum response body characters to return. Default: 32000. Max: 200000. Increase when you need to read a large document, API response, or dataset.",
                         },
+                        responseMode: {
+                            type: "string",
+                            description: "How to return the response. Default: auto (HTML pages become clean markdown; JSON/text stays raw). Use raw for exact bytes, markdown/text for extracted readable content, links for just links, metadata for page metadata, or matches with search.",
+                            enum: [
+                                "auto",
+                                "raw",
+                                "text",
+                                "markdown",
+                                "links",
+                                "metadata",
+                                "matches",
+                            ],
+                        },
+                        extract: {
+                            type: "string",
+                            description: "HTML extraction strategy. Default: readability. Use all-visible for visible body text/markdown, or none to convert the full HTML document.",
+                            enum: ["readability", "all-visible", "none"],
+                        },
+                        includeLinks: {
+                            type: "boolean",
+                            description: "Whether extracted HTML responses should include a compact links list. Default: true for extracted pages.",
+                        },
+                        search: {
+                            type: "object",
+                            description: "Optional post-fetch search over extracted content by default. Supports {query, queries, terms, regex, regexFlags, source:'extracted'|'raw', maxMatches, contextChars, caseSensitive}. Regex is safety-checked and bounded; prefer query/terms for simple grep-like searches.",
+                            properties: {
+                                query: { type: "string" },
+                                queries: { type: "array", items: { type: "string" } },
+                                terms: { type: "array", items: { type: "string" } },
+                                regex: { type: "string" },
+                                regexFlags: { type: "string" },
+                                source: { type: "string", enum: ["extracted", "raw"] },
+                                maxMatches: { type: "number" },
+                                contextChars: { type: "number" },
+                                caseSensitive: { type: "boolean" },
+                            },
+                        },
                         saveToFile: {
                             type: "string",
                             description: "Workspace file path to save the full response body to instead of returning it in context (e.g. 'analysis/page.html'). When set, returns only a compact summary {savedTo, status, bytes, preview}. Useful for large web pages or API responses that would overflow context.",
@@ -91,13 +129,19 @@ export function createFetchToolEntry(opts = {}) {
             },
             run: async (args) => {
                 const startTime = Date.now();
-                const rawUrl = args.url;
+                const rawUrl = String(args.url ?? "");
                 const method = normalizeExtensionProxyMethod(args.method || "GET");
                 if (!method) {
                     return "Unsupported HTTP method. Allowed methods: GET, POST, PUT, PATCH, DELETE, HEAD.";
                 }
-                const rawHeaders = args.headers || "{}";
-                const rawBody = args.body;
+                const rawHeaders = typeof args.headers === "string"
+                    ? args.headers
+                    : JSON.stringify(args.headers ?? {});
+                const rawBody = typeof args.body === "string"
+                    ? args.body
+                    : args.body === undefined || args.body === null
+                        ? undefined
+                        : JSON.stringify(args.body);
                 const timeoutMs = Math.min(Number(args.timeout_ms) || DEFAULT_TIMEOUT_MS, 30_000);
                 const requestedMaxChars = Number(args.maxChars);
                 const maxChars = Number.isFinite(requestedMaxChars) && requestedMaxChars > 0
@@ -216,6 +260,29 @@ export function createFetchToolEntry(opts = {}) {
                         body = "(could not read response body)";
                     }
                     body = redactString(body, secretValues);
+                    const contentType = response.headers.get("content-type")?.split(";")[0].trim() ??
+                        "text/plain";
+                    let displayBody;
+                    let processedMode = "raw";
+                    try {
+                        const processed = processWebContent({
+                            url: resolvedUrl,
+                            body,
+                            contentType,
+                            responseMode: String(args.responseMode ?? "auto"),
+                            extract: String(args.extract ?? "readability"),
+                            includeLinks: args.includeLinks === undefined
+                                ? true
+                                : parseBooleanArg(args.includeLinks),
+                            search: parseWebContentSearchOptions(args.search),
+                            maxChars,
+                        });
+                        processedMode = processed.mode;
+                        displayBody = formatWebContentResult(processed);
+                    }
+                    catch (err) {
+                        return `web-request post-processing error: ${err?.message ?? String(err)}`;
+                    }
                     // Audit log
                     console.log(`[fetch-tool] ${method} ${rawUrl} → ${response.status} (${elapsed}ms, keys: ${allUsedKeys.join(",") || "none"})`);
                     // saveToFile: write full body to workspace and return compact summary.
@@ -232,34 +299,26 @@ export function createFetchToolEntry(opts = {}) {
                                     : null;
                             if (!scope)
                                 throw new Error("No authenticated context for saveToFile");
-                            const contentType = response.headers.get("content-type")?.split(";")[0].trim() ??
-                                "text/plain";
                             await writeWorkspaceFile(scope, saveToFilePath, body, contentType, {
                                 maxFileBytes: SAVE_TO_FILE_MAX_BYTES,
                             });
                             const bytes = Buffer.byteLength(body, "utf8");
-                            const preview = body.slice(0, 2000);
+                            const preview = displayBody.slice(0, 2000);
                             return JSON.stringify({
                                 savedToFile: true,
                                 savedTo: saveToFilePath,
                                 status: response.status,
                                 bytes,
                                 contentType,
-                                preview: preview.length < body.length ? `${preview}…` : preview,
+                                responseMode: processedMode,
+                                preview: preview.length < displayBody.length ? `${preview}…` : preview,
                             });
                         }
                         catch (saveErr) {
                             return `saveToFile error: ${saveErr?.message ?? String(saveErr)}\n\nHTTP ${response.status} ${response.statusText}\n\n${body.slice(0, maxChars)}`;
                         }
                     }
-                    // Truncate very long responses for the agent. Default cap is 32 k
-                    // chars (~8 k tokens), enough to read a full article or scrape a
-                    // stats table without blowing out the model's context window. The
-                    // caller may request up to 200 000 chars via the maxChars input.
-                    if (body.length > maxChars) {
-                        body = body.slice(0, maxChars) + "\n... (truncated)";
-                    }
-                    return `HTTP ${response.status} ${response.statusText}\n\n${body}`;
+                    return `HTTP ${response.status} ${response.statusText}\n\n${displayBody}`;
                 }
                 catch (err) {
                     const elapsed = Date.now() - startTime;
@@ -279,4 +338,10 @@ export function createFetchToolEntry(opts = {}) {
         },
     };
 }
+function parseBooleanArg(value) {
+    if (typeof value === "boolean")
+        return value;
+    const normalized = String(value).trim().toLowerCase();
+    return normalized === "true" || normalized === "1" || normalized === "yes";
+}
 //# sourceMappingURL=fetch-tool.js.map

package/dist/extensions/fetch-tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-tool.js","sourceRoot":"","sources":["../../src/extensions/fetch-tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACL,mBAAmB,EACnB,iCAAiC,EACjC,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EACb,YAAY,EACZ,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,wBAAwB,EACxB,4BAA4B,GAC7B,MAAM,iBAAiB,CAAC;AAEzB,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC;;;;;;;;;GASG;AACH,MAAM,uBAAuB,GAA2B;IACtD,YAAY,EACV,uHAAuH;IACzH,MAAM,EACJ,kGAAkG;IACpG,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,WAAW,EACT,mEAAmE;IACrE,kBAAkB,EAAE,IAAI;IACxB,oBAAoB,EAAE,SAAS;IAC/B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,MAAM;IACxB,gBAAgB,EAAE,IAAI;IACtB,2BAA2B,EAAE,GAAG;CACjC,CAAC;AAEF,SAAS,oBAAoB,CAC3B,OAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAAE,CAAC;QACpE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAyB,EAAE;IAE3B,OAAO;QACL,aAAa,EAAE;YACb,IAAI,EAAE;gBACJ,WAAW,EAAE,u4BAAu4B;gBACp5B,UAAU,EAAE;oBACV,IAAI,EAAE,QAAiB;oBACvB,UAAU,EAAE;wBACV,GAAG,EAAE;4BACH,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,8EAA8E;yBACjF;wBACD,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,4BAA4B;4BACzC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;yBACxD;wBACD,OAAO,EAAE;4BACP,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,0HAA0H;yBAC7H;wBACD,IAAI,EAAE;4BACJ,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,yEAAyE;yBAC5E;wBACD,UAAU,EAAE;4BACV,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,qCAAqC,kBAAkB,eAAe;yBACpF;wBACD,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,qJAAqJ;yBACxJ;wBACD,UAAU,EAAE;4BACV,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,4QAA4Q;yBAC/Q;qBACF;oBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;iBAClB;aACF;YACD,GAAG,EAAE,KAAK,EAAE,IAA4B,EAAE,EAAE;gBAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC;gBACxB,MAAM,MAAM,GAAG,6BAA6B,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,gFAAgF,CAAC;gBAC1F,CAAC;gBACD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;gBACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC;gBAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,kBAAkB,EAC7C,MAAM,CACP,CAAC;gBACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC;oBACzD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC;oBACtC,CAAC,CAAC,MAAM,CAAC;gBAEb,yBAAyB;gBACzB,IAAI,WAAW,GAAG,MAAM,CAAC;gBACzB,IAAI,eAAe,GAAG,UAAU,CAAC;gBACjC,IAAI,YAAY,GAAG,OAAO,CAAC;gBAC3B,MAAM,WAAW,GAAa,EAAE,CAAC;gBACjC,MAAM,eAAe,GAAa,EAAE,CAAC;gBAErC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;wBACjD,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC;wBACjC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;wBACxC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAExD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;wBACxD,eAAe,GAAG,YAAY,CAAC,QAAQ,CAAC;wBACxC,WAAW,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;wBAC3C,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAE3D,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;4BACnD,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC;4BACnC,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACzC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAC3D,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAClB,OAAO,mCAAmC,GAAG,EAAE,OAAO,IAAI,GAAG,EAAE,CAAC;oBAClE,CAAC;gBACH,CAAC;gBACD,MAAM,YAAY,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;gBAE1D,6CAA6C;gBAC7C,IAAI,MAAM,4BAA4B,CAAC,WAAW,CAAC,EAAE,CAAC;oBACpD,OAAO,4DAA4D,MAAM,IAAI,CAAC;gBAChF,CAAC;gBAED,0CAA0C;gBAC1C,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/C,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;wBACjE,IAAI,CAAC,OAAO,EAAE,CAAC;4BACb,OAAO,QAAQ,MAAM,6EAA6E,CAAC;wBACrG,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAClB,OAAO,yBAAyB,GAAG,EAAE,OAAO,IAAI,GAAG,EAAE,CAAC;oBACxD,CAAC;gBACH,CAAC;gBAED,wEAAwE;gBACxE,mEAAmE;gBACnE,wEAAwE;gBACxE,mEAAmE;gBACnE,kEAAkE;gBAClE,qBAAqB;gBACrB,IAAI,OAA+B,CAAC;gBACpC,IAAI,CAAC;oBACH,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,yBAAyB,UAAU,EAAE,CAAC;gBAC/C,CAAC;gBACD,OAAO,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;gBAExC,mBAAmB;gBACnB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;gBAEhE,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,CAAC,MAAM,wBAAwB,EAAE,CAAC,IAAI,SAAS,CAAC;oBACnE,MAAM,SAAS,GAA2C;wBACxD,MAAM;wBACN,OAAO;wBACP,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC;oBACF,IAAI,UAAU;wBAAE,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC;oBAClD,IAAI,YAAY,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,SAAS,CAAC,IAAI,GAAG,YAAY,CAAC;wBAC9B,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;4BACzD,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;wBAC/C,CAAC;oBACH,CAAC;oBAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;oBACrD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAEvC,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;wBAClD,MAAM,WAAW,GAAG,QAAQ;4BAC1B,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,IAAI;4BACrC,CAAC,CAAC,IAAI,CAAC;wBACT,IACE,WAAW;4BACX,CAAC,MAAM,4BAA4B,CAAC,WAAW,CAAC,CAAC,EACjD,CAAC;4BACD,OAAO,+CAA+C,CAAC;wBACzD,CAAC;wBACD,IAAI,WAAW,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;4BACjE,IAAI,CAAC,OAAO,EAAE,CAAC;gCACb,OAAO,+DAA+D,CAAC;4BACzE,CAAC;wBACH,CAAC;wBACD,OAAO,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,iBACnD,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,QAC1D,EAAE,CAAC;oBACL,CAAC;oBAED,uEAAuE;oBACvE,MAAM,cAAc,GAClB,OAAQ,IAAgC,CAAC,UAAU,KAAK,QAAQ;wBAC9D,CAAC,CAAG,IAAgC,CAAC,UAAqB,CAAC,IAAI,EAAE;wBACjE,CAAC,CAAC,EAAE,CAAC;oBAET,IAAI,IAAY,CAAC;oBACjB,IAAI,CAAC;wBACH,+EAA+E;wBAC/E,MAAM,SAAS,GAAG,cAAc;4BAC9B,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,IAAI;4BAClB,CAAC,CAAC,iCAAiC,CAAC;wBACtC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;wBACpE,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,gCAAgC,CAAC;oBAC1C,CAAC;oBACD,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;oBAExC,YAAY;oBACZ,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,MAAM,QAAQ,CAAC,MAAM,KAAK,OAAO,aAAa,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG,CACjH,CAAC;oBAEF,uEAAuE;oBACvE,IAAI,cAAc,EAAE,CAAC;wBACnB,IAAI,CAAC;4BACH,MAAM,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,GAClD,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;4BAC9C,MAAM,EAAE,eAAe,EAAE,mBAAmB,EAAE,GAC5C,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;4BAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;4BAChC,MAAM,KAAK,GAAG,mBAAmB,EAAE,CAAC;4BACpC,MAAM,KAAK,GAAG,KAAK;gCACjB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAc,EAAE,OAAO,EAAE,KAAK,EAAE;gCAC3C,CAAC,CAAC,KAAK;oCACL,CAAC,CAAC,EAAE,KAAK,EAAE,MAAe,EAAE,OAAO,EAAE,KAAK,EAAE;oCAC5C,CAAC,CAAC,IAAI,CAAC;4BACX,IAAI,CAAC,KAAK;gCACR,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;4BAC7D,MAAM,WAAW,GACf,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;gCAC1D,YAAY,CAAC;4BACf,MAAM,kBAAkB,CACtB,KAAK,EACL,cAAc,EACd,IAAI,EACJ,WAAW,EACX;gCACE,YAAY,EAAE,sBAAsB;6BACrC,CACF,CAAC;4BACF,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;4BAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;4BACpC,OAAO,IAAI,CAAC,SAAS,CAAC;gCACpB,WAAW,EAAE,IAAI;gCACjB,OAAO,EAAE,cAAc;gCACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;gCACvB,KAAK;gCACL,WAAW;gCACX,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO;6BAChE,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,OAAY,EAAE,CAAC;4BACtB,OAAO,qBAAqB,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;wBACpJ,CAAC;oBACH,CAAC;oBAED,kEAAkE;oBAClE,iEAAiE;oBACjE,kEAAkE;oBAClE,iEAAiE;oBACjE,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;wBAC3B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,mBAAmB,CAAC;oBACvD,CAAC;oBAED,OAAO,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,OAAO,IAAI,EAAE,CAAC;gBACrE,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBACvC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;wBAC/B,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,eAAe,OAAO,KAAK,CAC5D,CAAC;wBACF,OAAO,2BAA2B,SAAS,KAAK,CAAC;oBACnD,CAAC;oBACD,MAAM,OAAO,GAAG,aAAa,CAC3B,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAC3B,YAAY,CACb,CAAC;oBACF,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,aAAa,OAAO,KAAK,OAAO,KAAK,CACtE,CAAC;oBACF,OAAO,mBAAmB,OAAO,EAAE,CAAC;gBACtC,CAAC;wBAAS,CAAC;oBACT,YAAY,CAAC,OAAO,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,QAAQ,EAAE,IAAI;SACf;KACF,CAAC;AACJ,CAAC","sourcesContent":["/**\n * Fetch tool — outbound HTTP for automations and agent use.\n *\n * NOTE: this is an *agent* tool (LLM function call), not an *extension* (the\n * sandboxed Alpine.js mini-app primitive). It lives in this directory because\n * it shares SSRF-safe URL/proxy helpers with the extension iframe proxy.\n *\n * Supports ${keys.NAME} reference substitution in URL, headers, and body.\n * Values are resolved server-side AFTER the model emits the tool call —\n * the raw secret never enters the model's context.\n */\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport {\n  collectSecretValues,\n  MAX_EXTENSION_PROXY_RESPONSE_SIZE,\n  normalizeExtensionProxyMethod,\n  readResponseTextWithLimit,\n  redactSecrets,\n  redactString,\n  sanitizeOutboundHeaders,\n} from \"./proxy-security.js\";\nimport {\n  createSsrfSafeDispatcher,\n  isBlockedExtensionUrlWithDns,\n} from \"./url-safety.js\";\n\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/**\n * Headers that mimic a current Chrome on macOS so anti-bot middleware (Cloudflare,\n * PerimeterX, Akamai) treats the request as a real user. We only fill in fields\n * the caller hasn't supplied — explicit headers (e.g. an `Authorization` header\n * for an API call) always win.\n *\n * `Accept-Encoding` deliberately omits `zstd` because Node's undici fetch only\n * decompresses `gzip`, `deflate`, and `br`. Advertising `zstd` would let some\n * servers send bytes we can't decode.\n */\nconst BROWSER_DEFAULT_HEADERS: Record<string, string> = {\n  \"User-Agent\":\n    \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36\",\n  Accept:\n    \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n  \"Accept-Language\": \"en-US,en;q=0.9\",\n  \"Accept-Encoding\": \"gzip, deflate, br\",\n  \"Sec-Ch-Ua\":\n    '\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"',\n  \"Sec-Ch-Ua-Mobile\": \"?0\",\n  \"Sec-Ch-Ua-Platform\": '\"macOS\"',\n  \"Sec-Fetch-Dest\": \"document\",\n  \"Sec-Fetch-Mode\": \"navigate\",\n  \"Sec-Fetch-Site\": \"none\",\n  \"Sec-Fetch-User\": \"?1\",\n  \"Upgrade-Insecure-Requests\": \"1\",\n};\n\nfunction applyBrowserDefaults(\n  headers: Record<string, string>,\n): Record<string, string> {\n  const seen = new Set(Object.keys(headers).map((k) => k.toLowerCase()));\n  const merged = { ...headers };\n  for (const [name, value] of Object.entries(BROWSER_DEFAULT_HEADERS)) {\n    if (!seen.has(name.toLowerCase())) merged[name] = value;\n  }\n  return merged;\n}\n\nexport interface FetchToolOptions {\n  /** Resolve ${keys.NAME} references. Injected by the plugin at setup time. */\n  resolveKeys?: (text: string) => Promise<{\n    resolved: string;\n    usedKeys: string[];\n    secretValues?: string[];\n  }>;\n  /** Validate URL against per-key allowlists. */\n  validateUrl?: (url: string, usedKeys: string[]) => Promise<boolean>;\n}\n\n/**\n * Create the fetch tool entry for the agent tool registry.\n */\nexport function createFetchToolEntry(\n  opts: FetchToolOptions = {},\n): Record<string, ActionEntry> {\n  return {\n    \"web-request\": {\n      tool: {\n        description: `Make an outbound HTTP request to any EXTERNAL URL — APIs, webhooks, and arbitrary web pages (HTML, RSS, JSON, etc.). Use this to fetch the contents of a URL the user pastes in chat. Sends realistic Chrome-on-macOS headers by default (User-Agent, Accept, Sec-Fetch-*) so most sites that block obvious bots will respond normally; pass an explicit header to override any default. Supports \\${keys.NAME} placeholders in url, headers, and body — these are resolved server-side from the user's saved keys (the raw value never enters your context). Example: \\${keys.SLACK_WEBHOOK} in the url field. IMPORTANT: Never use this to call internal /_agent-native/ endpoints or localhost action URLs — use the registered actions directly (e.g. \\`search-records\\`, \\`provider-api-request\\`, \\`update-resource\\`). Actions are already available as native tools; calling them via HTTP is slower and bypasses validation.`,\n        parameters: {\n          type: \"object\" as const,\n          properties: {\n            url: {\n              type: \"string\",\n              description:\n                'Full URL. May contain ${keys.NAME} references, e.g. \"${keys.SLACK_WEBHOOK}\".',\n            },\n            method: {\n              type: \"string\",\n              description: \"HTTP method. Default: GET.\",\n              enum: [\"GET\", \"POST\", \"PUT\", \"PATCH\", \"DELETE\", \"HEAD\"],\n            },\n            headers: {\n              type: \"string\",\n              description:\n                'JSON object of headers. May contain ${keys.NAME} references. Example: \\'{\"Authorization\": \"Bearer ${keys.API_TOKEN}\"}\\'.',\n            },\n            body: {\n              type: \"string\",\n              description:\n                \"Request body (for POST/PUT/PATCH). May contain ${keys.NAME} references.\",\n            },\n            timeout_ms: {\n              type: \"number\",\n              description: `Timeout in milliseconds. Default: ${DEFAULT_TIMEOUT_MS}. Max: 30000.`,\n            },\n            maxChars: {\n              type: \"number\",\n              description:\n                \"Maximum response body characters to return. Default: 32000. Max: 200000. Increase when you need to read a large document, API response, or dataset.\",\n            },\n            saveToFile: {\n              type: \"string\",\n              description:\n                \"Workspace file path to save the full response body to instead of returning it in context (e.g. 'analysis/page.html'). When set, returns only a compact summary {savedTo, status, bytes, preview}. Useful for large web pages or API responses that would overflow context.\",\n            },\n          },\n          required: [\"url\"],\n        },\n      },\n      run: async (args: Record<string, string>) => {\n        const startTime = Date.now();\n        const rawUrl = args.url;\n        const method = normalizeExtensionProxyMethod(args.method || \"GET\");\n        if (!method) {\n          return \"Unsupported HTTP method. Allowed methods: GET, POST, PUT, PATCH, DELETE, HEAD.\";\n        }\n        const rawHeaders = args.headers || \"{}\";\n        const rawBody = args.body;\n        const timeoutMs = Math.min(\n          Number(args.timeout_ms) || DEFAULT_TIMEOUT_MS,\n          30_000,\n        );\n        const requestedMaxChars = Number(args.maxChars);\n        const maxChars =\n          Number.isFinite(requestedMaxChars) && requestedMaxChars > 0\n            ? Math.min(requestedMaxChars, 200_000)\n            : 32_000;\n\n        // Resolve key references\n        let resolvedUrl = rawUrl;\n        let resolvedHeaders = rawHeaders;\n        let resolvedBody = rawBody;\n        const allUsedKeys: string[] = [];\n        const allSecretValues: string[] = [];\n\n        if (opts.resolveKeys) {\n          try {\n            const urlResult = await opts.resolveKeys(rawUrl);\n            resolvedUrl = urlResult.resolved;\n            allUsedKeys.push(...urlResult.usedKeys);\n            allSecretValues.push(...(urlResult.secretValues ?? []));\n\n            const headerResult = await opts.resolveKeys(rawHeaders);\n            resolvedHeaders = headerResult.resolved;\n            allUsedKeys.push(...headerResult.usedKeys);\n            allSecretValues.push(...(headerResult.secretValues ?? []));\n\n            if (rawBody) {\n              const bodyResult = await opts.resolveKeys(rawBody);\n              resolvedBody = bodyResult.resolved;\n              allUsedKeys.push(...bodyResult.usedKeys);\n              allSecretValues.push(...(bodyResult.secretValues ?? []));\n            }\n          } catch (err: any) {\n            return `Error resolving key references: ${err?.message ?? err}`;\n          }\n        }\n        const secretValues = collectSecretValues(allSecretValues);\n\n        // Block SSRF targets regardless of key usage\n        if (await isBlockedExtensionUrlWithDns(resolvedUrl)) {\n          return `Requests to private/internal addresses are not allowed: \"${rawUrl}\".`;\n        }\n\n        // Validate URL against per-key allowlists\n        if (opts.validateUrl && allUsedKeys.length > 0) {\n          try {\n            const allowed = await opts.validateUrl(resolvedUrl, allUsedKeys);\n            if (!allowed) {\n              return `URL \"${rawUrl}\" is not in the allowlist for the referenced keys. Check your key settings.`;\n            }\n          } catch (err: any) {\n            return `URL validation error: ${err?.message ?? err}`;\n          }\n        }\n\n        // Parse headers, then merge in browser-like defaults for any header the\n        // caller didn't already specify. Real-browser headers (User-Agent,\n        // Accept, Sec-Fetch-*) are what gets you past Cloudflare / PerimeterX /\n        // generic UA-sniffing middleware on sites the user pastes in chat;\n        // explicit caller headers always win so API calls keep their auth\n        // headers untouched.\n        let headers: Record<string, string>;\n        try {\n          headers = sanitizeOutboundHeaders(JSON.parse(resolvedHeaders));\n        } catch {\n          return `Invalid headers JSON: ${rawHeaders}`;\n        }\n        headers = applyBrowserDefaults(headers);\n\n        // Make the request\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), timeoutMs);\n\n        try {\n          const dispatcher = (await createSsrfSafeDispatcher()) ?? undefined;\n          const fetchOpts: RequestInit & { dispatcher?: unknown } = {\n            method,\n            headers,\n            signal: controller.signal,\n            redirect: \"manual\",\n          };\n          if (dispatcher) fetchOpts.dispatcher = dispatcher;\n          if (resolvedBody && [\"POST\", \"PUT\", \"PATCH\"].includes(method)) {\n            fetchOpts.body = resolvedBody;\n            if (!headers[\"content-type\"] && !headers[\"Content-Type\"]) {\n              headers[\"Content-Type\"] = \"application/json\";\n            }\n          }\n\n          const response = await fetch(resolvedUrl, fetchOpts);\n          const elapsed = Date.now() - startTime;\n\n          if (response.status >= 300 && response.status < 400) {\n            const location = response.headers.get(\"location\");\n            const redirectUrl = location\n              ? new URL(location, resolvedUrl).href\n              : null;\n            if (\n              redirectUrl &&\n              (await isBlockedExtensionUrlWithDns(redirectUrl))\n            ) {\n              return \"Redirect to private/internal address blocked.\";\n            }\n            if (redirectUrl && opts.validateUrl && allUsedKeys.length > 0) {\n              const allowed = await opts.validateUrl(redirectUrl, allUsedKeys);\n              if (!allowed) {\n                return \"Redirect URL is not in the allowlist for the referenced keys.\";\n              }\n            }\n            return `HTTP ${response.status} ${response.statusText}\\n\\nRedirect: ${\n              redirectUrl ? redactString(redirectUrl, secretValues) : \"(none)\"\n            }`;\n          }\n\n          // Check if caller wants to save to workspace file (before truncation).\n          const saveToFilePath =\n            typeof (args as Record<string, unknown>).saveToFile === \"string\"\n              ? ((args as Record<string, unknown>).saveToFile as string).trim()\n              : \"\";\n\n          let body: string;\n          try {\n            // When saving to file allow larger reads (20MB), otherwise cap at proxy limit.\n            const readLimit = saveToFilePath\n              ? 20 * 1024 * 1024\n              : MAX_EXTENSION_PROXY_RESPONSE_SIZE;\n            const result = await readResponseTextWithLimit(response, readLimit);\n            body = result.text;\n          } catch {\n            body = \"(could not read response body)\";\n          }\n          body = redactString(body, secretValues);\n\n          // Audit log\n          console.log(\n            `[fetch-tool] ${method} ${rawUrl} → ${response.status} (${elapsed}ms, keys: ${allUsedKeys.join(\",\") || \"none\"})`,\n          );\n\n          // saveToFile: write full body to workspace and return compact summary.\n          if (saveToFilePath) {\n            try {\n              const { writeWorkspaceFile, SAVE_TO_FILE_MAX_BYTES } =\n                await import(\"../workspace-files/store.js\");\n              const { getRequestOrgId, getRequestUserEmail } =\n                await import(\"../server/request-context.js\");\n              const orgId = getRequestOrgId();\n              const email = getRequestUserEmail();\n              const scope = orgId\n                ? { scope: \"org\" as const, scopeId: orgId }\n                : email\n                  ? { scope: \"user\" as const, scopeId: email }\n                  : null;\n              if (!scope)\n                throw new Error(\"No authenticated context for saveToFile\");\n              const contentType =\n                response.headers.get(\"content-type\")?.split(\";\")[0].trim() ??\n                \"text/plain\";\n              await writeWorkspaceFile(\n                scope,\n                saveToFilePath,\n                body,\n                contentType,\n                {\n                  maxFileBytes: SAVE_TO_FILE_MAX_BYTES,\n                },\n              );\n              const bytes = Buffer.byteLength(body, \"utf8\");\n              const preview = body.slice(0, 2000);\n              return JSON.stringify({\n                savedToFile: true,\n                savedTo: saveToFilePath,\n                status: response.status,\n                bytes,\n                contentType,\n                preview: preview.length < body.length ? `${preview}…` : preview,\n              });\n            } catch (saveErr: any) {\n              return `saveToFile error: ${saveErr?.message ?? String(saveErr)}\\n\\nHTTP ${response.status} ${response.statusText}\\n\\n${body.slice(0, maxChars)}`;\n            }\n          }\n\n          // Truncate very long responses for the agent. Default cap is 32 k\n          // chars (~8 k tokens), enough to read a full article or scrape a\n          // stats table without blowing out the model's context window. The\n          // caller may request up to 200 000 chars via the maxChars input.\n          if (body.length > maxChars) {\n            body = body.slice(0, maxChars) + \"\\n... (truncated)\";\n          }\n\n          return `HTTP ${response.status} ${response.statusText}\\n\\n${body}`;\n        } catch (err: any) {\n          const elapsed = Date.now() - startTime;\n          if (err?.name === \"AbortError\") {\n            console.log(\n              `[fetch-tool] ${method} ${rawUrl} → TIMEOUT (${elapsed}ms)`,\n            );\n            return `Request timed out after ${timeoutMs}ms.`;\n          }\n          const message = redactSecrets(\n            err?.message ?? String(err),\n            secretValues,\n          );\n          console.log(\n            `[fetch-tool] ${method} ${rawUrl} → ERROR: ${message} (${elapsed}ms)`,\n          );\n          return `Request failed: ${message}`;\n        } finally {\n          clearTimeout(timeout);\n        }\n      },\n      readOnly: true,\n    },\n  };\n}\n"]}
+{"version":3,"file":"fetch-tool.js","sourceRoot":"","sources":["../../src/extensions/fetch-tool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACL,mBAAmB,EACnB,iCAAiC,EACjC,6BAA6B,EAC7B,yBAAyB,EACzB,aAAa,EACb,YAAY,EACZ,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EACL,wBAAwB,EACxB,4BAA4B,GAC7B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,sBAAsB,EACtB,4BAA4B,EAC5B,iBAAiB,GAClB,MAAM,kBAAkB,CAAC;AAE1B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC;;;;;;;;;GASG;AACH,MAAM,uBAAuB,GAA2B;IACtD,YAAY,EACV,uHAAuH;IACzH,MAAM,EACJ,kGAAkG;IACpG,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,WAAW,EACT,mEAAmE;IACrE,kBAAkB,EAAE,IAAI;IACxB,oBAAoB,EAAE,SAAS;IAC/B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,UAAU;IAC5B,gBAAgB,EAAE,MAAM;IACxB,gBAAgB,EAAE,IAAI;IACtB,2BAA2B,EAAE,GAAG;CACjC,CAAC;AAEF,SAAS,oBAAoB,CAC3B,OAA+B;IAE/B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAAE,CAAC;QACpE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAyB,EAAE;IAE3B,OAAO;QACL,aAAa,EAAE;YACb,IAAI,EAAE;gBACJ,WAAW,EAAE,u4BAAu4B;gBACp5B,UAAU,EAAE;oBACV,IAAI,EAAE,QAAiB;oBACvB,UAAU,EAAE;wBACV,GAAG,EAAE;4BACH,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,8EAA8E;yBACjF;wBACD,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,4BAA4B;4BACzC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;yBACxD;wBACD,OAAO,EAAE;4BACP,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,0HAA0H;yBAC7H;wBACD,IAAI,EAAE;4BACJ,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,yEAAyE;yBAC5E;wBACD,UAAU,EAAE;4BACV,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,qCAAqC,kBAAkB,eAAe;yBACpF;wBACD,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,qJAAqJ;yBACxJ;wBACD,YAAY,EAAE;4BACZ,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,qPAAqP;4BACvP,IAAI,EAAE;gCACJ,MAAM;gCACN,KAAK;gCACL,MAAM;gCACN,UAAU;gCACV,OAAO;gCACP,UAAU;gCACV,SAAS;6BACV;yBACF;wBACD,OAAO,EAAE;4BACP,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,4IAA4I;4BAC9I,IAAI,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,MAAM,CAAC;yBAC7C;wBACD,YAAY,EAAE;4BACZ,IAAI,EAAE,SAAS;4BACf,WAAW,EACT,0GAA0G;yBAC7G;wBACD,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,8QAA8Q;4BAChR,UAAU,EAAE;gCACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gCACrD,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;gCACnD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCACzB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCAC9B,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE;gCACtD,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCAC9B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gCAChC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;6BACnC;yBACK;wBACR,UAAU,EAAE;4BACV,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,4QAA4Q;yBAC/Q;qBACF;oBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;iBAClB;aACF;YACD,GAAG,EAAE,KAAK,EAAE,IAA6B,EAAE,EAAE;gBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;gBACtC,MAAM,MAAM,GAAG,6BAA6B,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC;gBACnE,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,OAAO,gFAAgF,CAAC;gBAC1F,CAAC;gBACD,MAAM,UAAU,GACd,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;oBAC9B,CAAC,CAAC,IAAI,CAAC,OAAO;oBACd,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;gBACzC,MAAM,OAAO,GACX,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;oBAC3B,CAAC,CAAC,IAAI,CAAC,IAAI;oBACX,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI;wBAC7C,CAAC,CAAC,SAAS;wBACX,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,kBAAkB,EAC7C,MAAM,CACP,CAAC;gBACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC;oBACzD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,OAAO,CAAC;oBACtC,CAAC,CAAC,MAAM,CAAC;gBAEb,yBAAyB;gBACzB,IAAI,WAAW,GAAG,MAAM,CAAC;gBACzB,IAAI,eAAe,GAAG,UAAU,CAAC;gBACjC,IAAI,YAAY,GAAG,OAAO,CAAC;gBAC3B,MAAM,WAAW,GAAa,EAAE,CAAC;gBACjC,MAAM,eAAe,GAAa,EAAE,CAAC;gBAErC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;wBACjD,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC;wBACjC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;wBACxC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAExD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;wBACxD,eAAe,GAAG,YAAY,CAAC,QAAQ,CAAC;wBACxC,WAAW,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;wBAC3C,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAE3D,IAAI,OAAO,EAAE,CAAC;4BACZ,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;4BACnD,YAAY,GAAG,UAAU,CAAC,QAAQ,CAAC;4BACnC,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;4BACzC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;wBAC3D,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAClB,OAAO,mCAAmC,GAAG,EAAE,OAAO,IAAI,GAAG,EAAE,CAAC;oBAClE,CAAC;gBACH,CAAC;gBACD,MAAM,YAAY,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;gBAE1D,6CAA6C;gBAC7C,IAAI,MAAM,4BAA4B,CAAC,WAAW,CAAC,EAAE,CAAC;oBACpD,OAAO,4DAA4D,MAAM,IAAI,CAAC;gBAChF,CAAC;gBAED,0CAA0C;gBAC1C,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/C,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;wBACjE,IAAI,CAAC,OAAO,EAAE,CAAC;4BACb,OAAO,QAAQ,MAAM,6EAA6E,CAAC;wBACrG,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAClB,OAAO,yBAAyB,GAAG,EAAE,OAAO,IAAI,GAAG,EAAE,CAAC;oBACxD,CAAC;gBACH,CAAC;gBAED,wEAAwE;gBACxE,mEAAmE;gBACnE,wEAAwE;gBACxE,mEAAmE;gBACnE,kEAAkE;gBAClE,qBAAqB;gBACrB,IAAI,OAA+B,CAAC;gBACpC,IAAI,CAAC;oBACH,OAAO,GAAG,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,yBAAyB,UAAU,EAAE,CAAC;gBAC/C,CAAC;gBACD,OAAO,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;gBAExC,mBAAmB;gBACnB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;gBAEhE,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,CAAC,MAAM,wBAAwB,EAAE,CAAC,IAAI,SAAS,CAAC;oBACnE,MAAM,SAAS,GAA2C;wBACxD,MAAM;wBACN,OAAO;wBACP,MAAM,EAAE,UAAU,CAAC,MAAM;wBACzB,QAAQ,EAAE,QAAQ;qBACnB,CAAC;oBACF,IAAI,UAAU;wBAAE,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC;oBAClD,IAAI,YAAY,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC9D,SAAS,CAAC,IAAI,GAAG,YAAY,CAAC;wBAC9B,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;4BACzD,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;wBAC/C,CAAC;oBACH,CAAC;oBAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;oBACrD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAEvC,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;wBAClD,MAAM,WAAW,GAAG,QAAQ;4BAC1B,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,IAAI;4BACrC,CAAC,CAAC,IAAI,CAAC;wBACT,IACE,WAAW;4BACX,CAAC,MAAM,4BAA4B,CAAC,WAAW,CAAC,CAAC,EACjD,CAAC;4BACD,OAAO,+CAA+C,CAAC;wBACzD,CAAC;wBACD,IAAI,WAAW,IAAI,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC9D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;4BACjE,IAAI,CAAC,OAAO,EAAE,CAAC;gCACb,OAAO,+DAA+D,CAAC;4BACzE,CAAC;wBACH,CAAC;wBACD,OAAO,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,iBACnD,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,QAC1D,EAAE,CAAC;oBACL,CAAC;oBAED,uEAAuE;oBACvE,MAAM,cAAc,GAClB,OAAQ,IAAgC,CAAC,UAAU,KAAK,QAAQ;wBAC9D,CAAC,CAAG,IAAgC,CAAC,UAAqB,CAAC,IAAI,EAAE;wBACjE,CAAC,CAAC,EAAE,CAAC;oBAET,IAAI,IAAY,CAAC;oBACjB,IAAI,CAAC;wBACH,+EAA+E;wBAC/E,MAAM,SAAS,GAAG,cAAc;4BAC9B,CAAC,CAAC,EAAE,GAAG,IAAI,GAAG,IAAI;4BAClB,CAAC,CAAC,iCAAiC,CAAC;wBACtC,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;wBACpE,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC;wBACP,IAAI,GAAG,gCAAgC,CAAC;oBAC1C,CAAC;oBACD,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;oBACxC,MAAM,WAAW,GACf,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;wBAC1D,YAAY,CAAC;oBACf,IAAI,WAAmB,CAAC;oBACxB,IAAI,aAAa,GAAG,KAAK,CAAC;oBAC1B,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,iBAAiB,CAAC;4BAClC,GAAG,EAAE,WAAW;4BAChB,IAAI;4BACJ,WAAW;4BACX,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC;4BACjD,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC;4BAC9C,YAAY,EACV,IAAI,CAAC,YAAY,KAAK,SAAS;gCAC7B,CAAC,CAAC,IAAI;gCACN,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC;4BACxC,MAAM,EAAE,4BAA4B,CAAC,IAAI,CAAC,MAAM,CAAC;4BACjD,QAAQ;yBACT,CAAC,CAAC;wBACH,aAAa,GAAG,SAAS,CAAC,IAAI,CAAC;wBAC/B,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,CAAC;oBAClD,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAClB,OAAO,sCAAsC,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC7E,CAAC;oBAED,YAAY;oBACZ,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,MAAM,QAAQ,CAAC,MAAM,KAAK,OAAO,aAAa,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG,CACjH,CAAC;oBAEF,uEAAuE;oBACvE,IAAI,cAAc,EAAE,CAAC;wBACnB,IAAI,CAAC;4BACH,MAAM,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,GAClD,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;4BAC9C,MAAM,EAAE,eAAe,EAAE,mBAAmB,EAAE,GAC5C,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;4BAC/C,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;4BAChC,MAAM,KAAK,GAAG,mBAAmB,EAAE,CAAC;4BACpC,MAAM,KAAK,GAAG,KAAK;gCACjB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAc,EAAE,OAAO,EAAE,KAAK,EAAE;gCAC3C,CAAC,CAAC,KAAK;oCACL,CAAC,CAAC,EAAE,KAAK,EAAE,MAAe,EAAE,OAAO,EAAE,KAAK,EAAE;oCAC5C,CAAC,CAAC,IAAI,CAAC;4BACX,IAAI,CAAC,KAAK;gCACR,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;4BAC7D,MAAM,kBAAkB,CACtB,KAAK,EACL,cAAc,EACd,IAAI,EACJ,WAAW,EACX;gCACE,YAAY,EAAE,sBAAsB;6BACrC,CACF,CAAC;4BACF,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;4BAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;4BAC3C,OAAO,IAAI,CAAC,SAAS,CAAC;gCACpB,WAAW,EAAE,IAAI;gCACjB,OAAO,EAAE,cAAc;gCACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;gCACvB,KAAK;gCACL,WAAW;gCACX,YAAY,EAAE,aAAa;gCAC3B,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO;6BAChE,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,OAAY,EAAE,CAAC;4BACtB,OAAO,qBAAqB,OAAO,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;wBACpJ,CAAC;oBACH,CAAC;oBAED,OAAO,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,OAAO,WAAW,EAAE,CAAC;gBAC5E,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBACvC,IAAI,GAAG,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;wBAC/B,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,eAAe,OAAO,KAAK,CAC5D,CAAC;wBACF,OAAO,2BAA2B,SAAS,KAAK,CAAC;oBACnD,CAAC;oBACD,MAAM,OAAO,GAAG,aAAa,CAC3B,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAC3B,YAAY,CACb,CAAC;oBACF,OAAO,CAAC,GAAG,CACT,gBAAgB,MAAM,IAAI,MAAM,aAAa,OAAO,KAAK,OAAO,KAAK,CACtE,CAAC;oBACF,OAAO,mBAAmB,OAAO,EAAE,CAAC;gBACtC,CAAC;wBAAS,CAAC;oBACT,YAAY,CAAC,OAAO,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,QAAQ,EAAE,IAAI;SACf;KACF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACtD,OAAO,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,KAAK,CAAC;AAC7E,CAAC","sourcesContent":["/**\n * Fetch tool — outbound HTTP for automations and agent use.\n *\n * NOTE: this is an *agent* tool (LLM function call), not an *extension* (the\n * sandboxed Alpine.js mini-app primitive). It lives in this directory because\n * it shares SSRF-safe URL/proxy helpers with the extension iframe proxy.\n *\n * Supports ${keys.NAME} reference substitution in URL, headers, and body.\n * Values are resolved server-side AFTER the model emits the tool call —\n * the raw secret never enters the model's context.\n */\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport {\n  collectSecretValues,\n  MAX_EXTENSION_PROXY_RESPONSE_SIZE,\n  normalizeExtensionProxyMethod,\n  readResponseTextWithLimit,\n  redactSecrets,\n  redactString,\n  sanitizeOutboundHeaders,\n} from \"./proxy-security.js\";\nimport {\n  createSsrfSafeDispatcher,\n  isBlockedExtensionUrlWithDns,\n} from \"./url-safety.js\";\nimport {\n  formatWebContentResult,\n  parseWebContentSearchOptions,\n  processWebContent,\n} from \"./web-content.js\";\n\nconst DEFAULT_TIMEOUT_MS = 15_000;\n\n/**\n * Headers that mimic a current Chrome on macOS so anti-bot middleware (Cloudflare,\n * PerimeterX, Akamai) treats the request as a real user. We only fill in fields\n * the caller hasn't supplied — explicit headers (e.g. an `Authorization` header\n * for an API call) always win.\n *\n * `Accept-Encoding` deliberately omits `zstd` because Node's undici fetch only\n * decompresses `gzip`, `deflate`, and `br`. Advertising `zstd` would let some\n * servers send bytes we can't decode.\n */\nconst BROWSER_DEFAULT_HEADERS: Record<string, string> = {\n  \"User-Agent\":\n    \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36\",\n  Accept:\n    \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8\",\n  \"Accept-Language\": \"en-US,en;q=0.9\",\n  \"Accept-Encoding\": \"gzip, deflate, br\",\n  \"Sec-Ch-Ua\":\n    '\"Google Chrome\";v=\"131\", \"Chromium\";v=\"131\", \"Not_A Brand\";v=\"24\"',\n  \"Sec-Ch-Ua-Mobile\": \"?0\",\n  \"Sec-Ch-Ua-Platform\": '\"macOS\"',\n  \"Sec-Fetch-Dest\": \"document\",\n  \"Sec-Fetch-Mode\": \"navigate\",\n  \"Sec-Fetch-Site\": \"none\",\n  \"Sec-Fetch-User\": \"?1\",\n  \"Upgrade-Insecure-Requests\": \"1\",\n};\n\nfunction applyBrowserDefaults(\n  headers: Record<string, string>,\n): Record<string, string> {\n  const seen = new Set(Object.keys(headers).map((k) => k.toLowerCase()));\n  const merged = { ...headers };\n  for (const [name, value] of Object.entries(BROWSER_DEFAULT_HEADERS)) {\n    if (!seen.has(name.toLowerCase())) merged[name] = value;\n  }\n  return merged;\n}\n\nexport interface FetchToolOptions {\n  /** Resolve ${keys.NAME} references. Injected by the plugin at setup time. */\n  resolveKeys?: (text: string) => Promise<{\n    resolved: string;\n    usedKeys: string[];\n    secretValues?: string[];\n  }>;\n  /** Validate URL against per-key allowlists. */\n  validateUrl?: (url: string, usedKeys: string[]) => Promise<boolean>;\n}\n\n/**\n * Create the fetch tool entry for the agent tool registry.\n */\nexport function createFetchToolEntry(\n  opts: FetchToolOptions = {},\n): Record<string, ActionEntry> {\n  return {\n    \"web-request\": {\n      tool: {\n        description: `Make an outbound HTTP request to any EXTERNAL URL — APIs, webhooks, and arbitrary web pages (HTML, RSS, JSON, etc.). Use this to fetch the contents of a URL the user pastes in chat. Sends realistic Chrome-on-macOS headers by default (User-Agent, Accept, Sec-Fetch-*) so most sites that block obvious bots will respond normally; pass an explicit header to override any default. Supports \\${keys.NAME} placeholders in url, headers, and body — these are resolved server-side from the user's saved keys (the raw value never enters your context). Example: \\${keys.SLACK_WEBHOOK} in the url field. IMPORTANT: Never use this to call internal /_agent-native/ endpoints or localhost action URLs — use the registered actions directly (e.g. \\`search-records\\`, \\`provider-api-request\\`, \\`update-resource\\`). Actions are already available as native tools; calling them via HTTP is slower and bypasses validation.`,\n        parameters: {\n          type: \"object\" as const,\n          properties: {\n            url: {\n              type: \"string\",\n              description:\n                'Full URL. May contain ${keys.NAME} references, e.g. \"${keys.SLACK_WEBHOOK}\".',\n            },\n            method: {\n              type: \"string\",\n              description: \"HTTP method. Default: GET.\",\n              enum: [\"GET\", \"POST\", \"PUT\", \"PATCH\", \"DELETE\", \"HEAD\"],\n            },\n            headers: {\n              type: \"string\",\n              description:\n                'JSON object of headers. May contain ${keys.NAME} references. Example: \\'{\"Authorization\": \"Bearer ${keys.API_TOKEN}\"}\\'.',\n            },\n            body: {\n              type: \"string\",\n              description:\n                \"Request body (for POST/PUT/PATCH). May contain ${keys.NAME} references.\",\n            },\n            timeout_ms: {\n              type: \"number\",\n              description: `Timeout in milliseconds. Default: ${DEFAULT_TIMEOUT_MS}. Max: 30000.`,\n            },\n            maxChars: {\n              type: \"number\",\n              description:\n                \"Maximum response body characters to return. Default: 32000. Max: 200000. Increase when you need to read a large document, API response, or dataset.\",\n            },\n            responseMode: {\n              type: \"string\",\n              description:\n                \"How to return the response. Default: auto (HTML pages become clean markdown; JSON/text stays raw). Use raw for exact bytes, markdown/text for extracted readable content, links for just links, metadata for page metadata, or matches with search.\",\n              enum: [\n                \"auto\",\n                \"raw\",\n                \"text\",\n                \"markdown\",\n                \"links\",\n                \"metadata\",\n                \"matches\",\n              ],\n            },\n            extract: {\n              type: \"string\",\n              description:\n                \"HTML extraction strategy. Default: readability. Use all-visible for visible body text/markdown, or none to convert the full HTML document.\",\n              enum: [\"readability\", \"all-visible\", \"none\"],\n            },\n            includeLinks: {\n              type: \"boolean\",\n              description:\n                \"Whether extracted HTML responses should include a compact links list. Default: true for extracted pages.\",\n            },\n            search: {\n              type: \"object\",\n              description:\n                \"Optional post-fetch search over extracted content by default. Supports {query, queries, terms, regex, regexFlags, source:'extracted'|'raw', maxMatches, contextChars, caseSensitive}. Regex is safety-checked and bounded; prefer query/terms for simple grep-like searches.\",\n              properties: {\n                query: { type: \"string\" },\n                queries: { type: \"array\", items: { type: \"string\" } },\n                terms: { type: \"array\", items: { type: \"string\" } },\n                regex: { type: \"string\" },\n                regexFlags: { type: \"string\" },\n                source: { type: \"string\", enum: [\"extracted\", \"raw\"] },\n                maxMatches: { type: \"number\" },\n                contextChars: { type: \"number\" },\n                caseSensitive: { type: \"boolean\" },\n              },\n            } as any,\n            saveToFile: {\n              type: \"string\",\n              description:\n                \"Workspace file path to save the full response body to instead of returning it in context (e.g. 'analysis/page.html'). When set, returns only a compact summary {savedTo, status, bytes, preview}. Useful for large web pages or API responses that would overflow context.\",\n            },\n          },\n          required: [\"url\"],\n        },\n      },\n      run: async (args: Record<string, unknown>) => {\n        const startTime = Date.now();\n        const rawUrl = String(args.url ?? \"\");\n        const method = normalizeExtensionProxyMethod(args.method || \"GET\");\n        if (!method) {\n          return \"Unsupported HTTP method. Allowed methods: GET, POST, PUT, PATCH, DELETE, HEAD.\";\n        }\n        const rawHeaders =\n          typeof args.headers === \"string\"\n            ? args.headers\n            : JSON.stringify(args.headers ?? {});\n        const rawBody =\n          typeof args.body === \"string\"\n            ? args.body\n            : args.body === undefined || args.body === null\n              ? undefined\n              : JSON.stringify(args.body);\n        const timeoutMs = Math.min(\n          Number(args.timeout_ms) || DEFAULT_TIMEOUT_MS,\n          30_000,\n        );\n        const requestedMaxChars = Number(args.maxChars);\n        const maxChars =\n          Number.isFinite(requestedMaxChars) && requestedMaxChars > 0\n            ? Math.min(requestedMaxChars, 200_000)\n            : 32_000;\n\n        // Resolve key references\n        let resolvedUrl = rawUrl;\n        let resolvedHeaders = rawHeaders;\n        let resolvedBody = rawBody;\n        const allUsedKeys: string[] = [];\n        const allSecretValues: string[] = [];\n\n        if (opts.resolveKeys) {\n          try {\n            const urlResult = await opts.resolveKeys(rawUrl);\n            resolvedUrl = urlResult.resolved;\n            allUsedKeys.push(...urlResult.usedKeys);\n            allSecretValues.push(...(urlResult.secretValues ?? []));\n\n            const headerResult = await opts.resolveKeys(rawHeaders);\n            resolvedHeaders = headerResult.resolved;\n            allUsedKeys.push(...headerResult.usedKeys);\n            allSecretValues.push(...(headerResult.secretValues ?? []));\n\n            if (rawBody) {\n              const bodyResult = await opts.resolveKeys(rawBody);\n              resolvedBody = bodyResult.resolved;\n              allUsedKeys.push(...bodyResult.usedKeys);\n              allSecretValues.push(...(bodyResult.secretValues ?? []));\n            }\n          } catch (err: any) {\n            return `Error resolving key references: ${err?.message ?? err}`;\n          }\n        }\n        const secretValues = collectSecretValues(allSecretValues);\n\n        // Block SSRF targets regardless of key usage\n        if (await isBlockedExtensionUrlWithDns(resolvedUrl)) {\n          return `Requests to private/internal addresses are not allowed: \"${rawUrl}\".`;\n        }\n\n        // Validate URL against per-key allowlists\n        if (opts.validateUrl && allUsedKeys.length > 0) {\n          try {\n            const allowed = await opts.validateUrl(resolvedUrl, allUsedKeys);\n            if (!allowed) {\n              return `URL \"${rawUrl}\" is not in the allowlist for the referenced keys. Check your key settings.`;\n            }\n          } catch (err: any) {\n            return `URL validation error: ${err?.message ?? err}`;\n          }\n        }\n\n        // Parse headers, then merge in browser-like defaults for any header the\n        // caller didn't already specify. Real-browser headers (User-Agent,\n        // Accept, Sec-Fetch-*) are what gets you past Cloudflare / PerimeterX /\n        // generic UA-sniffing middleware on sites the user pastes in chat;\n        // explicit caller headers always win so API calls keep their auth\n        // headers untouched.\n        let headers: Record<string, string>;\n        try {\n          headers = sanitizeOutboundHeaders(JSON.parse(resolvedHeaders));\n        } catch {\n          return `Invalid headers JSON: ${rawHeaders}`;\n        }\n        headers = applyBrowserDefaults(headers);\n\n        // Make the request\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), timeoutMs);\n\n        try {\n          const dispatcher = (await createSsrfSafeDispatcher()) ?? undefined;\n          const fetchOpts: RequestInit & { dispatcher?: unknown } = {\n            method,\n            headers,\n            signal: controller.signal,\n            redirect: \"manual\",\n          };\n          if (dispatcher) fetchOpts.dispatcher = dispatcher;\n          if (resolvedBody && [\"POST\", \"PUT\", \"PATCH\"].includes(method)) {\n            fetchOpts.body = resolvedBody;\n            if (!headers[\"content-type\"] && !headers[\"Content-Type\"]) {\n              headers[\"Content-Type\"] = \"application/json\";\n            }\n          }\n\n          const response = await fetch(resolvedUrl, fetchOpts);\n          const elapsed = Date.now() - startTime;\n\n          if (response.status >= 300 && response.status < 400) {\n            const location = response.headers.get(\"location\");\n            const redirectUrl = location\n              ? new URL(location, resolvedUrl).href\n              : null;\n            if (\n              redirectUrl &&\n              (await isBlockedExtensionUrlWithDns(redirectUrl))\n            ) {\n              return \"Redirect to private/internal address blocked.\";\n            }\n            if (redirectUrl && opts.validateUrl && allUsedKeys.length > 0) {\n              const allowed = await opts.validateUrl(redirectUrl, allUsedKeys);\n              if (!allowed) {\n                return \"Redirect URL is not in the allowlist for the referenced keys.\";\n              }\n            }\n            return `HTTP ${response.status} ${response.statusText}\\n\\nRedirect: ${\n              redirectUrl ? redactString(redirectUrl, secretValues) : \"(none)\"\n            }`;\n          }\n\n          // Check if caller wants to save to workspace file (before truncation).\n          const saveToFilePath =\n            typeof (args as Record<string, unknown>).saveToFile === \"string\"\n              ? ((args as Record<string, unknown>).saveToFile as string).trim()\n              : \"\";\n\n          let body: string;\n          try {\n            // When saving to file allow larger reads (20MB), otherwise cap at proxy limit.\n            const readLimit = saveToFilePath\n              ? 20 * 1024 * 1024\n              : MAX_EXTENSION_PROXY_RESPONSE_SIZE;\n            const result = await readResponseTextWithLimit(response, readLimit);\n            body = result.text;\n          } catch {\n            body = \"(could not read response body)\";\n          }\n          body = redactString(body, secretValues);\n          const contentType =\n            response.headers.get(\"content-type\")?.split(\";\")[0].trim() ??\n            \"text/plain\";\n          let displayBody: string;\n          let processedMode = \"raw\";\n          try {\n            const processed = processWebContent({\n              url: resolvedUrl,\n              body,\n              contentType,\n              responseMode: String(args.responseMode ?? \"auto\"),\n              extract: String(args.extract ?? \"readability\"),\n              includeLinks:\n                args.includeLinks === undefined\n                  ? true\n                  : parseBooleanArg(args.includeLinks),\n              search: parseWebContentSearchOptions(args.search),\n              maxChars,\n            });\n            processedMode = processed.mode;\n            displayBody = formatWebContentResult(processed);\n          } catch (err: any) {\n            return `web-request post-processing error: ${err?.message ?? String(err)}`;\n          }\n\n          // Audit log\n          console.log(\n            `[fetch-tool] ${method} ${rawUrl} → ${response.status} (${elapsed}ms, keys: ${allUsedKeys.join(\",\") || \"none\"})`,\n          );\n\n          // saveToFile: write full body to workspace and return compact summary.\n          if (saveToFilePath) {\n            try {\n              const { writeWorkspaceFile, SAVE_TO_FILE_MAX_BYTES } =\n                await import(\"../workspace-files/store.js\");\n              const { getRequestOrgId, getRequestUserEmail } =\n                await import(\"../server/request-context.js\");\n              const orgId = getRequestOrgId();\n              const email = getRequestUserEmail();\n              const scope = orgId\n                ? { scope: \"org\" as const, scopeId: orgId }\n                : email\n                  ? { scope: \"user\" as const, scopeId: email }\n                  : null;\n              if (!scope)\n                throw new Error(\"No authenticated context for saveToFile\");\n              await writeWorkspaceFile(\n                scope,\n                saveToFilePath,\n                body,\n                contentType,\n                {\n                  maxFileBytes: SAVE_TO_FILE_MAX_BYTES,\n                },\n              );\n              const bytes = Buffer.byteLength(body, \"utf8\");\n              const preview = displayBody.slice(0, 2000);\n              return JSON.stringify({\n                savedToFile: true,\n                savedTo: saveToFilePath,\n                status: response.status,\n                bytes,\n                contentType,\n                responseMode: processedMode,\n                preview:\n                  preview.length < displayBody.length ? `${preview}…` : preview,\n              });\n            } catch (saveErr: any) {\n              return `saveToFile error: ${saveErr?.message ?? String(saveErr)}\\n\\nHTTP ${response.status} ${response.statusText}\\n\\n${body.slice(0, maxChars)}`;\n            }\n          }\n\n          return `HTTP ${response.status} ${response.statusText}\\n\\n${displayBody}`;\n        } catch (err: any) {\n          const elapsed = Date.now() - startTime;\n          if (err?.name === \"AbortError\") {\n            console.log(\n              `[fetch-tool] ${method} ${rawUrl} → TIMEOUT (${elapsed}ms)`,\n            );\n            return `Request timed out after ${timeoutMs}ms.`;\n          }\n          const message = redactSecrets(\n            err?.message ?? String(err),\n            secretValues,\n          );\n          console.log(\n            `[fetch-tool] ${method} ${rawUrl} → ERROR: ${message} (${elapsed}ms)`,\n          );\n          return `Request failed: ${message}`;\n        } finally {\n          clearTimeout(timeout);\n        }\n      },\n      readOnly: true,\n    },\n  };\n}\n\nfunction parseBooleanArg(value: unknown): boolean {\n  if (typeof value === \"boolean\") return value;\n  const normalized = String(value).trim().toLowerCase();\n  return normalized === \"true\" || normalized === \"1\" || normalized === \"yes\";\n}\n"]}

package/dist/extensions/web-content.d.ts

@@ -0,0 +1,61 @@
+export type WebResponseMode = "auto" | "raw" | "text" | "markdown" | "links" | "metadata" | "matches";
+export type WebExtractMode = "readability" | "all-visible" | "none";
+export type WebSearchSource = "extracted" | "raw";
+export interface WebContentSearchOptions {
+    query?: string | string[];
+    queries?: string[];
+    terms?: string[];
+    regex?: string;
+    regexFlags?: string;
+    caseSensitive?: boolean;
+    source?: WebSearchSource;
+    maxMatches?: number;
+    contextChars?: number;
+}
+export interface WebContentProcessOptions {
+    url: string;
+    body: string;
+    contentType?: string | null;
+    responseMode?: string;
+    extract?: string;
+    includeLinks?: boolean;
+    search?: WebContentSearchOptions | null;
+    maxChars?: number;
+}
+export interface WebContentLink {
+    text: string;
+    url: string;
+}
+export interface WebContentMatch {
+    kind: "query" | "term" | "regex";
+    query: string;
+    match: string;
+    index: number;
+    snippet: string;
+}
+export interface WebContentResult {
+    mode: Exclude<WebResponseMode, "auto">;
+    extract: WebExtractMode;
+    contentType: string | null;
+    title?: string;
+    excerpt?: string;
+    byline?: string;
+    siteName?: string;
+    lang?: string;
+    publishedTime?: string;
+    content?: string;
+    links?: WebContentLink[];
+    matches?: WebContentMatch[];
+    totalMatches?: number;
+    omittedMatches?: number;
+    searchSource?: WebSearchSource;
+    truncated?: boolean;
+    searchTruncated?: boolean;
+}
+export declare function hasWebContentSearch(search: WebContentSearchOptions | null | undefined): boolean;
+export declare function normalizeWebResponseMode(value: unknown, fallback?: WebResponseMode): WebResponseMode;
+export declare function normalizeWebExtractMode(value: unknown, fallback?: WebExtractMode): WebExtractMode;
+export declare function parseWebContentSearchOptions(value: unknown): WebContentSearchOptions | null;
+export declare function processWebContent(options: WebContentProcessOptions): WebContentResult;
+export declare function formatWebContentResult(result: WebContentResult): string;
+//# sourceMappingURL=web-content.d.ts.map

package/dist/extensions/web-content.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-content.d.ts","sourceRoot":"","sources":["../../src/extensions/web-content.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,eAAe,GACvB,MAAM,GACN,KAAK,GACL,MAAM,GACN,UAAU,GACV,OAAO,GACP,UAAU,GACV,SAAS,CAAC;AAEd,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,aAAa,GAAG,MAAM,CAAC;AACpE,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,KAAK,CAAC;AAElD,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,wBAAwB;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,MAAM,CAAC,EAAE,uBAAuB,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,OAAO,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IACvC,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,CAAC,EAAE,eAAe,EAAE,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,eAAe,CAAC;IAC/B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAiBD,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,uBAAuB,GAAG,IAAI,GAAG,SAAS,GACjD,OAAO,CAQT;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,OAAO,EACd,QAAQ,GAAE,eAAwB,GACjC,eAAe,CAgBjB;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,OAAO,EACd,QAAQ,GAAE,cAA8B,GACvC,cAAc,CAYhB;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,OAAO,GACb,uBAAuB,GAAG,IAAI,CAiBhC;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,wBAAwB,GAChC,gBAAgB,CAsElB;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAkCvE"}