@juicesharp/rpiv-web-tools 1.14.6 → 1.15.0

package/README.md

@@ -30,6 +30,7 @@ Pick one as the active backend; switch any time without losing the others' keys.
 ## Features
 
 - **Read any URL** - fetch http/https pages with HTML-to-text extraction, or get the raw response with `raw: true` (honoured by Brave/Serper/SearXNG; extraction providers — Tavily/Exa/Jina/Firecrawl/Ollama — always return their parsed text).
+- **GitHub URL interceptor (opt-in)** - github.com URLs route through `gh`/`git` for full repository content (file tree, README, individual file contents) instead of the rendered HTML page. Off by default; enable per-user via config or per-consumer at registration time. See [§GitHub URL interceptor](#github-url-interceptor).
 - **Large-page spillover** - oversized responses truncate inline and spill the full body to a temp file the model can read on demand.
 - **SSRF guard** - refuses loopback, RFC 1918, link-local, and cloud-metadata addresses (`localhost`, `127.0.0.0/8`, `10.0.0.0/8`, `169.254.0.0/16`, `172.16.0.0/12`, `192.168.0.0/16`, `::1`, `fc00::/7`, `fe80::/10`).
 - **Interactive setup** - `/web-tools` lists providers (active one first, configured ones marked) and writes to `~/.config/rpiv-web-tools/config.json` (chmod 0600); per-provider env vars also work and take precedence over persisted keys.
@@ -46,9 +47,11 @@ Then restart your Pi session.
 
 - **`web_search`** - query the active provider's search API and return titled snippets.
   1–10 results per call.
-- **`web_fetch`** - fetch an http/https URL through the active provider's content path
-  (raw HTTP+htmlToText for Brave/Serper/SearXNG; native extraction for Tavily/Exa/Jina/Firecrawl/Ollama),
-  truncate large responses with a temp-file spill for the full content.
+- **`web_fetch`** - read an http/https URL. Lookup order: opt-in URL interceptors
+  (see [§GitHub URL interceptor](#github-url-interceptor)), then the active provider's native
+  fetch endpoint when it has one (Tavily/Exa/Jina/Firecrawl/Ollama → vendor extraction;
+  Brave/Serper/SearXNG → shared raw HTTP + HTML-to-text fallback). Large responses truncate
+  inline and spill the full body to a temp file the model can read on demand.
 
 ### Schema - `web_search`
 
@@ -107,7 +110,8 @@ Throws on invalid URL, non-http(s) protocol, private/loopback hostnames (SSRF gu
 - **`/web-tools`** - pick the active provider and set its API key interactively.
   Providers already configured show `(configured)`; the active one is listed first with a `✓`.
   Pressing Enter on an empty input keeps the existing key for the chosen provider while
-  persisting the provider switch. Pass `--show` to see all per-provider keys (masked) and env var status.
+  persisting the provider switch. Pass `--show` to see all per-provider keys (masked), env var status,
+  and current URL interceptor states (see [§GitHub URL interceptor](#github-url-interceptor)).
 
 ## API key resolution (per active provider)
 
@@ -189,6 +193,45 @@ The provider automatically uses the correct API paths:
 - **Local** (`localhost`, `127.0.0.1`, `0.0.0.0`): `/api/experimental/web_search` and `/api/experimental/web_fetch`
 - **Cloud** (any other host): `/api/web_search` and `/api/web_fetch`
 
+## GitHub URL interceptor
+
+Routes github.com URLs through `gh` / `git` to return repository content (file tree, README, file content) instead of the rendered HTML. **Off by default.** Opt in two ways:
+
+```json
+// ~/.config/rpiv-web-tools/config.json — end-user opt-in
+{ "interceptors": { "github": true } }
+```
+
+```ts
+// or per-consumer at registration time (user config still wins)
+registerWebTools(pi, { interceptors: { github: true } });
+```
+
+When enabled, github.com URLs are parsed into `owner/repo/ref/path`; non-code paths (`/issues`, `/pulls`, `/discussions`, `/releases`, …) fall through to the active provider. The interceptor probes for `gh`, falls back to plain `git clone` (with a stderr hint to install `gh`), and uses the `gh api` JSON view for SHA-pinned URLs and repos above `maxRepoSizeMB`. Shallow clones (`--depth 1 --single-branch`) land in `clonePath`; successful clones cache by `owner/repo@ref` for the session. Auth flows through `gh`'s normal `GH_TOKEN`/`GITHUB_TOKEN` precedence — export `GITHUB_TOKEN` to reach private repos.
+
+Replace the boolean shorthand with an object to tune the defaults; object form implies opt-in.
+
+```json
+{
+  "interceptors": {
+    "github": {
+      "maxRepoSizeMB": 1000,
+      "cloneTimeoutSeconds": 90,
+      "clonePath": "/Users/me/.cache/pi-github-repos"
+    }
+  }
+}
+```
+
+| Field | Default | Purpose |
+|---|---|---|
+| `enabled` | `false` (top-level) / `true` (inside object form) | Master switch |
+| `maxRepoSizeMB` | `350` | Repos above this threshold skip the clone and use the API view |
+| `cloneTimeoutSeconds` | `30` | Kill the clone process after this many seconds |
+| `clonePath` | `$TMPDIR/pi-github-repos` | Where shallow clones land; one subdir per `owner/repo@ref` |
+
+`/web-tools --show` reports the current state at the bottom of its output (resolved token masked, `clonePath`, `maxRepoSizeMB`). The SSRF guard still runs first — a URL with a private/loopback host can't bypass it via a github.com path shape.
+
 ## Executor guidance overrides
 
 Override the `promptSnippet` / `promptGuidelines` the model sees for each tool by editing `~/.config/rpiv-web-tools/config.json`. Note the per-tool nesting under `guidance.web_search` / `guidance.web_fetch` — this differs from the flat `guidance` shape used by single-tool siblings (`rpiv-advisor`, `rpiv-todo`, `rpiv-ask-user-question`):
@@ -200,6 +243,9 @@ Override the `promptSnippet` / `promptGuidelines` the model sees for each tool b
     "exa": "sk-...",
     "brave": "sk-..."
   },
+  "interceptors": {
+    "github": true
+  },
   "guidance": {
     "web_search": {
       "promptSnippet": "Search the web for current docs and library versions",
@@ -217,6 +263,8 @@ Override the `promptSnippet` / `promptGuidelines` the model sees for each tool b
 
 Each field is independent: omit one and the built-in default is kept. Invalid values (empty string, wrong type, empty array) silently fall back to defaults. Changes take effect on the next Pi session start.
 
+The `interceptors` key is the GitHub URL interceptor opt-in — see [§GitHub URL interceptor](#github-url-interceptor) for the full schema (boolean shorthand or per-field overrides).
+
 ## Security note: `web_fetch` host guard
 
 `web_fetch` refuses URLs targeting loopback (`localhost`, `127.0.0.0/8`, `::1`), RFC 1918 private ranges (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`), link-local (`169.254.0.0/16`, including cloud-metadata at `169.254.169.254`), and IPv6 unique-local / link-local (`fc00::/7`, `fe80::/10`). Attempts surface as `Refusing to fetch private/loopback address: <host>`. This blocks the most common SSRF class — direct-literal targeting of internal services or cloud-metadata endpoints — without preventing legitimate public-web fetches.

package/index.ts

@@ -9,11 +9,28 @@
  */
 
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { buildInterceptors } from "./providers/interceptors/index.js";
 import { registerWebFetchTool, registerWebSearchConfigCommand, registerWebSearchTool } from "./web-tools.js";
 
 export { createSearchProvider } from "./providers/factory.js";
+export {
+	GITHUB_TOKEN_ENV_VAR,
+	GitHubInterceptor,
+	type GitHubInterceptorOptions,
+	type GitHubUrlInfo,
+	parseGitHubUrl,
+	resolveGitHubOptions,
+	type UrlInterceptor,
+} from "./providers/interceptors/index.js";
 
-export type { FetchResponse, SearchProvider, SearchResponse, SearchResult } from "./providers/types.js";
+export type {
+	FetchProvider,
+	FetchResponse,
+	FullProvider,
+	SearchProvider,
+	SearchResponse,
+	SearchResult,
+} from "./providers/types.js";
 export {
 	DEFAULT_WEB_FETCH_GUIDELINES,
 	DEFAULT_WEB_FETCH_SNIPPET,
@@ -24,7 +41,17 @@ export {
 	registerWebSearchTool,
 } from "./web-tools.js";
 
-export default function (pi: ExtensionAPI) {
+// Programmatic consumer-side opt-in for URL interceptors. Tier 2 in the
+// resolution model: end-user config (Tier 1) still wins. Default OFF —
+// existing rpiv-web-tools users see zero behavior change.
+export interface RegisterOptions {
+	interceptors?: {
+		github?: boolean;
+	};
+}
+
+export default function registerWebTools(pi: ExtensionAPI, opts?: RegisterOptions): void {
+	buildInterceptors(opts?.interceptors);
 	registerWebSearchTool(pi);
 	registerWebFetchTool(pi);
 	registerWebSearchConfigCommand(pi);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@juicesharp/rpiv-web-tools",
-	"version": "1.14.6",
+	"version": "1.15.0",
 	"description": "Pi extension. Web search and fetch for the model with pluggable providers (Brave, Tavily, Serper, Exa, Jina, Firecrawl, SearXNG, Ollama).",
 	"keywords": [
 		"pi-package",
@@ -57,7 +57,7 @@
 		]
 	},
 	"dependencies": {
-		"@juicesharp/rpiv-config": "^1.14.6"
+		"@juicesharp/rpiv-config": "^1.15.0"
 	},
 	"peerDependencies": {
 		"@earendil-works/pi-coding-agent": "*",

package/providers/brave.ts

@@ -1,5 +1,4 @@
-import { assertTextContentType, extractBodyAsText, fetchUrlOrThrow } from "./fetch-helpers.js";
-import type { FetchResponse, SearchProvider, SearchResponse, SearchResult } from "./types.js";
+import type { SearchProvider, SearchResponse, SearchResult } from "./types.js";
 
 const BRAVE_SEARCH_API_URL = "https://api.search.brave.com/res/v1/web/search";
 export const BRAVE_API_KEY_ENV_VAR = "BRAVE_SEARCH_API_KEY";
@@ -7,6 +6,7 @@ export const BRAVE_PROVIDER_META = {
 	name: "brave",
 	label: "Brave",
 	envVar: BRAVE_API_KEY_ENV_VAR,
+	roles: ["search"] as const,
 } as const;
 
 interface BraveRawResponse {
@@ -55,22 +55,4 @@ export class BraveProvider implements SearchProvider {
 		const raw = (await res.json()) as BraveRawResponse;
 		return { query, results: normalizeBraveResults(raw) };
 	}
-
-	// No apiKey guard: Brave's fetch() wraps the built-in HTTP+htmlToText
-	// pipeline and does not call any vendor endpoint. Adding a guard would
-	// break the "use any provider for fetch" contract.
-	async fetch(url: string, raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
-		const res = await fetchUrlOrThrow(url, signal);
-		const contentType = res.headers.get("content-type") ?? "";
-		assertTextContentType(contentType);
-
-		const { text, title } = await extractBodyAsText(res, contentType, raw);
-		const contentLengthHeader = res.headers.get("content-length");
-		return {
-			text,
-			title,
-			contentType: contentType || undefined,
-			contentLength: contentLengthHeader ? Number(contentLengthHeader) : undefined,
-		};
-	}
 }

package/providers/config.test.ts

@@ -0,0 +1,129 @@
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import { configPath } from "@juicesharp/rpiv-config";
+import { beforeEach, describe, expect, it } from "vitest";
+import { getConfigPath, readConfig, WebToolsConfigSchema, writeConfig } from "./config.js";
+
+const CONFIG_PATH = configPath("rpiv-web-tools");
+
+beforeEach(() => {
+	rmSync(CONFIG_PATH, { force: true });
+});
+
+function writeRaw(contents: string): void {
+	mkdirSync(dirname(CONFIG_PATH), { recursive: true });
+	writeFileSync(CONFIG_PATH, contents, "utf-8");
+}
+
+describe("getConfigPath", () => {
+	it("returns the canonical ~/.config/rpiv-web-tools/config.json", () => {
+		expect(getConfigPath()).toBe(CONFIG_PATH);
+	});
+});
+
+describe("readConfig — fail-soft posture", () => {
+	it("returns {} when the file does not exist", () => {
+		expect(readConfig()).toEqual({});
+	});
+
+	it("returns {} on malformed JSON (matches loadJsonConfig tolerance)", () => {
+		writeRaw("{ not valid json");
+		expect(readConfig()).toEqual({});
+	});
+
+	it("returns {} when the file is a directory (EISDIR)", () => {
+		mkdirSync(CONFIG_PATH, { recursive: true });
+		try {
+			expect(readConfig()).toEqual({});
+		} finally {
+			rmSync(CONFIG_PATH, { recursive: true, force: true });
+		}
+	});
+
+	it("returns {} when the schema validation fails hard (e.g. provider is a number)", () => {
+		writeRaw(JSON.stringify({ provider: 123 }));
+		expect(readConfig()).toEqual({});
+	});
+});
+
+describe("readConfig — released-shape compatibility", () => {
+	it("loads a minimal { provider, apiKeys } config unchanged", () => {
+		writeRaw(JSON.stringify({ provider: "brave", apiKeys: { brave: "k" } }));
+		expect(readConfig()).toEqual({ provider: "brave", apiKeys: { brave: "k" } });
+	});
+
+	it("loads the legacy top-level apiKey field", () => {
+		writeRaw(JSON.stringify({ apiKey: "legacy" }));
+		expect(readConfig()).toMatchObject({ apiKey: "legacy" });
+	});
+
+	it("preserves unknown top-level keys (otherField round-trip contract)", () => {
+		// The released /web-tools migrate-legacy-apiKey test relies on this:
+		// unknown keys MUST NOT be stripped by the schema reader.
+		writeRaw(JSON.stringify({ apiKey: "k", otherField: "keep" }));
+		const cfg = readConfig() as { otherField?: string };
+		expect(cfg.otherField).toBe("keep");
+	});
+
+	it("loads the guidance subtree with web_search + web_fetch", () => {
+		writeRaw(
+			JSON.stringify({
+				guidance: {
+					web_search: { promptSnippet: "snip", promptGuidelines: ["a", "b"] },
+					web_fetch: { promptSnippet: "snip2" },
+				},
+			}),
+		);
+		const cfg = readConfig();
+		expect(cfg.guidance?.web_search?.promptSnippet).toBe("snip");
+		expect(cfg.guidance?.web_fetch?.promptSnippet).toBe("snip2");
+	});
+});
+
+describe("readConfig — interceptors.github union", () => {
+	it("accepts the boolean true shorthand", () => {
+		writeRaw(JSON.stringify({ interceptors: { github: true } }));
+		expect(readConfig().interceptors?.github).toBe(true);
+	});
+
+	it("accepts the boolean false shorthand", () => {
+		writeRaw(JSON.stringify({ interceptors: { github: false } }));
+		expect(readConfig().interceptors?.github).toBe(false);
+	});
+
+	it("accepts the object override form", () => {
+		writeRaw(
+			JSON.stringify({
+				interceptors: { github: { maxRepoSizeMB: 1000, clonePath: "/x" } },
+			}),
+		);
+		const gh = readConfig().interceptors?.github;
+		expect(gh).toEqual({ maxRepoSizeMB: 1000, clonePath: "/x" });
+	});
+
+	it("falls back to {} when interceptors.github has a type-incompatible shape", () => {
+		// A number is neither boolean nor a GitHubInterceptorOptions object —
+		// hard schema failure → fail-soft to {}.
+		writeRaw(JSON.stringify({ interceptors: { github: 42 } }));
+		expect(readConfig()).toEqual({});
+	});
+});
+
+describe("writeConfig", () => {
+	it("round-trips a config through readConfig", () => {
+		expect(writeConfig({ provider: "brave", apiKeys: { brave: "k" } })).toBe(true);
+		expect(readConfig()).toEqual({ provider: "brave", apiKeys: { brave: "k" } });
+	});
+
+	it("preserves the interceptors.github stanza across save+load", () => {
+		expect(writeConfig({ interceptors: { github: { maxRepoSizeMB: 500 } } })).toBe(true);
+		expect(readConfig().interceptors?.github).toEqual({ maxRepoSizeMB: 500 });
+	});
+});
+
+describe("WebToolsConfigSchema — schema-only sanity", () => {
+	it("exists and is a TypeBox object", () => {
+		expect(WebToolsConfigSchema).toBeDefined();
+		expect(WebToolsConfigSchema.type).toBe("object");
+	});
+});

package/providers/config.ts

@@ -0,0 +1,96 @@
+/**
+ * Single typed reader/writer for ~/.config/rpiv-web-tools/config.json.
+ *
+ * Owns the canonical WebToolsConfigSchema. All schema fields are optional and
+ * unknown keys pass through (additionalProperties: true) so existing configs
+ * carrying legacy/unrelated fields keep working — required for the
+ * `otherField: "keep"` preservation contract the released `/web-tools`
+ * legacy-apiKey migration depends on.
+ *
+ * Validation is fail-soft (matching `loadJsonConfig` and `validateConfig` in
+ * rpiv-config): malformed JSON, EISDIR, or a hard schema violation all
+ * degrade to `{}`. The orchestrator never has to handle "config blew up at
+ * startup."
+ */
+
+import { configPath, GuidanceFieldsSchema, loadJsonConfig, saveJsonConfig } from "@juicesharp/rpiv-config";
+import { type Static, Type } from "typebox";
+import { Value } from "typebox/value";
+
+// The web_search / web_fetch tool-namespace wrapper is web-tools' concept, not
+// rpiv-config's. The leaf schema (`GuidanceFieldsSchema`) is sibling-agnostic
+// and lives in rpiv-config; this file only composes the tool-namespaced shell
+// around it.
+const WebToolsGuidanceSchema = Type.Object(
+	{
+		web_search: Type.Optional(GuidanceFieldsSchema),
+		web_fetch: Type.Optional(GuidanceFieldsSchema),
+	},
+	{ additionalProperties: true },
+);
+
+const GitHubInterceptorOptionsSchema = Type.Object(
+	{
+		enabled: Type.Optional(Type.Boolean()),
+		maxRepoSizeMB: Type.Optional(Type.Number()),
+		cloneTimeoutSeconds: Type.Optional(Type.Number()),
+		clonePath: Type.Optional(Type.String()),
+	},
+	{ additionalProperties: true },
+);
+
+const InterceptorsConfigSchema = Type.Object(
+	{
+		// Boolean shorthand or per-field overrides. `enabled: false` inside the
+		// object form is allowed but redundant — use the top-level `false`.
+		github: Type.Optional(Type.Union([Type.Boolean(), GitHubInterceptorOptionsSchema])),
+	},
+	{ additionalProperties: true },
+);
+
+export const WebToolsConfigSchema = Type.Object(
+	{
+		provider: Type.Optional(Type.String()),
+		apiKeys: Type.Optional(Type.Record(Type.String(), Type.String())),
+		baseUrls: Type.Optional(Type.Record(Type.String(), Type.String())),
+		// Legacy top-level Brave key. Auto-migrated to `apiKeys.brave` by the
+		// /web-tools save path — kept here for the load+rewrite round-trip.
+		apiKey: Type.Optional(Type.String()),
+		guidance: Type.Optional(WebToolsGuidanceSchema),
+		interceptors: Type.Optional(InterceptorsConfigSchema),
+	},
+	{ additionalProperties: true },
+);
+
+export type WebToolsConfig = Static<typeof WebToolsConfigSchema>;
+
+const CONFIG_PATH = configPath("rpiv-web-tools");
+
+export function getConfigPath(): string {
+	return CONFIG_PATH;
+}
+
+// Tolerant read: loadJsonConfig already swallows JSON parse failures + EISDIR
+// into `{}`; we then run a schema check that — on hard failure — falls back to
+// the same `{}`. Validation uses `Value.Check` (no mutation) rather than
+// `Value.Clean` (would strip unknown fields like the released `otherField`
+// pass-through contract).
+export function readConfig(): WebToolsConfig {
+	const raw = loadJsonConfig<unknown>(CONFIG_PATH);
+	if (!Value.Check(WebToolsConfigSchema, raw)) {
+		return {} as WebToolsConfig;
+	}
+	return raw as WebToolsConfig;
+}
+
+export function writeConfig(c: WebToolsConfig): boolean {
+	return saveJsonConfig(CONFIG_PATH, c);
+}
+
+// Plan-surface no-op. Phase 4 omits the in-memory cache the plan sketched —
+// the tests' direct-writeFileSync pattern makes per-test invalidation a
+// rewrite-the-suite job for marginal perf gain. Kept exported so that
+// consumers writing against the plan's API can call it without breaking.
+export function invalidateConfigCache(): void {
+	// no-op
+}

package/providers/exa.ts

@@ -1,4 +1,4 @@
-import type { FetchResponse, SearchProvider, SearchResponse, SearchResult } from "./types.js";
+import type { FetchResponse, FullProvider, SearchResponse, SearchResult } from "./types.js";
 
 const EXA_API_URL = "https://api.exa.ai/search";
 const EXA_CONTENTS_API_URL = "https://api.exa.ai/contents";
@@ -7,6 +7,7 @@ export const EXA_PROVIDER_META = {
 	name: "exa",
 	label: "Exa",
 	envVar: EXA_API_KEY_ENV_VAR,
+	roles: ["search", "fetch"] as const,
 } as const;
 const EXA_MAX_SNIPPET_CHARACTERS = 300;
 const EXA_MAX_FETCH_CHARACTERS = 10000;
@@ -30,7 +31,7 @@ function normalizeExaResults(results: ExaRawResult[]): SearchResult[] {
 	}));
 }
 
-export class ExaProvider implements SearchProvider {
+export class ExaProvider implements FullProvider {
 	readonly name = EXA_PROVIDER_META.name;
 	readonly label = EXA_PROVIDER_META.label;
 	readonly envVar = EXA_PROVIDER_META.envVar;

package/providers/factory.ts

@@ -6,14 +6,18 @@ import { OllamaProvider } from "./ollama.js";
 import { SearxngProvider } from "./searxng.js";
 import { SerperProvider } from "./serper.js";
 import { TavilyProvider } from "./tavily.js";
-import type { SearchProvider } from "./types.js";
+import type { FullProvider, SearchProvider } from "./types.js";
 
 export interface ProviderCredentials {
 	apiKey?: string;
 	baseUrl?: string;
 }
 
-export function createSearchProvider(name: string, creds: ProviderCredentials): SearchProvider {
+// The return union mirrors the role split: Brave/Serper/SearXNG are search-
+// only (SearchProvider); the other five expose native fetch endpoints too
+// (FullProvider). Consumers narrow with `"fetch" in provider` when they need
+// to dispatch on capability.
+export function createSearchProvider(name: string, creds: ProviderCredentials): SearchProvider | FullProvider {
 	const apiKey = creds.apiKey ?? "";
 	switch (name) {
 		case "brave":

package/providers/fetch-helpers.ts

@@ -1,8 +1,12 @@
 /**
  * Shared fetch helpers — HTTP client, content-type guards, and HTML-to-text
- * extraction used by providers that wrap the built-in pipeline (Brave, Serper).
+ * extraction used by providers that wrap the built-in pipeline (Brave, Serper,
+ * SearXNG). `fetchViaGenericHtml` is the one-stop entry point those providers
+ * delegate their `fetch()` method to.
  */
 
+import type { FetchResponse } from "./types.js";
+
 // ---------------------------------------------------------------------------
 // Constants
 // ---------------------------------------------------------------------------
@@ -115,3 +119,21 @@ export async function extractBodyAsText(
 	}
 	return { text: body };
 }
+
+// One-stop fetch helper for providers that have no native fetch endpoint
+// (Brave/Serper/SearXNG). Bundles the quartet — fetchUrlOrThrow →
+// content-type assertion → body extraction → FetchResponse envelope — so
+// providers collapse to a single delegating call.
+export async function fetchViaGenericHtml(url: string, raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
+	const res = await fetchUrlOrThrow(url, signal);
+	const contentType = res.headers.get("content-type") ?? "";
+	assertTextContentType(contentType);
+	const { text, title } = await extractBodyAsText(res, contentType, raw);
+	const contentLengthHeader = res.headers.get("content-length");
+	return {
+		text,
+		title,
+		contentType: contentType || undefined,
+		contentLength: contentLengthHeader ? Number(contentLengthHeader) : undefined,
+	};
+}

package/providers/firecrawl.ts

@@ -1,4 +1,4 @@
-import type { FetchResponse, SearchProvider, SearchResponse, SearchResult } from "./types.js";
+import type { FetchResponse, FullProvider, SearchResponse, SearchResult } from "./types.js";
 
 const FIRECRAWL_API_URL = "https://api.firecrawl.dev/v1";
 export const FIRECRAWL_API_KEY_ENV_VAR = "FIRECRAWL_API_KEY";
@@ -6,6 +6,7 @@ export const FIRECRAWL_PROVIDER_META = {
 	name: "firecrawl",
 	label: "Firecrawl",
 	envVar: FIRECRAWL_API_KEY_ENV_VAR,
+	roles: ["search", "fetch"] as const,
 } as const;
 
 interface FirecrawlSearchResult {
@@ -43,7 +44,7 @@ function normalizeFirecrawlResults(results: FirecrawlSearchResult[]): SearchResu
 	}));
 }
 
-export class FirecrawlProvider implements SearchProvider {
+export class FirecrawlProvider implements FullProvider {
 	readonly name = FIRECRAWL_PROVIDER_META.name;
 	readonly label = FIRECRAWL_PROVIDER_META.label;
 	readonly envVar = FIRECRAWL_PROVIDER_META.envVar;

package/providers/index.ts

@@ -12,6 +12,17 @@ export { BRAVE_API_KEY_ENV_VAR, BRAVE_PROVIDER_META, BraveProvider } from "./bra
 export { EXA_API_KEY_ENV_VAR, EXA_PROVIDER_META, ExaProvider } from "./exa.js";
 export { createSearchProvider, type ProviderCredentials } from "./factory.js";
 export { FIRECRAWL_API_KEY_ENV_VAR, FIRECRAWL_PROVIDER_META, FirecrawlProvider } from "./firecrawl.js";
+// URL interceptors live in providers/interceptors/. The github primitives
+// (parseGitHubUrl, GitHubUrlInfo, etc.) are re-exported from there.
+export {
+	clearCloneCache,
+	GITHUB_TOKEN_ENV_VAR,
+	GitHubInterceptor,
+	type GitHubInterceptorOptions,
+	type GitHubUrlInfo,
+	parseGitHubUrl,
+	type UrlInterceptor,
+} from "./interceptors/index.js";
 export { JINA_API_KEY_ENV_VAR, JINA_PROVIDER_META, JinaProvider } from "./jina.js";
 export {
 	configureOllama,
@@ -35,11 +46,14 @@ export {
 export { SERPER_API_KEY_ENV_VAR, SERPER_PROVIDER_META, SerperProvider } from "./serper.js";
 export { TAVILY_API_KEY_ENV_VAR, TAVILY_PROVIDER_META, TavilyProvider } from "./tavily.js";
 export type {
+	FetchProvider,
 	FetchResponse,
+	FullProvider,
 	ProviderConfigChange,
 	ProviderConfigCurrent,
 	ProviderConfigUi,
 	ProviderMeta,
+	ProviderRole,
 	SearchProvider,
 	SearchResponse,
 	SearchResult,