@autorag/librarian 2.5.1 → 2.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/NOTICE +3 -0
  2. package/README.md +230 -490
  3. package/dist/agent/agent.d.ts +40 -3
  4. package/dist/agent/index.d.ts +1 -1
  5. package/dist/agent/search-single-datasource-tool.d.ts +63 -0
  6. package/dist/agent/web-fetch-tool.d.ts +32 -0
  7. package/dist/agent/web-search-tool.d.ts +48 -0
  8. package/dist/cli/commands/lite-retrieve.d.ts +15 -0
  9. package/dist/cli/commands/search.d.ts +6 -0
  10. package/dist/cli/config.d.ts +20 -0
  11. package/dist/cli/index.js +17643 -10464
  12. package/dist/core.d.ts +3 -1
  13. package/dist/core.js +11081 -3945
  14. package/dist/datasource/connector-skill.d.ts +6 -0
  15. package/dist/datasource/crawler-skill.d.ts +2 -0
  16. package/dist/datasource/errors.d.ts +26 -0
  17. package/dist/datasource/index.d.ts +3 -2
  18. package/dist/datasource/skills/discrawl/skill.d.ts +1 -1
  19. package/dist/datasource/skills/discrawl/types.d.ts +2 -2
  20. package/dist/datasource/skills/github-gist/connector.d.ts +38 -0
  21. package/dist/datasource/skills/github-gist/index.d.ts +3 -0
  22. package/dist/datasource/skills/github-gist/semantic.d.ts +102 -0
  23. package/dist/datasource/skills/github-gist/skill.d.ts +39 -0
  24. package/dist/datasource/skills/lark/client.d.ts +23 -0
  25. package/dist/datasource/skills/lark/index.d.ts +2 -0
  26. package/dist/datasource/skills/lark/skill.d.ts +36 -0
  27. package/dist/datasource/skills/lazykatok/client.d.ts +28 -0
  28. package/dist/datasource/skills/lazykatok/index.d.ts +12 -0
  29. package/dist/datasource/skills/{katok → lazykatok}/methods.d.ts +12 -12
  30. package/dist/datasource/skills/lazykatok/paths.d.ts +40 -0
  31. package/dist/datasource/skills/{katok → lazykatok}/skill.d.ts +10 -10
  32. package/dist/datasource/skills/lazykatok/types.d.ts +134 -0
  33. package/dist/datasource/skills/mailcrawl/client.d.ts +2 -0
  34. package/dist/datasource/skills/mailcrawl/types.d.ts +12 -0
  35. package/dist/datasource/skills/slack/index.d.ts +8 -0
  36. package/dist/datasource/tool-naming.d.ts +11 -0
  37. package/dist/datasource/types.d.ts +4 -4
  38. package/dist/embedding-runtime/manifest.d.ts +8 -0
  39. package/dist/embedding-runtime/types.d.ts +4 -0
  40. package/dist/index.js +11598 -4461
  41. package/dist/minsync/types.d.ts +7 -0
  42. package/dist/process/terminate-tree.d.ts +2 -0
  43. package/dist/retrieval/engine.d.ts +14 -5
  44. package/dist/retrieval/index.d.ts +2 -1
  45. package/dist/retrieval/merger.d.ts +30 -5
  46. package/dist/retrieval/skip.d.ts +41 -0
  47. package/dist/retrieval/types.d.ts +24 -3
  48. package/dist/web/entities.d.ts +10 -0
  49. package/dist/web/fetch/convert.d.ts +35 -0
  50. package/dist/web/fetch/feeds.d.ts +37 -0
  51. package/dist/web/fetch/html-renderer.d.ts +54 -0
  52. package/dist/web/fetch/index.d.ts +12 -0
  53. package/dist/web/fetch/negotiate.d.ts +27 -0
  54. package/dist/web/fetch/page-loader.d.ts +60 -0
  55. package/dist/web/fetch/render.d.ts +23 -0
  56. package/dist/web/fetch/turndown/gfm.d.ts +11 -0
  57. package/dist/web/fetch/turndown/html.d.ts +5 -0
  58. package/dist/web/fetch/turndown/index.d.ts +10 -0
  59. package/dist/web/fetch/turndown/service.d.ts +21 -0
  60. package/dist/web/fetch/turndown/types.d.ts +73 -0
  61. package/dist/web/fetch/url-target.d.ts +29 -0
  62. package/dist/web/search/browser-headers.d.ts +17 -0
  63. package/dist/web/search/browser-loader.d.ts +56 -0
  64. package/dist/web/search/browser-page.d.ts +41 -0
  65. package/dist/web/search/credentials.d.ts +10 -0
  66. package/dist/web/search/format.d.ts +30 -0
  67. package/dist/web/search/index.d.ts +46 -0
  68. package/dist/web/search/model-auth.d.ts +36 -0
  69. package/dist/web/search/provider.d.ts +62 -0
  70. package/dist/web/search/providers/anthropic.d.ts +9 -0
  71. package/dist/web/search/providers/base.d.ts +106 -0
  72. package/dist/web/search/providers/codex.d.ts +9 -0
  73. package/dist/web/search/providers/duckduckgo.d.ts +22 -0
  74. package/dist/web/search/providers/ecosia.d.ts +29 -0
  75. package/dist/web/search/providers/gemini.d.ts +9 -0
  76. package/dist/web/search/providers/google.d.ts +22 -0
  77. package/dist/web/search/providers/mojeek.d.ts +24 -0
  78. package/dist/web/search/providers/parallel.d.ts +10 -0
  79. package/dist/web/search/providers/perplexity.d.ts +10 -0
  80. package/dist/web/search/providers/public.d.ts +15 -0
  81. package/dist/web/search/providers/responses-api.d.ts +26 -0
  82. package/dist/web/search/providers/searxng.d.ts +19 -0
  83. package/dist/web/search/providers/startpage.d.ts +25 -0
  84. package/dist/web/search/providers/utils.d.ts +39 -0
  85. package/dist/web/search/providers/xai.d.ts +9 -0
  86. package/dist/web/search/query.d.ts +193 -0
  87. package/dist/web/search/types.d.ts +198 -0
  88. package/dist/web/search/utils.d.ts +8 -0
  89. package/licenses/embedding-assets.json +20 -4
  90. package/package.json +2 -1
  91. package/skills/autorag/SKILL.md +2 -1
  92. package/skills/autorag-doctor/SKILL.md +238 -0
  93. package/skills/autorag-lite-search/SKILL.md +29 -7
  94. package/skills/autorag-lite-setup/SKILL.md +22 -15
  95. package/skills/autorag-setup/SKILL.md +40 -19
  96. package/dist/agent/search-datasource-tool.d.ts +0 -29
  97. package/dist/datasource/skills/katok/client.d.ts +0 -21
  98. package/dist/datasource/skills/katok/index.d.ts +0 -12
  99. package/dist/datasource/skills/katok/paths.d.ts +0 -40
  100. package/dist/datasource/skills/katok/types.d.ts +0 -123
@@ -0,0 +1,29 @@
1
+ /**
2
+ * URL normalization, MIME classification, and binary detection — ported from
3
+ * oh-my-pi's `packages/coding-agent/src/tools/fetch.ts` (MIT licensed).
4
+ */
5
+ /**
6
+ * Normalize a URL: repair a collapsed scheme, then add `https://` if missing.
7
+ */
8
+ export declare function normalizeUrl(url: string): string;
9
+ /**
10
+ * Normalize a MIME type (lowercase, strip charset/params).
11
+ */
12
+ export declare function normalizeMime(contentType: string): string;
13
+ export declare function getFilenameExtensionHint(filename: string): string;
14
+ /**
15
+ * Get extension from URL or Content-Disposition.
16
+ */
17
+ export declare function getExtensionHint(url: string, contentDisposition?: string): string;
18
+ /**
19
+ * Check if content type is convertible via AutoRAG's parser registry.
20
+ */
21
+ export declare function isConvertible(mime: string, extensionHint: string): boolean;
22
+ /**
23
+ * Heuristic: does the text sample look like binary (NUL bytes or high U+FFFD density)?
24
+ */
25
+ export declare function sampleLooksBinary(text: string): boolean;
26
+ /**
27
+ * Build a textual binary notice for unsupported binary payloads.
28
+ */
29
+ export declare function buildBinaryNotice(finalUrl: string, mime: string, byteLength?: number): string;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Browser-like navigation headers for credential-free search engines.
3
+ *
4
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
5
+ * `web/search/providers/browser-headers.ts`. oh-my-pi randomizes across a
6
+ * header-generator fingerprint pool; AutoRAG pins the deterministic Mac
7
+ * Chrome fallback profile (the same one oh-my-pi uses for non-randomized
8
+ * calls) so tests and diagnostics stay reproducible.
9
+ */
10
+ /**
11
+ * Build a desktop navigation fingerprint for one HTTP request.
12
+ * `randomized` is accepted for oh-my-pi call-site compatibility; the
13
+ * deterministic profile is always returned.
14
+ */
15
+ export declare function buildBrowserNavigationHeaders(_options?: {
16
+ randomized?: boolean;
17
+ }): Record<string, string>;
@@ -0,0 +1,56 @@
1
+ import type { BrowserFallbackOptions, LoadedHtmlPage, WebSearchBrowserLoader } from "./browser-page.js";
2
+ /** Minimal page surface the loader needs (structural subset of puppeteer-core's Page). */
3
+ export interface WebSearchBrowserPage {
4
+ goto(url: string, options?: {
5
+ waitUntil?: string;
6
+ timeout?: number;
7
+ }): Promise<{
8
+ status(): number;
9
+ } | null>;
10
+ content(): Promise<string>;
11
+ url(): string;
12
+ close(): Promise<void>;
13
+ setViewport(viewport: {
14
+ width: number;
15
+ height: number;
16
+ }): Promise<void>;
17
+ evaluateOnNewDocument(fn: () => void): Promise<void>;
18
+ waitForSelector(selector: string, options?: {
19
+ timeout?: number;
20
+ }): Promise<unknown>;
21
+ }
22
+ /** Minimal browser surface the loader needs (structural over puppeteer-core's Browser). */
23
+ export interface WebSearchBrowser {
24
+ newPage(): Promise<WebSearchBrowserPage>;
25
+ close(): Promise<void>;
26
+ }
27
+ /** Launch seam: production uses puppeteer-core with the detected executable; tests inject a fake. */
28
+ export type WebSearchBrowserLaunch = (executablePath: string) => Promise<WebSearchBrowser>;
29
+ export interface EnsureWebSearchBrowserLoaderOptions {
30
+ /** Executable existence probe (default: fs.existsSync). */
31
+ readonly exists?: (path: string) => boolean;
32
+ /** Candidate executable paths (default: env + platform defaults). */
33
+ readonly candidates?: readonly string[];
34
+ /** Browser launcher (default: lazy puppeteer-core). */
35
+ readonly launch?: WebSearchBrowserLaunch;
36
+ }
37
+ /** Install (or clear, with `undefined`) an explicit headless-browser loader, taking precedence over the default. */
38
+ export declare function setWebSearchBrowserLoader(loader: WebSearchBrowserLoader | undefined): void;
39
+ /** The recorded browser-escalation diagnostic (why no default loader is installed), if any. */
40
+ export declare function getWebSearchBrowserDiagnostic(): string | undefined;
41
+ /** Test hook: drop the explicit loader, the memoized default, and the diagnostic. */
42
+ export declare function resetWebSearchBrowserLoaderForTests(): void;
43
+ /**
44
+ * Probe for a local Chrome/Chromium and install the default escalation
45
+ * loader. Returns true when a loader is installed. Never throws: when no
46
+ * browser exists (or puppeteer-core cannot load), a diagnostic is recorded
47
+ * and the provider chain simply never escalates. Memoized per process.
48
+ */
49
+ export declare function ensureWebSearchBrowserLoader(options?: EnsureWebSearchBrowserLoaderOptions): Promise<boolean>;
50
+ /**
51
+ * Resolve the loader an escalation should use: an explicitly registered
52
+ * loader wins; otherwise the lazily-probed default. Returns undefined when
53
+ * no browser is available (degradation already recorded as a diagnostic).
54
+ */
55
+ export declare function resolveWebSearchBrowserLoader(): Promise<WebSearchBrowserLoader | undefined>;
56
+ export type { BrowserFallbackOptions, LoadedHtmlPage, WebSearchBrowserLoader };
@@ -0,0 +1,41 @@
1
+ import { setWebSearchBrowserLoader } from "./browser-loader.js";
2
+ import type { FetchImpl } from "./providers/base.js";
3
+ export { setWebSearchBrowserLoader };
4
+ /** HTML plus the response status and final URL after redirects or browser navigation. */
5
+ export interface LoadedHtmlPage {
6
+ html: string;
7
+ status: number;
8
+ url: string;
9
+ }
10
+ export interface BrowserFallbackOptions {
11
+ homeUrl?: string;
12
+ ready?: {
13
+ selector: string;
14
+ timeoutMs: number;
15
+ };
16
+ afterNavigation?: (page: unknown, signal: AbortSignal) => Promise<void>;
17
+ shouldFallback: (page: LoadedHtmlPage) => boolean;
18
+ attempts?: number;
19
+ retryDelayMs?: number;
20
+ }
21
+ /** Controls a browser-profiled fetch and its optional headless-browser fallback. */
22
+ export interface BrowserFetchOptions {
23
+ fetch?: FetchImpl;
24
+ signal: AbortSignal;
25
+ timeoutMs?: number;
26
+ randomizeHeaders?: boolean;
27
+ referer?: string;
28
+ init?: Omit<RequestInit, "headers" | "signal">;
29
+ headers?: Readonly<Record<string, string>>;
30
+ browser?: BrowserFallbackOptions;
31
+ }
32
+ /**
33
+ * Headless-browser loader. Receives the target URL and the provider's
34
+ * fallback options and must return the loaded page. The registry lives in
35
+ * `browser-loader.ts`, which ships a default puppeteer-core loader probed
36
+ * from the local Chrome/Chromium install; an explicit registration always
37
+ * takes precedence over the default.
38
+ */
39
+ export type WebSearchBrowserLoader = (url: string, options: BrowserFallbackOptions, signal: AbortSignal, timeoutMs: number) => Promise<LoadedHtmlPage>;
40
+ /** Fetch with a fresh browser profile, escalating rejected production responses to the installed headless browser. */
41
+ export declare function browserFetch(url: string, options: BrowserFetchOptions): Promise<LoadedHtmlPage>;
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Environment-variable credential resolution for web search providers.
3
+ *
4
+ * Replaces oh-my-pi's AuthStorage/SQLite credential broker: AutoRAG keeps
5
+ * secrets external (environment only — never written to config files, logs,
6
+ * or diagnostics), so a provider is "available" exactly when its documented
7
+ * env var holds a non-empty value.
8
+ */
9
+ /** Return the first non-empty environment value among `names`, or undefined. */
10
+ export declare function envCredential(...names: string[]): string | undefined;
@@ -0,0 +1,30 @@
1
+ /**
2
+ * LLM-facing formatting for web search responses.
3
+ *
4
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/index.ts`
5
+ * (formatForLLM) and its prompts/tools + prompts/system web-search markdown.
6
+ */
7
+ import type { SearchResponse } from "./types.js";
8
+ /**
9
+ * System prompt passed to LLM-mediated providers. Kept for forward
10
+ * compatibility — every provider AutoRAG ships today returns sources
11
+ * directly, but the SearchParams contract carries it.
12
+ */
13
+ export declare const WEB_SEARCH_SYSTEM_PROMPT = "Web research assistant: accurate, well-sourced, comprehensive answers.\n\n<priorities>\n1. Accuracy > speed; verify claims across multiple sources when possible.\n2. Primary > secondary: official docs, papers, announcements > blog summaries.\n3. Recency matters: note publication dates; prefer recent sources for time-sensitive topics.\n4. Uncertainty: distinguish confirmed facts from inferences.\n</priorities>\n\n<synthesis>\n- Direct answer first; then supporting evidence.\n- Quote or paraphrase specific sources; no vague attributions.\n- Source conflicts: acknowledge discrepancy; identify the more authoritative source.\n- Technical topics: prefer official documentation and specifications.\n- News/events: prefer primary reporting over aggregators.\n- Concrete data: version numbers, dates, exact figures, code snippets, specific examples.\n</synthesis>\n\n<format>\n- Thorough, in-depth coverage with specific evidence; no surface-level summaries.\n- Omit filler and unnecessary hedging; do NOT sacrifice detail for brevity.\n- Include publication dates when recency affects relevance.\n- Clear sections for multiple aspects.\n- Cite sources inline using provided search results.\n</format>";
14
+ /**
15
+ * Tool description shared by the `web_search` agent tool and CLI surfaces.
16
+ * Ported from oh-my-pi prompts/tools/web-search.md.
17
+ */
18
+ export declare const WEB_SEARCH_TOOL_DESCRIPTION = "Web search: current information beyond knowledge cutoff.\n\n- SHOULD prefer primary sources (papers, official docs); corroborate key claims with multiple sources.\n- MUST link cited sources in final response.\n- NEVER use for programmatically accessible content or known URLs (GitHub repos/issues, known arXiv papers, Wikipedia pages, official docs) \u2014 use web_fetch on the URL directly.\n- `query`: every provider supports Google-style `site:`/`-site:`, `after:`/`before:` (YYYY-MM-DD), `inurl:`, `intitle:`, `filetype:`, `\"exact phrase\"`, `-term`, `OR`. Map constraints to native filters when available; otherwise results are filtered leniently. If a constraint matches nothing, it is relaxed and reported; zero results are never forced.";
19
+ /** Format an age in seconds as a compact relative string ("45s", "2h", "3d"). */
20
+ export declare function formatAge(ageSeconds: number | undefined): string;
21
+ /** Format response for LLM consumption. `notes` lead the output (e.g. relaxed-constraint warnings). */
22
+ export declare function formatForLLM(response: SearchResponse, notes?: readonly string[]): string;
23
+ /**
24
+ * `true` when the response carries an actual result the caller can read: an
25
+ * answer, a source, or a citation. Search-attempt metadata (`searchQueries`,
26
+ * `relatedQuestions`) is an annotation on a real result, never a result — a
27
+ * provider that only reports "I ran a search" must not win the chain while a
28
+ * healthy fallback would have returned sources.
29
+ */
30
+ export declare function hasRenderableSearchContent(response: SearchResponse): boolean;
@@ -0,0 +1,46 @@
1
+ import type { ModelNativeSearchAuth } from "./model-auth.js";
2
+ import { type SearchProviderId, type SearchResponse } from "./types.js";
3
+ export interface WebSearchQueryParams {
4
+ query: string;
5
+ recency?: "day" | "week" | "month" | "year";
6
+ limit?: number;
7
+ max_tokens?: number;
8
+ temperature?: number;
9
+ num_search_results?: number;
10
+ provider?: SearchProviderId | "auto";
11
+ }
12
+ export interface WebSearchResultDetails {
13
+ response: SearchResponse;
14
+ error?: string;
15
+ }
16
+ export interface WebSearchExecuteResult {
17
+ content: Array<{
18
+ type: "text";
19
+ text: string;
20
+ }>;
21
+ details: WebSearchResultDetails;
22
+ }
23
+ export interface WebSearchExecuteOptions {
24
+ signal?: AbortSignal;
25
+ /** Per-provider transport hard timeout in milliseconds (default 60s, capped at 300s). */
26
+ timeoutMs?: number;
27
+ /**
28
+ * Budget for the whole fallback chain in milliseconds (default: twice the
29
+ * per-provider timeout). Without it a chain of stalled providers costs one
30
+ * per-provider timeout each and the tool call looks hung.
31
+ */
32
+ totalTimeoutMs?: number;
33
+ /** Prioritized providers for this request; unlisted providers keep their built-in order. */
34
+ order?: readonly SearchProviderId[];
35
+ /** Providers this request must never use, including fallbacks. */
36
+ exclude?: readonly SearchProviderId[];
37
+ /** Agent model credential for the model-native providers (per request, never global). */
38
+ modelAuth?: ModelNativeSearchAuth;
39
+ /** Transport injection for tests/proxies; forwarded to providers that accept it. */
40
+ fetch?: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
41
+ }
42
+ /** Execute a web search through the provider fallback chain. */
43
+ export declare function executeWebSearch(params: WebSearchQueryParams, options?: WebSearchExecuteOptions): Promise<WebSearchExecuteResult>;
44
+ export { getSearchProvider, type SearchProviderRouting } from "./provider.js";
45
+ export type { SearchProviderId, SearchResponse } from "./types.js";
46
+ export { isSearchProviderId, isSearchProviderPreference } from "./types.js";
@@ -0,0 +1,36 @@
1
+ /** Agent model credential injected for model-native web search. */
2
+ export interface ModelNativeSearchAuth {
3
+ /** pi model provider id, e.g. "anthropic" | "openai" | "google" | "xai". */
4
+ readonly provider: string;
5
+ readonly apiKey: string;
6
+ readonly baseUrl?: string;
7
+ readonly modelId?: string;
8
+ }
9
+ /** Credential view consumed by the model-native search providers. */
10
+ export interface ModelNativeCredential {
11
+ readonly apiKey: string;
12
+ readonly baseUrl?: string;
13
+ readonly modelId?: string;
14
+ readonly source: "injected" | "env";
15
+ }
16
+ /** Model-native search provider ids as they appear in the chain. */
17
+ export type ModelNativeSearchProviderId = "gemini" | "anthropic" | "codex" | "xai";
18
+ /**
19
+ * Resolve the credential a model-native search provider should use for this
20
+ * request: the caller's agent credential when its provider family matches,
21
+ * else the provider's conventional model environment key. The credential is
22
+ * never stored in module state — it travels with the request so two agents
23
+ * in one process cannot overwrite each other's key or base URL.
24
+ */
25
+ export declare function resolveModelNativeCredential(searchProvider: ModelNativeSearchProviderId, auth?: ModelNativeSearchAuth): ModelNativeCredential | undefined;
26
+ /**
27
+ * Map a resolved agent model (provider + apiKey) onto the injectable auth
28
+ * shape. Returns undefined for providers with no model-native search route
29
+ * (e.g. openrouter, local gateways) or when no apiKey was resolved.
30
+ */
31
+ export declare function modelNativeAuthFromAgentModel(resolved: {
32
+ readonly provider: string;
33
+ readonly apiKey?: string;
34
+ readonly baseUrl?: string;
35
+ readonly modelId?: string;
36
+ }): ModelNativeSearchAuth | undefined;
@@ -0,0 +1,62 @@
1
+ import type { SearchAvailabilityContext, SearchProvider, SearchProviderContract } from "./providers/base.js";
2
+ import { type SearchProviderId } from "./types.js";
3
+ export type { FetchImpl, SearchAvailabilityContext, SearchParams, SearchProviderContract, } from "./providers/base.js";
4
+ export { SearchProvider } from "./providers/base.js";
5
+ export { SEARCH_PROVIDER_ORDER } from "./types.js";
6
+ /** Cheap, sync metadata accessor — never triggers a provider load. */
7
+ export declare function getSearchProviderLabel(id: SearchProviderId): string;
8
+ /** Format one provider failure for the user-facing fallback summary. */
9
+ export declare function formatSearchProviderFailure(error: unknown, provider: Pick<SearchProvider, "id" | "label">): string;
10
+ /** Format the ordered provider fallback failures for terminal/tool output. */
11
+ export declare function formatSearchProviderFailures(failures: readonly {
12
+ provider: Pick<SearchProvider, "id" | "label">;
13
+ error: unknown;
14
+ }[]): string;
15
+ /**
16
+ * Resolve and cache a provider instance. First call for a given id loads the
17
+ * underlying module; subsequent calls return the cached singleton.
18
+ * Instances registered through {@link registerSearchProvider} take
19
+ * precedence over lazy module loads (host embedding, tests).
20
+ */
21
+ export declare function getSearchProvider(id: SearchProviderId): Promise<SearchProviderContract>;
22
+ /**
23
+ * Register (or replace) a provider instance, bypassing the lazy module load.
24
+ * Test seam and host-embedding hook; production provider modules register
25
+ * nothing themselves.
26
+ */
27
+ export declare function registerSearchProvider(provider: SearchProviderContract): void;
28
+ /** Drop every registered/cached provider instance (tests). */
29
+ export declare function clearRegisteredSearchProviders(): void;
30
+ /**
31
+ * Per-request routing. Configuration never mutates module state: the caller
32
+ * (agent tool, CLI) owns its own routing and passes it down, so two agents
33
+ * in one process cannot rewrite each other's provider order or exclusions.
34
+ */
35
+ export interface SearchProviderRouting {
36
+ /** Prioritized providers; unlisted providers keep their built-in relative order. */
37
+ readonly order?: readonly SearchProviderId[];
38
+ /** Providers this request must never use, including fallbacks. */
39
+ readonly exclude?: readonly SearchProviderId[];
40
+ /** Terminal-first provider that bypasses exclusion (an explicit `provider` argument). */
41
+ readonly forcedProvider?: SearchProviderId;
42
+ }
43
+ /** `true` when `routing` excludes `id` from web search (auto chain and the Public Web fan-out). */
44
+ export declare function isSearchProviderExcluded(id: SearchProviderId, exclude?: readonly SearchProviderId[]): boolean;
45
+ export interface SearchProviderCandidate {
46
+ id: SearchProviderId;
47
+ explicit: boolean;
48
+ }
49
+ /**
50
+ * Return provider candidates in fallback order without loading their modules.
51
+ * `routing.forcedProvider` is terminal-first and bypasses exclusion; providers
52
+ * the caller listed in `routing.order` are treated as explicit selections and
53
+ * resolve through `isExplicitlyAvailable`. Invalid ids are ignored defensively.
54
+ */
55
+ export declare function resolveProviderCandidates(routing?: SearchProviderRouting): SearchProviderCandidate[];
56
+ /**
57
+ * Resolve the complete available provider chain.
58
+ *
59
+ * This compatibility helper loads every candidate. Search execution should use
60
+ * {@link resolveProviderCandidates} so fallback modules load only when reached.
61
+ */
62
+ export declare function resolveProviderChain(routing?: SearchProviderRouting, context?: SearchAvailabilityContext): Promise<SearchProviderContract[]>;
@@ -0,0 +1,9 @@
1
+ import { type SearchResponse } from "../types.js";
2
+ import type { SearchAvailabilityContext, SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export declare class AnthropicProvider extends SearchProvider {
5
+ readonly id: "anthropic";
6
+ readonly label = "Anthropic";
7
+ isAvailable(context?: SearchAvailabilityContext): boolean;
8
+ search(params: SearchParams): Promise<SearchResponse>;
9
+ }
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Shared web search provider contract.
3
+ *
4
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/providers/base.ts`
5
+ * with one adaptation: oh-my-pi resolves credentials through its AuthStorage
6
+ * broker; AutoRAG providers read their documented environment variables
7
+ * directly (see `../credentials.ts`), so `isAvailable` takes no arguments.
8
+ */
9
+ import type { ModelNativeSearchAuth } from "../model-auth.js";
10
+ import type { StructuredQuery } from "../query.js";
11
+ import type { SearchProviderId, SearchResponse } from "../types.js";
12
+ /** Minimal fetch signature providers accept for transport injection (tests, proxies). */
13
+ export type FetchImpl = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
14
+ /**
15
+ * Shared web search parameters passed to providers.
16
+ */
17
+ export interface SearchParams {
18
+ query: string;
19
+ /**
20
+ * Structured view of `query`, parsed once by the search pipeline:
21
+ * Google-style directives (`site:`, `before:`/`after:`, `inurl:`,
22
+ * `intitle:`, `filetype:`, quoted phrases, `OR` groups, `-exclusions`)
23
+ * extracted into fields.
24
+ *
25
+ * Providers SHOULD map constraints onto native API parameters
26
+ * (domain/date filters) or engine query syntax (`formatQuery`) where the
27
+ * upstream supports them, and lean lenient otherwise: the pipeline
28
+ * post-filters every response with `applyQueryConstraints`, which
29
+ * relaxes any constraint that would eliminate all results — so a
30
+ * best-effort search always beats an empty one. When absent (direct
31
+ * provider calls), parse with `parseSearchQuery(params.query)`.
32
+ */
33
+ parsedQuery?: StructuredQuery;
34
+ limit?: number;
35
+ /**
36
+ * Temporal filter narrowing results to the specified time window.
37
+ *
38
+ * Providers MUST interpret this as a pure time filter. Providers MUST NOT
39
+ * use recency as an implicit signal to change topic scope, content domain,
40
+ * or ranking strategy. Providers that do not support temporal filtering
41
+ * MUST ignore this field silently; they MUST NOT approximate it by
42
+ * rewriting the query or altering any other request parameter.
43
+ */
44
+ recency?: "day" | "week" | "month" | "year";
45
+ systemPrompt?: string;
46
+ signal?: AbortSignal;
47
+ /** Hard timeout for this provider's search transport, in milliseconds. */
48
+ timeoutMs?: number;
49
+ fetch?: FetchImpl;
50
+ maxOutputTokens?: number;
51
+ numSearchResults?: number;
52
+ temperature?: number;
53
+ /**
54
+ * Agent model credential for the model-native providers, carried per
55
+ * request so concurrent agents never share one process-wide key.
56
+ */
57
+ modelAuth?: ModelNativeSearchAuth;
58
+ /** Providers this request must not use, including inside the `public` fan-out. */
59
+ excludedProviders?: readonly SearchProviderId[];
60
+ }
61
+ /** Per-request context consulted by provider availability checks. */
62
+ export interface SearchAvailabilityContext {
63
+ readonly modelAuth?: ModelNativeSearchAuth;
64
+ }
65
+ /**
66
+ * Structural provider contract accepted by the registry. Production
67
+ * providers extend {@link SearchProvider} (which supplies the
68
+ * `isExplicitlyAvailable` default); test fakes and host embeddings may be
69
+ * plain objects matching this shape.
70
+ */
71
+ export interface SearchProviderContract {
72
+ readonly id: SearchProviderId;
73
+ readonly label: string;
74
+ isAvailable(context?: SearchAvailabilityContext): Promise<boolean> | boolean;
75
+ isExplicitlyAvailable?(context?: SearchAvailabilityContext): Promise<boolean> | boolean;
76
+ search(params: SearchParams): Promise<SearchResponse>;
77
+ }
78
+ /** Base class for web search providers. */
79
+ export declare abstract class SearchProvider implements SearchProviderContract {
80
+ abstract readonly id: SearchProviderId;
81
+ abstract readonly label: string;
82
+ /**
83
+ * Indicates whether this provider has the credentials/config it needs to
84
+ * service a request right now. AutoRAG providers consult their documented
85
+ * environment variables; credential-free providers always return true.
86
+ *
87
+ * Drives auto-chain admission: providers that return `false` are skipped
88
+ * when the chain walks the order. Explicit selection uses
89
+ * {@link isExplicitlyAvailable} instead.
90
+ */
91
+ abstract isAvailable(context?: SearchAvailabilityContext): Promise<boolean> | boolean;
92
+ /**
93
+ * Returns `true` when this provider should run when the user explicitly
94
+ * selects it, even if {@link isAvailable} would reject it for the auto
95
+ * chain. Providers that ship an unauthenticated fallback override this so
96
+ * explicit selection still routes through the fallback rather than
97
+ * silently falling back to another provider.
98
+ *
99
+ * Defaults to mirroring {@link isAvailable}.
100
+ */
101
+ isExplicitlyAvailable(context?: SearchAvailabilityContext): Promise<boolean> | boolean;
102
+ /**
103
+ * Execute a search.
104
+ */
105
+ abstract search(params: SearchParams): Promise<SearchResponse>;
106
+ }
@@ -0,0 +1,9 @@
1
+ import type { SearchResponse } from "../types.js";
2
+ import type { SearchAvailabilityContext, SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export declare class CodexProvider extends SearchProvider {
5
+ readonly id: "codex";
6
+ readonly label = "OpenAI";
7
+ isAvailable(context?: SearchAvailabilityContext): boolean;
8
+ search(params: SearchParams): Promise<SearchResponse>;
9
+ }
@@ -0,0 +1,22 @@
1
+ /**
2
+ * DuckDuckGo's no-JS HTML search frontend. POST `q=…` to receive a static
3
+ * results page we can parse without a real browser. The Instant Answer API
4
+ * (`api.duckduckgo.com`) was tried first but it only returns content for
5
+ * Wikipedia/Wolfram-Alpha-style topics — empty for the vast majority of
6
+ * agent queries (see #3799).
7
+ *
8
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
9
+ * `packages/coding-agent/src/web/search/providers/duckduckgo.ts`.
10
+ */
11
+ import type { SearchProviderId, SearchResponse } from "../types.js";
12
+ import type { SearchParams } from "./base.js";
13
+ import { SearchProvider } from "./base.js";
14
+ export declare function localeToKl(lang: string | undefined): string | undefined;
15
+ export declare function searchDuckDuckGo(params: SearchParams): Promise<SearchResponse>;
16
+ export declare class DuckDuckGoProvider extends SearchProvider {
17
+ readonly id: SearchProviderId;
18
+ readonly label = "DuckDuckGo";
19
+ isAvailable(): boolean;
20
+ isExplicitlyAvailable(): boolean;
21
+ search(params: SearchParams): Promise<SearchResponse>;
22
+ }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Ecosia serves a server-rendered Vue/Nuxt results page (no `__NUXT_DATA__`
3
+ * JSON island — probed 2026-07), so both load paths parse the same markup:
4
+ * `<article data-test-id="organic-result">` blocks whose title anchor carries
5
+ * the final target URL directly (no redirect wrapper). The site fronts search
6
+ * with Cloudflare. Requests start with a browser-profiled fetch and escalate
7
+ * to the shared stealth browser only when the response is blocked or fails.
8
+ *
9
+ * Recency is ignored: Ecosia's web results expose no date filter in the UI
10
+ * and the legacy Bing-era `freshness` param is a server-side no-op (verified
11
+ * live), so per the {@link SearchParams.recency} contract the field must not
12
+ * be approximated.
13
+ *
14
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
15
+ * `packages/coding-agent/src/web/search/providers/ecosia.ts`.
16
+ */
17
+ import type { SearchProviderId, SearchResponse } from "../types.js";
18
+ import type { SearchParams } from "./base.js";
19
+ import { SearchProvider } from "./base.js";
20
+ /** Execute an Ecosia web search and parse the server-rendered result page. */
21
+ export declare function searchEcosia(params: SearchParams): Promise<SearchResponse>;
22
+ /** Search provider for Ecosia (no API key required). */
23
+ export declare class EcosiaProvider extends SearchProvider {
24
+ readonly id: SearchProviderId;
25
+ readonly label = "Ecosia";
26
+ isAvailable(): boolean;
27
+ isExplicitlyAvailable(): boolean;
28
+ search(params: SearchParams): Promise<SearchResponse>;
29
+ }
@@ -0,0 +1,9 @@
1
+ import { type SearchResponse } from "../types.js";
2
+ import type { SearchAvailabilityContext, SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export declare class GeminiProvider extends SearchProvider {
5
+ readonly id: "gemini";
6
+ readonly label = "Gemini";
7
+ isAvailable(context?: SearchAvailabilityContext): boolean;
8
+ search(params: SearchParams): Promise<SearchResponse>;
9
+ }
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Google's server-rendered search results page (the `udm=14` "Web" tab
3
+ * variant), scraped via browser-profiled fetch with a headless-browser
4
+ * fallback. Google blocks automated requests aggressively, so the provider
5
+ * detects traffic challenges and JavaScript challenges and surfaces them as
6
+ * provider-tagged 429s so the chain advances to the next engine.
7
+ *
8
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
9
+ * `packages/coding-agent/src/web/search/providers/google.ts`.
10
+ */
11
+ import type { SearchProviderId, SearchResponse } from "../types.js";
12
+ import type { SearchParams } from "./base.js";
13
+ import { SearchProvider } from "./base.js";
14
+ /** Execute a Google web search with fetch-first loading and a headless-browser fallback. */
15
+ export declare function searchGoogle(params: SearchParams): Promise<SearchResponse>;
16
+ /** Fetch-first Google Search provider with a headless-browser fallback; no API key is required. */
17
+ export declare class GoogleProvider extends SearchProvider {
18
+ readonly id: SearchProviderId;
19
+ readonly label = "Google";
20
+ isAvailable(): boolean;
21
+ search(params: SearchParams): Promise<SearchResponse>;
22
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Mojeek's independent index served as a server-rendered HTML results page.
3
+ * Each organic result renders as `ul.results-standard > li` with the title
4
+ * in `h2 > a.title` (href is the direct target URL) and the preview text in
5
+ * `p.s`. The site fronts with an ALTCHA proof-of-work captcha; the
6
+ * headless-browser fallback solves it when available, otherwise the fetch-only
7
+ * path surfaces the wall as a provider-tagged 429.
8
+ *
9
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT)
10
+ * `packages/coding-agent/src/web/search/providers/mojeek.ts`.
11
+ */
12
+ import type { SearchProviderId, SearchResponse } from "../types.js";
13
+ import type { SearchParams } from "./base.js";
14
+ import { SearchProvider } from "./base.js";
15
+ /** Execute a Mojeek web search against the standard HTML results page. */
16
+ export declare function searchMojeek(params: SearchParams): Promise<SearchResponse>;
17
+ /** Search provider for Mojeek (independent index, no API key required). */
18
+ export declare class MojeekProvider extends SearchProvider {
19
+ readonly id: SearchProviderId;
20
+ readonly label = "Mojeek";
21
+ isAvailable(): boolean;
22
+ isExplicitlyAvailable(): boolean;
23
+ search(params: SearchParams): Promise<SearchResponse>;
24
+ }
@@ -0,0 +1,10 @@
1
+ import { type SearchResponse } from "../types.js";
2
+ import type { SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export declare class ParallelProvider extends SearchProvider {
5
+ readonly id: "parallel";
6
+ readonly label = "Parallel";
7
+ /** The public MCP endpoint needs no credential of any kind. */
8
+ isAvailable(): boolean;
9
+ search(params: SearchParams): Promise<SearchResponse>;
10
+ }
@@ -0,0 +1,10 @@
1
+ import { type SearchResponse } from "../types.js";
2
+ import type { SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export declare class PerplexityProvider extends SearchProvider {
5
+ readonly id: "perplexity";
6
+ readonly label = "Perplexity";
7
+ /** The anonymous ask endpoint needs no credential of any kind. */
8
+ isAvailable(): boolean;
9
+ search(params: SearchParams): Promise<SearchResponse>;
10
+ }
@@ -0,0 +1,15 @@
1
+ import { type SearchResponse } from "../types.js";
2
+ import type { SearchParams } from "./base.js";
3
+ import { SearchProvider } from "./base.js";
4
+ export interface PublicWebDeadlines {
5
+ softMs?: number;
6
+ hardMs?: number;
7
+ }
8
+ export declare function searchPublicWeb(params: SearchParams, deadlines?: PublicWebDeadlines): Promise<SearchResponse>;
9
+ export declare class PublicWebProvider extends SearchProvider {
10
+ readonly id: "public";
11
+ readonly label = "Public Web";
12
+ isAvailable(): boolean;
13
+ isExplicitlyAvailable(): boolean;
14
+ search(params: SearchParams): Promise<SearchResponse>;
15
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * Shared OpenAI-Responses-API web search transport.
3
+ *
4
+ * Both `codex` (api.openai.com) and `xai` (api.x.ai) speak the Responses API
5
+ * shape with a hosted `web_search` tool, so their transports share one
6
+ * implementation parameterized by provider id, base URL, and default model.
7
+ *
8
+ * Ported from oh-my-pi (can1357/oh-my-pi, MIT) `web/search/providers/codex.ts`
9
+ * and `xai.ts`. Dropped: oh-my-pi's ChatGPT-backend SSE stream (it speaks an
10
+ * undocumented request shape tied to ChatGPT OAuth) — AutoRAG uses the
11
+ * documented non-streaming Responses API with a plain API key via
12
+ * `../model-auth.ts`.
13
+ */
14
+ import { type ModelNativeSearchProviderId } from "../model-auth.js";
15
+ import { type SearchResponse } from "../types.js";
16
+ import type { SearchParams } from "./base.js";
17
+ export interface ResponsesApiSearchOptions {
18
+ /** Search provider id ("codex" | "xai") for credential resolution and errors. */
19
+ readonly providerId: Extract<ModelNativeSearchProviderId, "codex" | "xai">;
20
+ readonly defaultBaseUrl: string;
21
+ readonly defaultModel: string;
22
+ readonly modelEnvVar: string;
23
+ readonly missingCredentialMessage: string;
24
+ }
25
+ /** Execute a Responses-API web search for the given provider configuration. */
26
+ export declare function searchResponsesApi(options: ResponsesApiSearchOptions, params: SearchParams): Promise<SearchResponse>;