@demigodmode/pi-web-agent 1.11.0 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +2 -0
  3. package/dist/backends/config.d.ts +13 -0
  4. package/dist/backends/config.js +44 -1
  5. package/dist/backends/doctor.js +27 -29
  6. package/dist/backends/factory.d.ts +18 -7
  7. package/dist/backends/factory.js +126 -99
  8. package/dist/backends/failure.d.ts +11 -0
  9. package/dist/backends/failure.js +34 -0
  10. package/dist/backends/fallback-policy.d.ts +31 -0
  11. package/dist/backends/fallback-policy.js +239 -0
  12. package/dist/backends/provider-failure.d.ts +21 -0
  13. package/dist/backends/provider-failure.js +110 -0
  14. package/dist/backends/provider-health.d.ts +29 -0
  15. package/dist/backends/provider-health.js +49 -0
  16. package/dist/commands/web-agent-config.d.ts +14 -1
  17. package/dist/commands/web-agent-config.js +75 -3
  18. package/dist/extension.js +47 -3
  19. package/dist/extract/bot-check.d.ts +1 -0
  20. package/dist/extract/bot-check.js +10 -0
  21. package/dist/extract/readability.d.ts +4 -0
  22. package/dist/extract/readability.js +67 -3
  23. package/dist/extract/section-selector.d.ts +14 -0
  24. package/dist/extract/section-selector.js +233 -0
  25. package/dist/fetch/destination-policy.d.ts +32 -0
  26. package/dist/fetch/destination-policy.js +24 -0
  27. package/dist/fetch/firecrawl-fetch.d.ts +1 -1
  28. package/dist/fetch/firecrawl-fetch.js +74 -46
  29. package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
  30. package/dist/fetch/guard-proxy-fetch.js +82 -0
  31. package/dist/fetch/guard-proxy.d.ts +58 -0
  32. package/dist/fetch/guard-proxy.js +420 -0
  33. package/dist/fetch/guarded-fetch.d.ts +7 -0
  34. package/dist/fetch/guarded-fetch.js +75 -0
  35. package/dist/fetch/headless-fetch.d.ts +11 -2
  36. package/dist/fetch/headless-fetch.js +190 -13
  37. package/dist/fetch/http-fetch.d.ts +1 -1
  38. package/dist/fetch/http-fetch.js +29 -8
  39. package/dist/fetch/network-guard.d.ts +82 -0
  40. package/dist/fetch/network-guard.js +275 -0
  41. package/dist/orchestration/answer-synthesizer.js +2 -0
  42. package/dist/orchestration/evidence-quality.d.ts +3 -2
  43. package/dist/orchestration/evidence-quality.js +2 -1
  44. package/dist/orchestration/index.d.ts +26 -7
  45. package/dist/orchestration/index.js +9 -2
  46. package/dist/orchestration/research-orchestrator.d.ts +23 -7
  47. package/dist/orchestration/research-orchestrator.js +60 -26
  48. package/dist/orchestration/research-types.d.ts +13 -1
  49. package/dist/orchestration/research-worker.d.ts +2 -4
  50. package/dist/orchestration/research-worker.js +51 -15
  51. package/dist/orchestration/stop-decider.js +3 -1
  52. package/dist/presentation/config-store.js +6 -0
  53. package/dist/presentation/explore-presentation.js +3 -1
  54. package/dist/presentation/fetch-presentation.js +16 -9
  55. package/dist/presentation/search-presentation.d.ts +2 -1
  56. package/dist/presentation/search-presentation.js +13 -1
  57. package/dist/readers/resolver.d.ts +3 -7
  58. package/dist/search/brave.d.ts +1 -2
  59. package/dist/search/brave.js +23 -80
  60. package/dist/search/duckduckgo.d.ts +7 -3
  61. package/dist/search/duckduckgo.js +17 -18
  62. package/dist/search/exa.d.ts +1 -2
  63. package/dist/search/exa.js +15 -76
  64. package/dist/search/fanout.d.ts +12 -0
  65. package/dist/search/fanout.js +86 -47
  66. package/dist/search/json-provider.d.ts +32 -0
  67. package/dist/search/json-provider.js +76 -0
  68. package/dist/search/searxng.d.ts +1 -2
  69. package/dist/search/searxng.js +15 -57
  70. package/dist/search/tavily.d.ts +1 -2
  71. package/dist/search/tavily.js +17 -74
  72. package/dist/search/youcom.d.ts +4 -2
  73. package/dist/search/youcom.js +49 -75
  74. package/dist/tools/web-explore.d.ts +9 -0
  75. package/dist/tools/web-explore.js +16 -2
  76. package/dist/tools/web-fetch-headless.d.ts +3 -5
  77. package/dist/tools/web-fetch-headless.js +3 -3
  78. package/dist/tools/web-fetch.d.ts +3 -5
  79. package/dist/tools/web-fetch.js +3 -3
  80. package/dist/tools/web-search.js +41 -103
  81. package/dist/types.d.ts +48 -0
  82. package/package.json +3 -3
@@ -1,4 +1,4 @@
1
- import type { SearchProviderName } from '../types.js';
1
+ import type { Attempt, SearchProviderName } from '../types.js';
2
2
  export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
3
3
  export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
4
4
  export type ResearchEvidence = {
@@ -27,6 +27,13 @@ export type ResearchWorkerResult = {
27
27
  exhaustedBudget: boolean;
28
28
  fanoutProviders?: SearchProviderName[];
29
29
  fanoutSkipped?: SearchProviderName[];
30
+ searchCoveragePartial?: boolean;
31
+ searchAttempts?: Attempt[];
32
+ fetchAttempts?: Attempt[];
33
+ terminalFailure?: {
34
+ code: string;
35
+ message: string;
36
+ };
30
37
  };
31
38
  export type ResearchRunMetadata = {
32
39
  searchPasses: number;
@@ -35,6 +42,11 @@ export type ResearchRunMetadata = {
35
42
  exhaustedBudget: boolean;
36
43
  fanoutProviders?: SearchProviderName[];
37
44
  fanoutSkipped?: SearchProviderName[];
45
+ searchCoveragePartial?: boolean;
46
+ terminalFailure?: {
47
+ code: string;
48
+ message: string;
49
+ };
38
50
  };
39
51
  export type ResearchOrchestratorDecision = {
40
52
  action: 'answer';
@@ -1,12 +1,10 @@
1
- import type { WebFetchResponse, WebSearchResponse } from '../types.js';
1
+ import type { ResearchFetchInput, WebFetchResponse, WebSearchResponse } from '../types.js';
2
2
  import type { ResearchWorkerResult } from './research-types.js';
3
3
  export declare function createResearchWorker({ search, fetchPage }: {
4
4
  search: (input: {
5
5
  query: string;
6
6
  }) => Promise<WebSearchResponse>;
7
- fetchPage: (input: {
8
- url: string;
9
- }) => Promise<WebFetchResponse>;
7
+ fetchPage: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
10
8
  }): {
11
9
  run({ query, maxSearchRounds, maxFetches }: {
12
10
  query: string;
@@ -1,22 +1,24 @@
1
+ import { failureOf, isTerminalFailure } from '../backends/failure.js';
1
2
  import { selectCandidates } from './candidate-selector.js';
2
3
  import { classifySourceProfile } from './source-profile.js';
4
+ import { selectRelevantExcerpt } from '../extract/section-selector.js';
5
+ import { hasBotCheckContent } from '../extract/bot-check.js';
3
6
  function classifySource(url) {
4
7
  return classifySourceProfile(url).sourceKind;
5
8
  }
6
- function summarizeText(text, maxLength = 180) {
7
- return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
8
- }
9
9
  function isReaderMethod(method) {
10
10
  return method === 'github' || method === 'pdf' || method === 'youtube';
11
11
  }
12
- function isBotCheckContent({ title = '', text }) {
13
- return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
12
+ function isBotCheckContent({ title = '', text, botCheck }) {
13
+ if (botCheck)
14
+ return true;
15
+ return hasBotCheckContent(`${title}\n${text}`);
14
16
  }
15
- function evidenceFromFetch(fetched, fallbackTitle) {
17
+ function evidenceFromFetch(fetched, fallbackTitle, query) {
16
18
  const content = fetched.content;
17
19
  if (fetched.status !== 'ok' || !content)
18
20
  return null;
19
- if (isBotCheckContent({ title: content.title, text: content.text }))
21
+ if (isBotCheckContent({ title: content.title, text: content.text, botCheck: content.botCheck }))
20
22
  return null;
21
23
  // A successful reader read with usable text is primary content, exempt from the
22
24
  // package-page filter below.
@@ -39,14 +41,14 @@ function evidenceFromFetch(fetched, fallbackTitle) {
39
41
  url: fetched.url,
40
42
  sourceKind,
41
43
  method: fetched.metadata.method,
42
- summary: summarizeText(content.text),
43
- supports: [summarizeText(content.text, 120)]
44
+ summary: selectRelevantExcerpt(content.text, query, 180),
45
+ supports: [selectRelevantExcerpt(content.text, query, 120)]
44
46
  };
45
47
  }
46
48
  function lowValueOutcomeFromFetch(fetched) {
47
49
  if (fetched.status !== 'ok' || !fetched.content)
48
50
  return null;
49
- if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text })) {
51
+ if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text, botCheck: fetched.content.botCheck })) {
50
52
  return {
51
53
  kind: 'bot-check',
52
54
  url: fetched.url,
@@ -80,6 +82,22 @@ export function createResearchWorker({ search, fetchPage }) {
80
82
  };
81
83
  }
82
84
  const searchResult = await search({ query });
85
+ const searchCoveragePartial = searchResult.metadata.coverage?.partial === true;
86
+ const searchAttempts = searchResult.metadata.attempts;
87
+ if (isTerminalFailure(failureOf(searchResult))) {
88
+ return {
89
+ searchQueries,
90
+ evidence,
91
+ gaps: [],
92
+ lowValueOutcomes,
93
+ exhaustedBudget: false,
94
+ searchAttempts,
95
+ terminalFailure: {
96
+ code: searchResult.error?.code ?? 'SEARCH_FAILED',
97
+ message: `${searchResult.error?.message ?? 'Search failed.'} (${searchResult.error?.failure?.kind})`
98
+ }
99
+ };
100
+ }
83
101
  const fanoutProviders = searchResult.metadata.fanout?.providers;
84
102
  const fanoutSkipped = searchResult.metadata.fanout?.skipped;
85
103
  if (searchResult.status !== 'ok') {
@@ -96,7 +114,9 @@ export function createResearchWorker({ search, fetchPage }) {
96
114
  suggestedHeadlessUrl,
97
115
  exhaustedBudget: false,
98
116
  fanoutProviders,
99
- fanoutSkipped
117
+ fanoutSkipped,
118
+ searchCoveragePartial,
119
+ searchAttempts
100
120
  };
101
121
  }
102
122
  if (searchResult.results.length === 0) {
@@ -113,7 +133,9 @@ export function createResearchWorker({ search, fetchPage }) {
113
133
  suggestedHeadlessUrl,
114
134
  exhaustedBudget: false,
115
135
  fanoutProviders,
116
- fanoutSkipped
136
+ fanoutSkipped,
137
+ searchCoveragePartial,
138
+ searchAttempts
117
139
  };
118
140
  }
119
141
  const candidates = selectCandidates({
@@ -122,10 +144,13 @@ export function createResearchWorker({ search, fetchPage }) {
122
144
  seenUrls: new Set(evidence.map((item) => item.url)),
123
145
  maxCandidates: maxFetches
124
146
  });
147
+ const fetchAttempts = [];
125
148
  for (const candidate of candidates) {
126
- const fetched = await fetchPage({ url: candidate.url });
149
+ const fetched = await fetchPage({ url: candidate.url, query });
150
+ if (fetched.metadata.attempts)
151
+ fetchAttempts.push(...fetched.metadata.attempts);
127
152
  if (fetched.status === 'ok') {
128
- const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
153
+ const parsedEvidence = evidenceFromFetch(fetched, candidate.title, query);
129
154
  if (parsedEvidence) {
130
155
  evidence.push(parsedEvidence);
131
156
  continue;
@@ -136,6 +161,14 @@ export function createResearchWorker({ search, fetchPage }) {
136
161
  }
137
162
  continue;
138
163
  }
164
+ // guard_refused and friends are final; never hand them to headless.
165
+ if (isTerminalFailure(failureOf(fetched))) {
166
+ gaps.push({
167
+ kind: 'fetch-failed',
168
+ message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
169
+ });
170
+ continue;
171
+ }
139
172
  if (fetched.status === 'needs_headless') {
140
173
  if (!suggestedHeadlessUrl) {
141
174
  suggestedHeadlessUrl = fetched.url;
@@ -156,7 +189,10 @@ export function createResearchWorker({ search, fetchPage }) {
156
189
  suggestedHeadlessUrl,
157
190
  exhaustedBudget: false,
158
191
  fanoutProviders,
159
- fanoutSkipped
192
+ fanoutSkipped,
193
+ searchCoveragePartial,
194
+ searchAttempts,
195
+ fetchAttempts
160
196
  };
161
197
  }
162
198
  };
@@ -1,6 +1,8 @@
1
1
  import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
2
2
  function activeCaveatReasons(evidence, quality) {
3
- const reasons = quality?.caveatReasons ?? [];
3
+ // Partial search coverage is reported as a caveat but never changes the decision (#55):
4
+ // a strong answer stays an answer, and web-explore adds just the coverage sentence.
5
+ const reasons = (quality?.caveatReasons ?? []).filter((reason) => reason !== 'partial-search-coverage');
4
6
  if (!hasOfficialDocsAndApi(evidence))
5
7
  return reasons;
6
8
  return reasons.filter((reason) => reason !== 'low-diversity');
@@ -65,6 +65,12 @@ function serializeBackendConfigOverride(config) {
65
65
  const { password: _password, ...proxy } = config.proxy;
66
66
  backends.proxy = { ...proxy };
67
67
  }
68
+ if (config.network) {
69
+ backends.network = {
70
+ ...config.network,
71
+ ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
72
+ };
73
+ }
68
74
  return { backends };
69
75
  }
70
76
  async function readConfigFileForWrite(filePath) {
@@ -1,3 +1,4 @@
1
+ import { attemptLines } from './search-presentation.js';
1
2
  function internalReaderLabel(method) {
2
3
  if (method === 'headless')
3
4
  return 'web_fetch_headless';
@@ -49,7 +50,8 @@ export function buildExplorePresentation(result) {
49
50
  'Sources',
50
51
  ...result.sources.map((source) => `- [${internalReaderLabel(source.method)}] ${source.title}: ${source.url}`),
51
52
  internalSummary ? `\nInternal tools\n${internalSummary}` : undefined,
52
- result.caveat ? `\nCaveat\n${result.caveat}` : undefined
53
+ result.caveat ? `\nCaveat\n${result.caveat}` : undefined,
54
+ attemptLines(result.metadata?.attempts)
53
55
  ]
54
56
  .filter((line) => line !== undefined)
55
57
  .join('\n');
@@ -1,3 +1,4 @@
1
+ import { attemptLines } from './search-presentation.js';
1
2
  function countWords(text) {
2
3
  return text?.trim() ? text.trim().split(/\s+/).length : undefined;
3
4
  }
@@ -24,15 +25,21 @@ export function buildFetchPresentation(result) {
24
25
  preview: result.content?.title
25
26
  ? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
26
27
  : firstExcerpt(result.content?.text),
27
- verbose: result.status === 'ok'
28
- ? [
29
- `URL: ${result.url}`,
30
- result.content?.title ? `Title: ${result.content.title}` : undefined,
31
- firstExcerpt(result.content?.text, 500)
32
- ]
33
- .filter(Boolean)
34
- .join('\n')
35
- : undefined
28
+ verbose: [
29
+ ...(result.status === 'ok'
30
+ ? [
31
+ `URL: ${result.url}`,
32
+ result.content?.title ? `Title: ${result.content.title}` : undefined,
33
+ firstExcerpt(result.content?.text, 500),
34
+ result.metadata.blockedSubresources
35
+ ? `Blocked private-address requests: ${result.metadata.blockedSubresources}`
36
+ : undefined
37
+ ]
38
+ : []),
39
+ attemptLines(result.metadata.attempts)
40
+ ]
41
+ .filter(Boolean)
42
+ .join('\n') || undefined
36
43
  },
37
44
  metrics: {
38
45
  wordCount,
@@ -1,3 +1,4 @@
1
- import type { WebSearchResponse } from '../types.js';
1
+ import type { Attempt, WebSearchResponse } from '../types.js';
2
2
  import type { PresentationEnvelope } from './types.js';
3
+ export declare function attemptLines(attempts: Attempt[] | undefined): string | undefined;
3
4
  export declare function buildSearchPresentation(result: WebSearchResponse): PresentationEnvelope;
@@ -10,6 +10,18 @@ function fanoutNote(result) {
10
10
  }
11
11
  return '';
12
12
  }
13
+ export function attemptLines(attempts) {
14
+ const interesting = (attempts ?? []).filter((a) => a.outcome !== 'results' && a.outcome !== 'empty');
15
+ if (interesting.length === 0)
16
+ return undefined;
17
+ return interesting
18
+ .map((a) => {
19
+ const kind = a.failure?.kind ? ` (${a.failure.kind})` : '';
20
+ const until = a.cooldownUntil !== undefined ? `, cooling down until ${new Date(a.cooldownUntil).toISOString()}` : '';
21
+ return `${a.backend}: ${a.outcome}${a.skipReason ? ` [${a.skipReason}]` : ''}${kind}${until}`;
22
+ })
23
+ .join('\n');
24
+ }
13
25
  function formatCompact(result) {
14
26
  const fallbackPrefix = result.metadata.fallbackFrom
15
27
  ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
@@ -34,7 +46,7 @@ export function buildSearchPresentation(result) {
34
46
  views: {
35
47
  compact: formatCompact(result),
36
48
  preview: preview || undefined,
37
- verbose: verbose || undefined
49
+ verbose: [verbose, attemptLines(result.metadata.attempts)].filter(Boolean).join('\n') || undefined
38
50
  },
39
51
  metrics: {
40
52
  resultCount: result.results.length,
@@ -1,12 +1,8 @@
1
- import type { WebFetchResponse } from '../types.js';
1
+ import type { ResearchFetchInput, WebFetchResponse } from '../types.js';
2
2
  import type { SpecialContentReader } from './types.js';
3
3
  type ResolverDeps = {
4
4
  readers: SpecialContentReader[];
5
- fallback: (input: {
6
- url: string;
7
- }) => Promise<WebFetchResponse>;
5
+ fallback: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
8
6
  };
9
- export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: {
10
- url: string;
11
- }) => Promise<WebFetchResponse>;
7
+ export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: ResearchFetchInput) => Promise<WebFetchResponse>;
12
8
  export {};
@@ -1,7 +1,6 @@
1
- import type { WebSearchResponse } from '../types.js';
2
1
  export declare function createBraveSearchTool({ apiKey, fetchImpl }: {
3
2
  apiKey?: string;
4
3
  fetchImpl?: typeof fetch;
5
4
  }): ({ query }: {
6
5
  query: string;
7
- }) => Promise<WebSearchResponse>;
6
+ }) => Promise<import("../types.js").WebSearchResponse>;
@@ -1,83 +1,26 @@
1
- import { buildSearchPresentation } from '../presentation/search-presentation.js';
1
+ import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
2
2
  const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
3
- function resultWithPresentation(result) {
4
- return { ...result, presentation: buildSearchPresentation(result) };
5
- }
6
- function normalizeResults(response) {
7
- return (response.web?.results ?? []).flatMap((item) => {
8
- if (typeof item.title !== 'string' || typeof item.url !== 'string') {
9
- return [];
10
- }
11
- return [
12
- {
13
- title: item.title,
14
- url: item.url,
15
- snippet: typeof item.description === 'string' ? item.description : ''
16
- }
17
- ];
18
- });
19
- }
20
- function buildBraveUrl(query) {
21
- const url = new URL(BRAVE_WEB_SEARCH_URL);
22
- url.searchParams.set('q', query);
23
- return url.toString();
24
- }
25
3
  export function createBraveSearchTool({ apiKey, fetchImpl = fetch }) {
26
- return async function braveSearch({ query }) {
27
- const normalizedQuery = query.trim();
28
- if (!normalizedQuery) {
29
- return resultWithPresentation({
30
- status: 'error',
31
- results: [],
32
- metadata: { backend: 'brave', cacheHit: false },
33
- error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
34
- });
35
- }
36
- if (!apiKey?.trim()) {
37
- return resultWithPresentation({
38
- status: 'error',
39
- results: [],
40
- metadata: { backend: 'brave', cacheHit: false },
41
- error: {
42
- code: 'BACKEND_CONFIG_INVALID',
43
- message: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.'
44
- }
45
- });
46
- }
47
- try {
48
- const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
49
- headers: {
50
- Accept: 'application/json',
51
- 'X-Subscription-Token': apiKey
52
- }
53
- });
54
- if (!response.ok) {
55
- throw new Error(`HTTP ${response.status}`);
56
- }
57
- const parsed = (await response.json());
58
- const results = normalizeResults(parsed);
59
- if (results.length === 0) {
60
- return resultWithPresentation({
61
- status: 'error',
62
- results: [],
63
- metadata: { backend: 'brave', cacheHit: false },
64
- error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
65
- });
66
- }
67
- return resultWithPresentation({
68
- status: 'ok',
69
- results,
70
- metadata: { backend: 'brave', cacheHit: false }
71
- });
72
- }
73
- catch (error) {
74
- const rawMessage = error instanceof Error ? error.message : String(error);
75
- return resultWithPresentation({
76
- status: 'error',
77
- results: [],
78
- metadata: { backend: 'brave', cacheHit: false },
79
- error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
80
- });
81
- }
82
- };
4
+ return createJsonSearchProvider({
5
+ name: 'brave',
6
+ label: 'Brave',
7
+ apiKey,
8
+ missingKeyMessage: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.',
9
+ fetchImpl,
10
+ request: (query) => {
11
+ const url = new URL(BRAVE_WEB_SEARCH_URL);
12
+ url.searchParams.set('q', query);
13
+ return { url: url.toString(), init: { headers: { Accept: 'application/json', 'X-Subscription-Token': apiKey ?? '' } } };
14
+ },
15
+ // A search response (`type: 'search'`) with no `web` block, or a `web` block with no `results`,
16
+ // is Brave's valid empty response. Anything else without `web` (an error-shaped or partial 200)
17
+ // is not trusted as empty.
18
+ normalize: (json) => normalizeResultsArray(json, (body) => {
19
+ if (body.web === undefined)
20
+ return body.type === 'search' ? [] : undefined;
21
+ if (!body.web || typeof body.web !== 'object')
22
+ return undefined;
23
+ return body.web.results === undefined ? [] : body.web.results;
24
+ }, 'description')
25
+ });
83
26
  }
@@ -10,9 +10,13 @@ export declare const DUCKDUCKGO_HEADERS: {
10
10
  readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
11
11
  readonly 'Accept-Language': "en-US,en;q=0.9";
12
12
  };
13
- export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl, retries, sleep }?: {
13
+ export declare class DuckDuckGoHttpError extends Error {
14
+ readonly status: number;
15
+ readonly headers: Headers;
16
+ constructor(status: number, headers: Headers);
17
+ }
18
+ /** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
19
+ export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl }?: {
14
20
  fetchImpl?: typeof fetch;
15
- retries?: number;
16
- sleep?: (ms: number) => Promise<void>;
17
21
  }): Promise<string>;
18
22
  export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;
@@ -26,25 +26,24 @@ export const DUCKDUCKGO_HEADERS = {
26
26
  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
27
27
  'Accept-Language': 'en-US,en;q=0.9'
28
28
  };
29
- const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
30
- export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch, retries = 1, sleep = defaultSleep } = {}) {
31
- let lastError;
32
- for (let attempt = 0; attempt <= retries; attempt += 1) {
33
- try {
34
- const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
35
- if (!response.ok) {
36
- throw new Error(`DuckDuckGo request failed with ${response.status}`);
37
- }
38
- return response.text();
39
- }
40
- catch (error) {
41
- lastError = error;
42
- if (attempt < retries) {
43
- await sleep(500);
44
- }
45
- }
29
+ export class DuckDuckGoHttpError extends Error {
30
+ status;
31
+ headers;
32
+ constructor(status, headers) {
33
+ super(`DuckDuckGo request failed with ${status}`);
34
+ this.status = status;
35
+ this.headers = headers;
36
+ this.name = 'DuckDuckGoHttpError';
37
+ }
38
+ }
39
+ /** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
40
+ export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch } = {}) {
41
+ const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
42
+ if (!response.ok) {
43
+ await response.body?.cancel().catch(() => undefined);
44
+ throw new DuckDuckGoHttpError(response.status, response.headers);
46
45
  }
47
- throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
46
+ return response.text();
48
47
  }
49
48
  export function parseDuckDuckGoResults(html) {
50
49
  const $ = cheerio.load(html);
@@ -1,7 +1,6 @@
1
- import type { WebSearchResponse } from '../types.js';
2
1
  export declare function createExaSearchTool({ apiKey, fetchImpl }: {
3
2
  apiKey?: string;
4
3
  fetchImpl?: typeof fetch;
5
4
  }): ({ query }: {
6
5
  query: string;
7
- }) => Promise<WebSearchResponse>;
6
+ }) => Promise<import("../types.js").WebSearchResponse>;
@@ -1,81 +1,20 @@
1
- import { buildSearchPresentation } from '../presentation/search-presentation.js';
1
+ import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
2
2
  const EXA_SEARCH_URL = 'https://api.exa.ai/search';
3
- function resultWithPresentation(result) {
4
- return { ...result, presentation: buildSearchPresentation(result) };
5
- }
6
- function normalizeResults(response) {
7
- return (response.results ?? []).flatMap((item) => {
8
- if (typeof item.title !== 'string' || typeof item.url !== 'string') {
9
- return [];
10
- }
11
- return [
12
- {
13
- title: item.title,
14
- url: item.url,
15
- snippet: typeof item.text === 'string' ? item.text : ''
16
- }
17
- ];
18
- });
19
- }
20
3
  export function createExaSearchTool({ apiKey, fetchImpl = fetch }) {
21
- return async function exaSearch({ query }) {
22
- const normalizedQuery = query.trim();
23
- if (!normalizedQuery) {
24
- return resultWithPresentation({
25
- status: 'error',
26
- results: [],
27
- metadata: { backend: 'exa', cacheHit: false },
28
- error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
29
- });
30
- }
31
- if (!apiKey?.trim()) {
32
- return resultWithPresentation({
33
- status: 'error',
34
- results: [],
35
- metadata: { backend: 'exa', cacheHit: false },
36
- error: {
37
- code: 'BACKEND_CONFIG_INVALID',
38
- message: 'Exa search requires EXA_API_KEY.'
39
- }
40
- });
41
- }
42
- try {
43
- const response = await fetchImpl(EXA_SEARCH_URL, {
4
+ return createJsonSearchProvider({
5
+ name: 'exa',
6
+ label: 'Exa',
7
+ apiKey,
8
+ missingKeyMessage: 'Exa search requires EXA_API_KEY.',
9
+ fetchImpl,
10
+ request: (query) => ({
11
+ url: EXA_SEARCH_URL,
12
+ init: {
44
13
  method: 'POST',
45
- headers: {
46
- Accept: 'application/json',
47
- 'Content-Type': 'application/json',
48
- 'x-api-key': apiKey
49
- },
50
- body: JSON.stringify({ query: normalizedQuery, numResults: 10 })
51
- });
52
- if (!response.ok) {
53
- throw new Error(`HTTP ${response.status}`);
54
- }
55
- const parsed = (await response.json());
56
- const results = normalizeResults(parsed);
57
- if (results.length === 0) {
58
- return resultWithPresentation({
59
- status: 'error',
60
- results: [],
61
- metadata: { backend: 'exa', cacheHit: false },
62
- error: { code: 'NO_RESULTS', message: 'Exa returned no usable results for this query.' }
63
- });
14
+ headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'x-api-key': apiKey ?? '' },
15
+ body: JSON.stringify({ query, numResults: 10 })
64
16
  }
65
- return resultWithPresentation({
66
- status: 'ok',
67
- results,
68
- metadata: { backend: 'exa', cacheHit: false }
69
- });
70
- }
71
- catch (error) {
72
- const rawMessage = error instanceof Error ? error.message : String(error);
73
- return resultWithPresentation({
74
- status: 'error',
75
- results: [],
76
- metadata: { backend: 'exa', cacheHit: false },
77
- error: { code: 'FETCH_FAILED', message: `Exa search request failed: ${rawMessage}` }
78
- });
79
- }
80
- };
17
+ }),
18
+ normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'text')
19
+ });
81
20
  }
@@ -5,10 +5,22 @@ export type FanoutProvider = {
5
5
  query: string;
6
6
  }) => Promise<WebSearchResponse>;
7
7
  };
8
+ export declare const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
9
+ /** Longest a policy-wrapped provider can legitimately take: two timed-out calls and the retry sleep. */
10
+ export declare function fanoutBackstopMs(timeoutMs: number): number;
11
+ type SearchFn = FanoutProvider['search'];
12
+ /**
13
+ * Per-call timeout. Wrap it INSIDE the retry policy so a stalled call is a transient
14
+ * failure the policy can retry once (#55).
15
+ */
16
+ export declare function withCallTimeout(search: SearchFn, timeoutMs: number, name: SearchProviderName): SearchFn;
8
17
  export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
9
18
  providers: FanoutProvider[];
10
19
  mode: Exclude<FanoutMode, 'off'>;
20
+ /** Per-call timeout the providers apply themselves (see withCallTimeout). Fanout only keeps a
21
+ * backstop that can't cut a retry short. */
11
22
  timeoutMs?: number;
12
23
  }): ({ query }: {
13
24
  query: string;
14
25
  }) => Promise<WebSearchResponse>;
26
+ export {};