@demigodmode/pi-web-agent 1.11.0 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/backends/config.d.ts +13 -0
  3. package/dist/backends/config.js +44 -1
  4. package/dist/backends/factory.d.ts +15 -0
  5. package/dist/backends/factory.js +118 -91
  6. package/dist/backends/failure.d.ts +11 -0
  7. package/dist/backends/failure.js +34 -0
  8. package/dist/backends/fallback-policy.d.ts +33 -0
  9. package/dist/backends/fallback-policy.js +239 -0
  10. package/dist/backends/provider-failure.d.ts +21 -0
  11. package/dist/backends/provider-failure.js +111 -0
  12. package/dist/backends/provider-health.d.ts +29 -0
  13. package/dist/backends/provider-health.js +49 -0
  14. package/dist/commands/web-agent-config.d.ts +14 -1
  15. package/dist/commands/web-agent-config.js +75 -3
  16. package/dist/extension.js +47 -3
  17. package/dist/fetch/destination-policy.d.ts +32 -0
  18. package/dist/fetch/destination-policy.js +24 -0
  19. package/dist/fetch/firecrawl-fetch.js +64 -45
  20. package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
  21. package/dist/fetch/guard-proxy-fetch.js +82 -0
  22. package/dist/fetch/guard-proxy.d.ts +58 -0
  23. package/dist/fetch/guard-proxy.js +420 -0
  24. package/dist/fetch/guarded-fetch.d.ts +7 -0
  25. package/dist/fetch/guarded-fetch.js +75 -0
  26. package/dist/fetch/headless-fetch.d.ts +10 -2
  27. package/dist/fetch/headless-fetch.js +181 -9
  28. package/dist/fetch/http-fetch.js +16 -1
  29. package/dist/fetch/network-guard.d.ts +82 -0
  30. package/dist/fetch/network-guard.js +275 -0
  31. package/dist/orchestration/answer-synthesizer.js +2 -0
  32. package/dist/orchestration/evidence-quality.d.ts +3 -2
  33. package/dist/orchestration/evidence-quality.js +2 -1
  34. package/dist/orchestration/index.d.ts +23 -0
  35. package/dist/orchestration/index.js +9 -2
  36. package/dist/orchestration/research-orchestrator.d.ts +21 -1
  37. package/dist/orchestration/research-orchestrator.js +40 -7
  38. package/dist/orchestration/research-types.d.ts +13 -1
  39. package/dist/orchestration/research-worker.js +38 -3
  40. package/dist/orchestration/stop-decider.js +3 -1
  41. package/dist/presentation/config-store.js +6 -0
  42. package/dist/presentation/explore-presentation.js +3 -1
  43. package/dist/presentation/fetch-presentation.js +16 -9
  44. package/dist/presentation/search-presentation.d.ts +2 -1
  45. package/dist/presentation/search-presentation.js +13 -1
  46. package/dist/search/brave.d.ts +1 -2
  47. package/dist/search/brave.js +23 -80
  48. package/dist/search/duckduckgo.d.ts +7 -3
  49. package/dist/search/duckduckgo.js +17 -18
  50. package/dist/search/exa.d.ts +1 -2
  51. package/dist/search/exa.js +15 -76
  52. package/dist/search/fanout.d.ts +12 -0
  53. package/dist/search/fanout.js +86 -47
  54. package/dist/search/json-provider.d.ts +32 -0
  55. package/dist/search/json-provider.js +76 -0
  56. package/dist/search/searxng.d.ts +1 -2
  57. package/dist/search/searxng.js +15 -57
  58. package/dist/search/tavily.d.ts +1 -2
  59. package/dist/search/tavily.js +17 -74
  60. package/dist/search/youcom.d.ts +1 -2
  61. package/dist/search/youcom.js +15 -76
  62. package/dist/tools/web-explore.d.ts +9 -0
  63. package/dist/tools/web-explore.js +16 -2
  64. package/dist/tools/web-search.js +41 -103
  65. package/dist/types.d.ts +40 -0
  66. package/package.json +3 -3
@@ -19,6 +19,7 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
19
19
  evidence: import("./research-types.js").ResearchEvidence[];
20
20
  workerPass: import("./research-types.js").ResearchWorkerResult;
21
21
  metadata: {
22
+ attempts?: import("../types.js").Attempt[] | undefined;
22
23
  searchPasses: number;
23
24
  fetchedPages: number;
24
25
  headlessAttempts: number;
@@ -27,5 +28,27 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
27
28
  fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
28
29
  fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
29
30
  };
31
+ terminalFailure?: undefined;
32
+ } | {
33
+ decision: import("./research-types.js").ResearchOrchestratorDecision;
34
+ evidence: never[];
35
+ workerPass: import("./research-types.js").ResearchWorkerResult;
36
+ metadata: {
37
+ attempts?: import("../types.js").Attempt[] | undefined;
38
+ searchPasses: number;
39
+ fetchedPages: number;
40
+ headlessAttempts: number;
41
+ exhaustedBudget: boolean;
42
+ caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
43
+ fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
44
+ fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
45
+ };
46
+ terminalFailure: {
47
+ code: string;
48
+ message: string;
49
+ };
30
50
  }>;
51
+ } & {
52
+ /** Releases the backend set this workflow created. Injected capabilities are left alone. */
53
+ close(): Promise<void>;
31
54
  };
@@ -2,14 +2,21 @@ import { createBackendSet } from '../backends/factory.js';
2
2
  import { createResearchOrchestrator } from './research-orchestrator.js';
3
3
  import { createResearchWorker } from './research-worker.js';
4
4
  export function createResearchWorkflow({ backendConfig, search, fetchPage, headlessFetch } = {}) {
5
- const backends = createBackendSet(backendConfig);
5
+ // Only build (and own) a backend set when something wasn't injected.
6
+ const backends = search && fetchPage && headlessFetch ? undefined : createBackendSet(backendConfig);
6
7
  const resolvedSearch = search ?? backends.search;
7
8
  const resolvedFetchPage = fetchPage ?? backends.fetchPage;
8
9
  const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
9
10
  const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
10
- return createResearchOrchestrator({
11
+ const orchestrator = createResearchOrchestrator({
11
12
  worker,
12
13
  fetchDirect: resolvedFetchPage,
13
14
  headlessFetch: resolvedHeadlessFetch
14
15
  });
16
+ return Object.assign(orchestrator, {
17
+ /** Releases the backend set this workflow created. Injected capabilities are left alone. */
18
+ async close() {
19
+ await backends?.close?.();
20
+ }
21
+ });
15
22
  }
@@ -1,4 +1,4 @@
1
- import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
1
+ import type { Attempt, SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
2
2
  import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
3
3
  import { type EvidenceCaveatReason } from './evidence-quality.js';
4
4
  export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
@@ -23,6 +23,7 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
23
23
  evidence: ResearchEvidence[];
24
24
  workerPass: ResearchWorkerResult;
25
25
  metadata: {
26
+ attempts?: Attempt[] | undefined;
26
27
  searchPasses: number;
27
28
  fetchedPages: number;
28
29
  headlessAttempts: number;
@@ -31,5 +32,24 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
31
32
  fanoutProviders: SearchProviderName[] | undefined;
32
33
  fanoutSkipped: SearchProviderName[] | undefined;
33
34
  };
35
+ terminalFailure?: undefined;
36
+ } | {
37
+ decision: ResearchOrchestratorDecision;
38
+ evidence: never[];
39
+ workerPass: ResearchWorkerResult;
40
+ metadata: {
41
+ attempts?: Attempt[] | undefined;
42
+ searchPasses: number;
43
+ fetchedPages: number;
44
+ headlessAttempts: number;
45
+ exhaustedBudget: boolean;
46
+ caveatReasons: EvidenceCaveatReason[];
47
+ fanoutProviders: SearchProviderName[] | undefined;
48
+ fanoutSkipped: SearchProviderName[] | undefined;
49
+ };
50
+ terminalFailure: {
51
+ code: string;
52
+ message: string;
53
+ };
34
54
  }>;
35
55
  };
@@ -1,3 +1,4 @@
1
+ import { failureOf, isTerminalFailure } from '../backends/failure.js';
1
2
  import { rankEvidence } from './evidence-ranker.js';
2
3
  import { planSearchQueries } from './query-planner.js';
3
4
  import { classifySourceProfile } from './source-profile.js';
@@ -79,7 +80,7 @@ function shouldRetryDirectWithHeadless(result, evidence) {
79
80
  return false;
80
81
  return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
81
82
  }
82
- function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
83
+ function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped, attempts }) {
83
84
  return {
84
85
  searchPasses: previousQueries.length,
85
86
  fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
@@ -87,7 +88,8 @@ function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutco
87
88
  exhaustedBudget,
88
89
  caveatReasons,
89
90
  fanoutProviders,
90
- fanoutSkipped
91
+ fanoutSkipped,
92
+ ...(attempts && attempts.length > 0 ? { attempts } : {})
91
93
  };
92
94
  }
93
95
  function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
@@ -114,21 +116,33 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
114
116
  const suggestedHeadlessUrls = [];
115
117
  let headlessAttempts = 0;
116
118
  let lastPass;
119
+ let searchCoveragePartial = false;
117
120
  const fanoutProvidersSeen = new Set();
118
121
  const fanoutSkippedSeen = new Set();
122
+ // Search and fetch attempts from every pass and direct URL, for verbose provenance.
123
+ const runAttempts = [];
119
124
  function fanoutSnapshot() {
120
125
  const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
121
126
  const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
122
- return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
127
+ return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined, attempts: [...runAttempts] };
123
128
  }
124
129
  if (fetchDirect) {
125
130
  for (const url of extractDirectUrls(query).slice(0, 3)) {
126
131
  const directResult = await fetchDirect({ url });
132
+ if (directResult.metadata.attempts)
133
+ runAttempts.push(...directResult.metadata.attempts);
127
134
  const directEvidence = evidenceFromFetch(directResult);
128
135
  if (directEvidence) {
129
136
  allEvidence.push(directEvidence);
130
137
  continue;
131
138
  }
139
+ if (isTerminalFailure(failureOf(directResult))) {
140
+ allGaps.push({
141
+ kind: 'fetch-failed',
142
+ message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
143
+ });
144
+ continue;
145
+ }
132
146
  if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
133
147
  if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
134
148
  headlessAttempts++;
@@ -161,7 +175,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
161
175
  const quality = analyzeEvidenceQuality({
162
176
  evidence: ranked,
163
177
  gaps: allGaps,
164
- lowValueOutcomes: allLowValueOutcomes
178
+ lowValueOutcomes: allLowValueOutcomes,
179
+ partialSearchCoverage: searchCoveragePartial
165
180
  });
166
181
  return {
167
182
  decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
@@ -200,6 +215,21 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
200
215
  maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
201
216
  });
202
217
  lastPass = pass;
218
+ if (pass.searchAttempts)
219
+ runAttempts.push(...pass.searchAttempts);
220
+ if (pass.fetchAttempts)
221
+ runAttempts.push(...pass.fetchAttempts);
222
+ if (pass.searchCoveragePartial)
223
+ searchCoveragePartial = true;
224
+ if (pass.terminalFailure) {
225
+ return {
226
+ decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked: [], exhaustedBudget: false }),
227
+ evidence: [],
228
+ workerPass: combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget: false }),
229
+ metadata: buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget: false, ...fanoutSnapshot() }),
230
+ terminalFailure: pass.terminalFailure
231
+ };
232
+ }
203
233
  pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
204
234
  pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
205
235
  allEvidence.push(...pass.evidence);
@@ -211,7 +241,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
211
241
  const quality = analyzeEvidenceQuality({
212
242
  evidence: ranked,
213
243
  gaps: allGaps,
214
- lowValueOutcomes: allLowValueOutcomes
244
+ lowValueOutcomes: allLowValueOutcomes,
245
+ partialSearchCoverage: searchCoveragePartial
215
246
  });
216
247
  const decision = decideNextResearchStep({
217
248
  evidence: ranked,
@@ -232,7 +263,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
232
263
  const updatedQuality = analyzeEvidenceQuality({
233
264
  evidence: updatedRanked,
234
265
  gaps: allGaps,
235
- lowValueOutcomes: allLowValueOutcomes
266
+ lowValueOutcomes: allLowValueOutcomes,
267
+ partialSearchCoverage: searchCoveragePartial
236
268
  });
237
269
  const updatedDecision = decideNextResearchStep({
238
270
  evidence: updatedRanked,
@@ -328,7 +360,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
328
360
  const quality = analyzeEvidenceQuality({
329
361
  evidence: ranked,
330
362
  gaps: allGaps,
331
- lowValueOutcomes: allLowValueOutcomes
363
+ lowValueOutcomes: allLowValueOutcomes,
364
+ partialSearchCoverage: searchCoveragePartial
332
365
  });
333
366
  return {
334
367
  decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked, exhaustedBudget: true }),
@@ -1,4 +1,4 @@
1
- import type { SearchProviderName } from '../types.js';
1
+ import type { Attempt, SearchProviderName } from '../types.js';
2
2
  export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
3
3
  export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
4
4
  export type ResearchEvidence = {
@@ -27,6 +27,13 @@ export type ResearchWorkerResult = {
27
27
  exhaustedBudget: boolean;
28
28
  fanoutProviders?: SearchProviderName[];
29
29
  fanoutSkipped?: SearchProviderName[];
30
+ searchCoveragePartial?: boolean;
31
+ searchAttempts?: Attempt[];
32
+ fetchAttempts?: Attempt[];
33
+ terminalFailure?: {
34
+ code: string;
35
+ message: string;
36
+ };
30
37
  };
31
38
  export type ResearchRunMetadata = {
32
39
  searchPasses: number;
@@ -35,6 +42,11 @@ export type ResearchRunMetadata = {
35
42
  exhaustedBudget: boolean;
36
43
  fanoutProviders?: SearchProviderName[];
37
44
  fanoutSkipped?: SearchProviderName[];
45
+ searchCoveragePartial?: boolean;
46
+ terminalFailure?: {
47
+ code: string;
48
+ message: string;
49
+ };
38
50
  };
39
51
  export type ResearchOrchestratorDecision = {
40
52
  action: 'answer';
@@ -1,3 +1,4 @@
1
+ import { failureOf, isTerminalFailure } from '../backends/failure.js';
1
2
  import { selectCandidates } from './candidate-selector.js';
2
3
  import { classifySourceProfile } from './source-profile.js';
3
4
  function classifySource(url) {
@@ -80,6 +81,22 @@ export function createResearchWorker({ search, fetchPage }) {
80
81
  };
81
82
  }
82
83
  const searchResult = await search({ query });
84
+ const searchCoveragePartial = searchResult.metadata.coverage?.partial === true;
85
+ const searchAttempts = searchResult.metadata.attempts;
86
+ if (isTerminalFailure(failureOf(searchResult))) {
87
+ return {
88
+ searchQueries,
89
+ evidence,
90
+ gaps: [],
91
+ lowValueOutcomes,
92
+ exhaustedBudget: false,
93
+ searchAttempts,
94
+ terminalFailure: {
95
+ code: searchResult.error?.code ?? 'SEARCH_FAILED',
96
+ message: `${searchResult.error?.message ?? 'Search failed.'} (${searchResult.error?.failure?.kind})`
97
+ }
98
+ };
99
+ }
83
100
  const fanoutProviders = searchResult.metadata.fanout?.providers;
84
101
  const fanoutSkipped = searchResult.metadata.fanout?.skipped;
85
102
  if (searchResult.status !== 'ok') {
@@ -96,7 +113,9 @@ export function createResearchWorker({ search, fetchPage }) {
96
113
  suggestedHeadlessUrl,
97
114
  exhaustedBudget: false,
98
115
  fanoutProviders,
99
- fanoutSkipped
116
+ fanoutSkipped,
117
+ searchCoveragePartial,
118
+ searchAttempts
100
119
  };
101
120
  }
102
121
  if (searchResult.results.length === 0) {
@@ -113,7 +132,9 @@ export function createResearchWorker({ search, fetchPage }) {
113
132
  suggestedHeadlessUrl,
114
133
  exhaustedBudget: false,
115
134
  fanoutProviders,
116
- fanoutSkipped
135
+ fanoutSkipped,
136
+ searchCoveragePartial,
137
+ searchAttempts
117
138
  };
118
139
  }
119
140
  const candidates = selectCandidates({
@@ -122,8 +143,11 @@ export function createResearchWorker({ search, fetchPage }) {
122
143
  seenUrls: new Set(evidence.map((item) => item.url)),
123
144
  maxCandidates: maxFetches
124
145
  });
146
+ const fetchAttempts = [];
125
147
  for (const candidate of candidates) {
126
148
  const fetched = await fetchPage({ url: candidate.url });
149
+ if (fetched.metadata.attempts)
150
+ fetchAttempts.push(...fetched.metadata.attempts);
127
151
  if (fetched.status === 'ok') {
128
152
  const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
129
153
  if (parsedEvidence) {
@@ -136,6 +160,14 @@ export function createResearchWorker({ search, fetchPage }) {
136
160
  }
137
161
  continue;
138
162
  }
163
+ // guard_refused and friends are final; never hand them to headless.
164
+ if (isTerminalFailure(failureOf(fetched))) {
165
+ gaps.push({
166
+ kind: 'fetch-failed',
167
+ message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
168
+ });
169
+ continue;
170
+ }
139
171
  if (fetched.status === 'needs_headless') {
140
172
  if (!suggestedHeadlessUrl) {
141
173
  suggestedHeadlessUrl = fetched.url;
@@ -156,7 +188,10 @@ export function createResearchWorker({ search, fetchPage }) {
156
188
  suggestedHeadlessUrl,
157
189
  exhaustedBudget: false,
158
190
  fanoutProviders,
159
- fanoutSkipped
191
+ fanoutSkipped,
192
+ searchCoveragePartial,
193
+ searchAttempts,
194
+ fetchAttempts
160
195
  };
161
196
  }
162
197
  };
@@ -1,6 +1,8 @@
1
1
  import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
2
2
  function activeCaveatReasons(evidence, quality) {
3
- const reasons = quality?.caveatReasons ?? [];
3
+ // Partial search coverage is reported as a caveat but never changes the decision (#55):
4
+ // a strong answer stays an answer, and web-explore adds just the coverage sentence.
5
+ const reasons = (quality?.caveatReasons ?? []).filter((reason) => reason !== 'partial-search-coverage');
4
6
  if (!hasOfficialDocsAndApi(evidence))
5
7
  return reasons;
6
8
  return reasons.filter((reason) => reason !== 'low-diversity');
@@ -65,6 +65,12 @@ function serializeBackendConfigOverride(config) {
65
65
  const { password: _password, ...proxy } = config.proxy;
66
66
  backends.proxy = { ...proxy };
67
67
  }
68
+ if (config.network) {
69
+ backends.network = {
70
+ ...config.network,
71
+ ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
72
+ };
73
+ }
68
74
  return { backends };
69
75
  }
70
76
  async function readConfigFileForWrite(filePath) {
@@ -1,3 +1,4 @@
1
+ import { attemptLines } from './search-presentation.js';
1
2
  function internalReaderLabel(method) {
2
3
  if (method === 'headless')
3
4
  return 'web_fetch_headless';
@@ -49,7 +50,8 @@ export function buildExplorePresentation(result) {
49
50
  'Sources',
50
51
  ...result.sources.map((source) => `- [${internalReaderLabel(source.method)}] ${source.title}: ${source.url}`),
51
52
  internalSummary ? `\nInternal tools\n${internalSummary}` : undefined,
52
- result.caveat ? `\nCaveat\n${result.caveat}` : undefined
53
+ result.caveat ? `\nCaveat\n${result.caveat}` : undefined,
54
+ attemptLines(result.metadata?.attempts)
53
55
  ]
54
56
  .filter((line) => line !== undefined)
55
57
  .join('\n');
@@ -1,3 +1,4 @@
1
+ import { attemptLines } from './search-presentation.js';
1
2
  function countWords(text) {
2
3
  return text?.trim() ? text.trim().split(/\s+/).length : undefined;
3
4
  }
@@ -24,15 +25,21 @@ export function buildFetchPresentation(result) {
24
25
  preview: result.content?.title
25
26
  ? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
26
27
  : firstExcerpt(result.content?.text),
27
- verbose: result.status === 'ok'
28
- ? [
29
- `URL: ${result.url}`,
30
- result.content?.title ? `Title: ${result.content.title}` : undefined,
31
- firstExcerpt(result.content?.text, 500)
32
- ]
33
- .filter(Boolean)
34
- .join('\n')
35
- : undefined
28
+ verbose: [
29
+ ...(result.status === 'ok'
30
+ ? [
31
+ `URL: ${result.url}`,
32
+ result.content?.title ? `Title: ${result.content.title}` : undefined,
33
+ firstExcerpt(result.content?.text, 500),
34
+ result.metadata.blockedSubresources
35
+ ? `Blocked private-address requests: ${result.metadata.blockedSubresources}`
36
+ : undefined
37
+ ]
38
+ : []),
39
+ attemptLines(result.metadata.attempts)
40
+ ]
41
+ .filter(Boolean)
42
+ .join('\n') || undefined
36
43
  },
37
44
  metrics: {
38
45
  wordCount,
@@ -1,3 +1,4 @@
1
- import type { WebSearchResponse } from '../types.js';
1
+ import type { Attempt, WebSearchResponse } from '../types.js';
2
2
  import type { PresentationEnvelope } from './types.js';
3
+ export declare function attemptLines(attempts: Attempt[] | undefined): string | undefined;
3
4
  export declare function buildSearchPresentation(result: WebSearchResponse): PresentationEnvelope;
@@ -10,6 +10,18 @@ function fanoutNote(result) {
10
10
  }
11
11
  return '';
12
12
  }
13
+ export function attemptLines(attempts) {
14
+ const interesting = (attempts ?? []).filter((a) => a.outcome !== 'results' && a.outcome !== 'empty');
15
+ if (interesting.length === 0)
16
+ return undefined;
17
+ return interesting
18
+ .map((a) => {
19
+ const kind = a.failure?.kind ? ` (${a.failure.kind})` : '';
20
+ const until = a.cooldownUntil !== undefined ? `, cooling down until ${new Date(a.cooldownUntil).toISOString()}` : '';
21
+ return `${a.backend}: ${a.outcome}${a.skipReason ? ` [${a.skipReason}]` : ''}${kind}${until}`;
22
+ })
23
+ .join('\n');
24
+ }
13
25
  function formatCompact(result) {
14
26
  const fallbackPrefix = result.metadata.fallbackFrom
15
27
  ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
@@ -34,7 +46,7 @@ export function buildSearchPresentation(result) {
34
46
  views: {
35
47
  compact: formatCompact(result),
36
48
  preview: preview || undefined,
37
- verbose: verbose || undefined
49
+ verbose: [verbose, attemptLines(result.metadata.attempts)].filter(Boolean).join('\n') || undefined
38
50
  },
39
51
  metrics: {
40
52
  resultCount: result.results.length,
@@ -1,7 +1,6 @@
1
- import type { WebSearchResponse } from '../types.js';
2
1
  export declare function createBraveSearchTool({ apiKey, fetchImpl }: {
3
2
  apiKey?: string;
4
3
  fetchImpl?: typeof fetch;
5
4
  }): ({ query }: {
6
5
  query: string;
7
- }) => Promise<WebSearchResponse>;
6
+ }) => Promise<import("../types.js").WebSearchResponse>;
@@ -1,83 +1,26 @@
1
- import { buildSearchPresentation } from '../presentation/search-presentation.js';
1
+ import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
2
2
  const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
3
- function resultWithPresentation(result) {
4
- return { ...result, presentation: buildSearchPresentation(result) };
5
- }
6
- function normalizeResults(response) {
7
- return (response.web?.results ?? []).flatMap((item) => {
8
- if (typeof item.title !== 'string' || typeof item.url !== 'string') {
9
- return [];
10
- }
11
- return [
12
- {
13
- title: item.title,
14
- url: item.url,
15
- snippet: typeof item.description === 'string' ? item.description : ''
16
- }
17
- ];
18
- });
19
- }
20
- function buildBraveUrl(query) {
21
- const url = new URL(BRAVE_WEB_SEARCH_URL);
22
- url.searchParams.set('q', query);
23
- return url.toString();
24
- }
25
3
  export function createBraveSearchTool({ apiKey, fetchImpl = fetch }) {
26
- return async function braveSearch({ query }) {
27
- const normalizedQuery = query.trim();
28
- if (!normalizedQuery) {
29
- return resultWithPresentation({
30
- status: 'error',
31
- results: [],
32
- metadata: { backend: 'brave', cacheHit: false },
33
- error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
34
- });
35
- }
36
- if (!apiKey?.trim()) {
37
- return resultWithPresentation({
38
- status: 'error',
39
- results: [],
40
- metadata: { backend: 'brave', cacheHit: false },
41
- error: {
42
- code: 'BACKEND_CONFIG_INVALID',
43
- message: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.'
44
- }
45
- });
46
- }
47
- try {
48
- const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
49
- headers: {
50
- Accept: 'application/json',
51
- 'X-Subscription-Token': apiKey
52
- }
53
- });
54
- if (!response.ok) {
55
- throw new Error(`HTTP ${response.status}`);
56
- }
57
- const parsed = (await response.json());
58
- const results = normalizeResults(parsed);
59
- if (results.length === 0) {
60
- return resultWithPresentation({
61
- status: 'error',
62
- results: [],
63
- metadata: { backend: 'brave', cacheHit: false },
64
- error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
65
- });
66
- }
67
- return resultWithPresentation({
68
- status: 'ok',
69
- results,
70
- metadata: { backend: 'brave', cacheHit: false }
71
- });
72
- }
73
- catch (error) {
74
- const rawMessage = error instanceof Error ? error.message : String(error);
75
- return resultWithPresentation({
76
- status: 'error',
77
- results: [],
78
- metadata: { backend: 'brave', cacheHit: false },
79
- error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
80
- });
81
- }
82
- };
4
+ return createJsonSearchProvider({
5
+ name: 'brave',
6
+ label: 'Brave',
7
+ apiKey,
8
+ missingKeyMessage: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.',
9
+ fetchImpl,
10
+ request: (query) => {
11
+ const url = new URL(BRAVE_WEB_SEARCH_URL);
12
+ url.searchParams.set('q', query);
13
+ return { url: url.toString(), init: { headers: { Accept: 'application/json', 'X-Subscription-Token': apiKey ?? '' } } };
14
+ },
15
+ // A search response (`type: 'search'`) with no `web` block, or a `web` block with no `results`,
16
+ // is Brave's valid empty response. Anything else without `web` (an error-shaped or partial 200)
17
+ // is not trusted as empty.
18
+ normalize: (json) => normalizeResultsArray(json, (body) => {
19
+ if (body.web === undefined)
20
+ return body.type === 'search' ? [] : undefined;
21
+ if (!body.web || typeof body.web !== 'object')
22
+ return undefined;
23
+ return body.web.results === undefined ? [] : body.web.results;
24
+ }, 'description')
25
+ });
83
26
  }
@@ -10,9 +10,13 @@ export declare const DUCKDUCKGO_HEADERS: {
10
10
  readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
11
11
  readonly 'Accept-Language': "en-US,en;q=0.9";
12
12
  };
13
- export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl, retries, sleep }?: {
13
+ export declare class DuckDuckGoHttpError extends Error {
14
+ readonly status: number;
15
+ readonly headers: Headers;
16
+ constructor(status: number, headers: Headers);
17
+ }
18
+ /** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
19
+ export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl }?: {
14
20
  fetchImpl?: typeof fetch;
15
- retries?: number;
16
- sleep?: (ms: number) => Promise<void>;
17
21
  }): Promise<string>;
18
22
  export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;
@@ -26,25 +26,24 @@ export const DUCKDUCKGO_HEADERS = {
26
26
  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
27
27
  'Accept-Language': 'en-US,en;q=0.9'
28
28
  };
29
- const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
30
- export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch, retries = 1, sleep = defaultSleep } = {}) {
31
- let lastError;
32
- for (let attempt = 0; attempt <= retries; attempt += 1) {
33
- try {
34
- const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
35
- if (!response.ok) {
36
- throw new Error(`DuckDuckGo request failed with ${response.status}`);
37
- }
38
- return response.text();
39
- }
40
- catch (error) {
41
- lastError = error;
42
- if (attempt < retries) {
43
- await sleep(500);
44
- }
45
- }
29
+ export class DuckDuckGoHttpError extends Error {
30
+ status;
31
+ headers;
32
+ constructor(status, headers) {
33
+ super(`DuckDuckGo request failed with ${status}`);
34
+ this.status = status;
35
+ this.headers = headers;
36
+ this.name = 'DuckDuckGoHttpError';
37
+ }
38
+ }
39
+ /** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
40
+ export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch } = {}) {
41
+ const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
42
+ if (!response.ok) {
43
+ await response.body?.cancel().catch(() => undefined);
44
+ throw new DuckDuckGoHttpError(response.status, response.headers);
46
45
  }
47
- throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
46
+ return response.text();
48
47
  }
49
48
  export function parseDuckDuckGoResults(html) {
50
49
  const $ = cheerio.load(html);
@@ -1,7 +1,6 @@
1
- import type { WebSearchResponse } from '../types.js';
2
1
  export declare function createExaSearchTool({ apiKey, fetchImpl }: {
3
2
  apiKey?: string;
4
3
  fetchImpl?: typeof fetch;
5
4
  }): ({ query }: {
6
5
  query: string;
7
- }) => Promise<WebSearchResponse>;
6
+ }) => Promise<import("../types.js").WebSearchResponse>;