@demigodmode/pi-web-agent 1.11.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +2 -0
- package/dist/backends/config.d.ts +13 -0
- package/dist/backends/config.js +44 -1
- package/dist/backends/doctor.js +27 -29
- package/dist/backends/factory.d.ts +18 -7
- package/dist/backends/factory.js +126 -99
- package/dist/backends/failure.d.ts +11 -0
- package/dist/backends/failure.js +34 -0
- package/dist/backends/fallback-policy.d.ts +31 -0
- package/dist/backends/fallback-policy.js +239 -0
- package/dist/backends/provider-failure.d.ts +21 -0
- package/dist/backends/provider-failure.js +110 -0
- package/dist/backends/provider-health.d.ts +29 -0
- package/dist/backends/provider-health.js +49 -0
- package/dist/commands/web-agent-config.d.ts +14 -1
- package/dist/commands/web-agent-config.js +75 -3
- package/dist/extension.js +47 -3
- package/dist/extract/bot-check.d.ts +1 -0
- package/dist/extract/bot-check.js +10 -0
- package/dist/extract/readability.d.ts +4 -0
- package/dist/extract/readability.js +67 -3
- package/dist/extract/section-selector.d.ts +14 -0
- package/dist/extract/section-selector.js +233 -0
- package/dist/fetch/destination-policy.d.ts +32 -0
- package/dist/fetch/destination-policy.js +24 -0
- package/dist/fetch/firecrawl-fetch.d.ts +1 -1
- package/dist/fetch/firecrawl-fetch.js +74 -46
- package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
- package/dist/fetch/guard-proxy-fetch.js +82 -0
- package/dist/fetch/guard-proxy.d.ts +58 -0
- package/dist/fetch/guard-proxy.js +420 -0
- package/dist/fetch/guarded-fetch.d.ts +7 -0
- package/dist/fetch/guarded-fetch.js +75 -0
- package/dist/fetch/headless-fetch.d.ts +11 -2
- package/dist/fetch/headless-fetch.js +190 -13
- package/dist/fetch/http-fetch.d.ts +1 -1
- package/dist/fetch/http-fetch.js +29 -8
- package/dist/fetch/network-guard.d.ts +82 -0
- package/dist/fetch/network-guard.js +275 -0
- package/dist/orchestration/answer-synthesizer.js +2 -0
- package/dist/orchestration/evidence-quality.d.ts +3 -2
- package/dist/orchestration/evidence-quality.js +2 -1
- package/dist/orchestration/index.d.ts +26 -7
- package/dist/orchestration/index.js +9 -2
- package/dist/orchestration/research-orchestrator.d.ts +23 -7
- package/dist/orchestration/research-orchestrator.js +60 -26
- package/dist/orchestration/research-types.d.ts +13 -1
- package/dist/orchestration/research-worker.d.ts +2 -4
- package/dist/orchestration/research-worker.js +51 -15
- package/dist/orchestration/stop-decider.js +3 -1
- package/dist/presentation/config-store.js +6 -0
- package/dist/presentation/explore-presentation.js +3 -1
- package/dist/presentation/fetch-presentation.js +16 -9
- package/dist/presentation/search-presentation.d.ts +2 -1
- package/dist/presentation/search-presentation.js +13 -1
- package/dist/readers/resolver.d.ts +3 -7
- package/dist/search/brave.d.ts +1 -2
- package/dist/search/brave.js +23 -80
- package/dist/search/duckduckgo.d.ts +7 -3
- package/dist/search/duckduckgo.js +17 -18
- package/dist/search/exa.d.ts +1 -2
- package/dist/search/exa.js +15 -76
- package/dist/search/fanout.d.ts +12 -0
- package/dist/search/fanout.js +86 -47
- package/dist/search/json-provider.d.ts +32 -0
- package/dist/search/json-provider.js +76 -0
- package/dist/search/searxng.d.ts +1 -2
- package/dist/search/searxng.js +15 -57
- package/dist/search/tavily.d.ts +1 -2
- package/dist/search/tavily.js +17 -74
- package/dist/search/youcom.d.ts +4 -2
- package/dist/search/youcom.js +49 -75
- package/dist/tools/web-explore.d.ts +9 -0
- package/dist/tools/web-explore.js +16 -2
- package/dist/tools/web-fetch-headless.d.ts +3 -5
- package/dist/tools/web-fetch-headless.js +3 -3
- package/dist/tools/web-fetch.d.ts +3 -5
- package/dist/tools/web-fetch.js +3 -3
- package/dist/tools/web-search.js +41 -103
- package/dist/types.d.ts +48 -0
- package/package.json +3 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SearchProviderName } from '../types.js';
|
|
1
|
+
import type { Attempt, SearchProviderName } from '../types.js';
|
|
2
2
|
export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
|
|
3
3
|
export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
|
|
4
4
|
export type ResearchEvidence = {
|
|
@@ -27,6 +27,13 @@ export type ResearchWorkerResult = {
|
|
|
27
27
|
exhaustedBudget: boolean;
|
|
28
28
|
fanoutProviders?: SearchProviderName[];
|
|
29
29
|
fanoutSkipped?: SearchProviderName[];
|
|
30
|
+
searchCoveragePartial?: boolean;
|
|
31
|
+
searchAttempts?: Attempt[];
|
|
32
|
+
fetchAttempts?: Attempt[];
|
|
33
|
+
terminalFailure?: {
|
|
34
|
+
code: string;
|
|
35
|
+
message: string;
|
|
36
|
+
};
|
|
30
37
|
};
|
|
31
38
|
export type ResearchRunMetadata = {
|
|
32
39
|
searchPasses: number;
|
|
@@ -35,6 +42,11 @@ export type ResearchRunMetadata = {
|
|
|
35
42
|
exhaustedBudget: boolean;
|
|
36
43
|
fanoutProviders?: SearchProviderName[];
|
|
37
44
|
fanoutSkipped?: SearchProviderName[];
|
|
45
|
+
searchCoveragePartial?: boolean;
|
|
46
|
+
terminalFailure?: {
|
|
47
|
+
code: string;
|
|
48
|
+
message: string;
|
|
49
|
+
};
|
|
38
50
|
};
|
|
39
51
|
export type ResearchOrchestratorDecision = {
|
|
40
52
|
action: 'answer';
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
import type { WebFetchResponse, WebSearchResponse } from '../types.js';
|
|
1
|
+
import type { ResearchFetchInput, WebFetchResponse, WebSearchResponse } from '../types.js';
|
|
2
2
|
import type { ResearchWorkerResult } from './research-types.js';
|
|
3
3
|
export declare function createResearchWorker({ search, fetchPage }: {
|
|
4
4
|
search: (input: {
|
|
5
5
|
query: string;
|
|
6
6
|
}) => Promise<WebSearchResponse>;
|
|
7
|
-
fetchPage: (input:
|
|
8
|
-
url: string;
|
|
9
|
-
}) => Promise<WebFetchResponse>;
|
|
7
|
+
fetchPage: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
|
|
10
8
|
}): {
|
|
11
9
|
run({ query, maxSearchRounds, maxFetches }: {
|
|
12
10
|
query: string;
|
|
@@ -1,22 +1,24 @@
|
|
|
1
|
+
import { failureOf, isTerminalFailure } from '../backends/failure.js';
|
|
1
2
|
import { selectCandidates } from './candidate-selector.js';
|
|
2
3
|
import { classifySourceProfile } from './source-profile.js';
|
|
4
|
+
import { selectRelevantExcerpt } from '../extract/section-selector.js';
|
|
5
|
+
import { hasBotCheckContent } from '../extract/bot-check.js';
|
|
3
6
|
function classifySource(url) {
|
|
4
7
|
return classifySourceProfile(url).sourceKind;
|
|
5
8
|
}
|
|
6
|
-
function summarizeText(text, maxLength = 180) {
|
|
7
|
-
return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
|
|
8
|
-
}
|
|
9
9
|
function isReaderMethod(method) {
|
|
10
10
|
return method === 'github' || method === 'pdf' || method === 'youtube';
|
|
11
11
|
}
|
|
12
|
-
function isBotCheckContent({ title = '', text }) {
|
|
13
|
-
|
|
12
|
+
function isBotCheckContent({ title = '', text, botCheck }) {
|
|
13
|
+
if (botCheck)
|
|
14
|
+
return true;
|
|
15
|
+
return hasBotCheckContent(`${title}\n${text}`);
|
|
14
16
|
}
|
|
15
|
-
function evidenceFromFetch(fetched, fallbackTitle) {
|
|
17
|
+
function evidenceFromFetch(fetched, fallbackTitle, query) {
|
|
16
18
|
const content = fetched.content;
|
|
17
19
|
if (fetched.status !== 'ok' || !content)
|
|
18
20
|
return null;
|
|
19
|
-
if (isBotCheckContent({ title: content.title, text: content.text }))
|
|
21
|
+
if (isBotCheckContent({ title: content.title, text: content.text, botCheck: content.botCheck }))
|
|
20
22
|
return null;
|
|
21
23
|
// A successful reader read with usable text is primary content, exempt from the
|
|
22
24
|
// package-page filter below.
|
|
@@ -39,14 +41,14 @@ function evidenceFromFetch(fetched, fallbackTitle) {
|
|
|
39
41
|
url: fetched.url,
|
|
40
42
|
sourceKind,
|
|
41
43
|
method: fetched.metadata.method,
|
|
42
|
-
summary:
|
|
43
|
-
supports: [
|
|
44
|
+
summary: selectRelevantExcerpt(content.text, query, 180),
|
|
45
|
+
supports: [selectRelevantExcerpt(content.text, query, 120)]
|
|
44
46
|
};
|
|
45
47
|
}
|
|
46
48
|
function lowValueOutcomeFromFetch(fetched) {
|
|
47
49
|
if (fetched.status !== 'ok' || !fetched.content)
|
|
48
50
|
return null;
|
|
49
|
-
if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text })) {
|
|
51
|
+
if (isBotCheckContent({ title: fetched.content.title, text: fetched.content.text, botCheck: fetched.content.botCheck })) {
|
|
50
52
|
return {
|
|
51
53
|
kind: 'bot-check',
|
|
52
54
|
url: fetched.url,
|
|
@@ -80,6 +82,22 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
80
82
|
};
|
|
81
83
|
}
|
|
82
84
|
const searchResult = await search({ query });
|
|
85
|
+
const searchCoveragePartial = searchResult.metadata.coverage?.partial === true;
|
|
86
|
+
const searchAttempts = searchResult.metadata.attempts;
|
|
87
|
+
if (isTerminalFailure(failureOf(searchResult))) {
|
|
88
|
+
return {
|
|
89
|
+
searchQueries,
|
|
90
|
+
evidence,
|
|
91
|
+
gaps: [],
|
|
92
|
+
lowValueOutcomes,
|
|
93
|
+
exhaustedBudget: false,
|
|
94
|
+
searchAttempts,
|
|
95
|
+
terminalFailure: {
|
|
96
|
+
code: searchResult.error?.code ?? 'SEARCH_FAILED',
|
|
97
|
+
message: `${searchResult.error?.message ?? 'Search failed.'} (${searchResult.error?.failure?.kind})`
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
}
|
|
83
101
|
const fanoutProviders = searchResult.metadata.fanout?.providers;
|
|
84
102
|
const fanoutSkipped = searchResult.metadata.fanout?.skipped;
|
|
85
103
|
if (searchResult.status !== 'ok') {
|
|
@@ -96,7 +114,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
96
114
|
suggestedHeadlessUrl,
|
|
97
115
|
exhaustedBudget: false,
|
|
98
116
|
fanoutProviders,
|
|
99
|
-
fanoutSkipped
|
|
117
|
+
fanoutSkipped,
|
|
118
|
+
searchCoveragePartial,
|
|
119
|
+
searchAttempts
|
|
100
120
|
};
|
|
101
121
|
}
|
|
102
122
|
if (searchResult.results.length === 0) {
|
|
@@ -113,7 +133,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
113
133
|
suggestedHeadlessUrl,
|
|
114
134
|
exhaustedBudget: false,
|
|
115
135
|
fanoutProviders,
|
|
116
|
-
fanoutSkipped
|
|
136
|
+
fanoutSkipped,
|
|
137
|
+
searchCoveragePartial,
|
|
138
|
+
searchAttempts
|
|
117
139
|
};
|
|
118
140
|
}
|
|
119
141
|
const candidates = selectCandidates({
|
|
@@ -122,10 +144,13 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
122
144
|
seenUrls: new Set(evidence.map((item) => item.url)),
|
|
123
145
|
maxCandidates: maxFetches
|
|
124
146
|
});
|
|
147
|
+
const fetchAttempts = [];
|
|
125
148
|
for (const candidate of candidates) {
|
|
126
|
-
const fetched = await fetchPage({ url: candidate.url });
|
|
149
|
+
const fetched = await fetchPage({ url: candidate.url, query });
|
|
150
|
+
if (fetched.metadata.attempts)
|
|
151
|
+
fetchAttempts.push(...fetched.metadata.attempts);
|
|
127
152
|
if (fetched.status === 'ok') {
|
|
128
|
-
const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
|
|
153
|
+
const parsedEvidence = evidenceFromFetch(fetched, candidate.title, query);
|
|
129
154
|
if (parsedEvidence) {
|
|
130
155
|
evidence.push(parsedEvidence);
|
|
131
156
|
continue;
|
|
@@ -136,6 +161,14 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
136
161
|
}
|
|
137
162
|
continue;
|
|
138
163
|
}
|
|
164
|
+
// guard_refused and friends are final; never hand them to headless.
|
|
165
|
+
if (isTerminalFailure(failureOf(fetched))) {
|
|
166
|
+
gaps.push({
|
|
167
|
+
kind: 'fetch-failed',
|
|
168
|
+
message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
|
|
169
|
+
});
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
139
172
|
if (fetched.status === 'needs_headless') {
|
|
140
173
|
if (!suggestedHeadlessUrl) {
|
|
141
174
|
suggestedHeadlessUrl = fetched.url;
|
|
@@ -156,7 +189,10 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
156
189
|
suggestedHeadlessUrl,
|
|
157
190
|
exhaustedBudget: false,
|
|
158
191
|
fanoutProviders,
|
|
159
|
-
fanoutSkipped
|
|
192
|
+
fanoutSkipped,
|
|
193
|
+
searchCoveragePartial,
|
|
194
|
+
searchAttempts,
|
|
195
|
+
fetchAttempts
|
|
160
196
|
};
|
|
161
197
|
}
|
|
162
198
|
};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
|
|
2
2
|
function activeCaveatReasons(evidence, quality) {
|
|
3
|
-
|
|
3
|
+
// Partial search coverage is reported as a caveat but never changes the decision (#55):
|
|
4
|
+
// a strong answer stays an answer, and web-explore adds just the coverage sentence.
|
|
5
|
+
const reasons = (quality?.caveatReasons ?? []).filter((reason) => reason !== 'partial-search-coverage');
|
|
4
6
|
if (!hasOfficialDocsAndApi(evidence))
|
|
5
7
|
return reasons;
|
|
6
8
|
return reasons.filter((reason) => reason !== 'low-diversity');
|
|
@@ -65,6 +65,12 @@ function serializeBackendConfigOverride(config) {
|
|
|
65
65
|
const { password: _password, ...proxy } = config.proxy;
|
|
66
66
|
backends.proxy = { ...proxy };
|
|
67
67
|
}
|
|
68
|
+
if (config.network) {
|
|
69
|
+
backends.network = {
|
|
70
|
+
...config.network,
|
|
71
|
+
...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
|
|
72
|
+
};
|
|
73
|
+
}
|
|
68
74
|
return { backends };
|
|
69
75
|
}
|
|
70
76
|
async function readConfigFileForWrite(filePath) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attemptLines } from './search-presentation.js';
|
|
1
2
|
function internalReaderLabel(method) {
|
|
2
3
|
if (method === 'headless')
|
|
3
4
|
return 'web_fetch_headless';
|
|
@@ -49,7 +50,8 @@ export function buildExplorePresentation(result) {
|
|
|
49
50
|
'Sources',
|
|
50
51
|
...result.sources.map((source) => `- [${internalReaderLabel(source.method)}] ${source.title}: ${source.url}`),
|
|
51
52
|
internalSummary ? `\nInternal tools\n${internalSummary}` : undefined,
|
|
52
|
-
result.caveat ? `\nCaveat\n${result.caveat}` : undefined
|
|
53
|
+
result.caveat ? `\nCaveat\n${result.caveat}` : undefined,
|
|
54
|
+
attemptLines(result.metadata?.attempts)
|
|
53
55
|
]
|
|
54
56
|
.filter((line) => line !== undefined)
|
|
55
57
|
.join('\n');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attemptLines } from './search-presentation.js';
|
|
1
2
|
function countWords(text) {
|
|
2
3
|
return text?.trim() ? text.trim().split(/\s+/).length : undefined;
|
|
3
4
|
}
|
|
@@ -24,15 +25,21 @@ export function buildFetchPresentation(result) {
|
|
|
24
25
|
preview: result.content?.title
|
|
25
26
|
? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
|
|
26
27
|
: firstExcerpt(result.content?.text),
|
|
27
|
-
verbose:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
verbose: [
|
|
29
|
+
...(result.status === 'ok'
|
|
30
|
+
? [
|
|
31
|
+
`URL: ${result.url}`,
|
|
32
|
+
result.content?.title ? `Title: ${result.content.title}` : undefined,
|
|
33
|
+
firstExcerpt(result.content?.text, 500),
|
|
34
|
+
result.metadata.blockedSubresources
|
|
35
|
+
? `Blocked private-address requests: ${result.metadata.blockedSubresources}`
|
|
36
|
+
: undefined
|
|
37
|
+
]
|
|
38
|
+
: []),
|
|
39
|
+
attemptLines(result.metadata.attempts)
|
|
40
|
+
]
|
|
41
|
+
.filter(Boolean)
|
|
42
|
+
.join('\n') || undefined
|
|
36
43
|
},
|
|
37
44
|
metrics: {
|
|
38
45
|
wordCount,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
1
|
+
import type { Attempt, WebSearchResponse } from '../types.js';
|
|
2
2
|
import type { PresentationEnvelope } from './types.js';
|
|
3
|
+
export declare function attemptLines(attempts: Attempt[] | undefined): string | undefined;
|
|
3
4
|
export declare function buildSearchPresentation(result: WebSearchResponse): PresentationEnvelope;
|
|
@@ -10,6 +10,18 @@ function fanoutNote(result) {
|
|
|
10
10
|
}
|
|
11
11
|
return '';
|
|
12
12
|
}
|
|
13
|
+
export function attemptLines(attempts) {
|
|
14
|
+
const interesting = (attempts ?? []).filter((a) => a.outcome !== 'results' && a.outcome !== 'empty');
|
|
15
|
+
if (interesting.length === 0)
|
|
16
|
+
return undefined;
|
|
17
|
+
return interesting
|
|
18
|
+
.map((a) => {
|
|
19
|
+
const kind = a.failure?.kind ? ` (${a.failure.kind})` : '';
|
|
20
|
+
const until = a.cooldownUntil !== undefined ? `, cooling down until ${new Date(a.cooldownUntil).toISOString()}` : '';
|
|
21
|
+
return `${a.backend}: ${a.outcome}${a.skipReason ? ` [${a.skipReason}]` : ''}${kind}${until}`;
|
|
22
|
+
})
|
|
23
|
+
.join('\n');
|
|
24
|
+
}
|
|
13
25
|
function formatCompact(result) {
|
|
14
26
|
const fallbackPrefix = result.metadata.fallbackFrom
|
|
15
27
|
? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
|
|
@@ -34,7 +46,7 @@ export function buildSearchPresentation(result) {
|
|
|
34
46
|
views: {
|
|
35
47
|
compact: formatCompact(result),
|
|
36
48
|
preview: preview || undefined,
|
|
37
|
-
verbose: verbose || undefined
|
|
49
|
+
verbose: [verbose, attemptLines(result.metadata.attempts)].filter(Boolean).join('\n') || undefined
|
|
38
50
|
},
|
|
39
51
|
metrics: {
|
|
40
52
|
resultCount: result.results.length,
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import type { WebFetchResponse } from '../types.js';
|
|
1
|
+
import type { ResearchFetchInput, WebFetchResponse } from '../types.js';
|
|
2
2
|
import type { SpecialContentReader } from './types.js';
|
|
3
3
|
type ResolverDeps = {
|
|
4
4
|
readers: SpecialContentReader[];
|
|
5
|
-
fallback: (input:
|
|
6
|
-
url: string;
|
|
7
|
-
}) => Promise<WebFetchResponse>;
|
|
5
|
+
fallback: (input: ResearchFetchInput) => Promise<WebFetchResponse>;
|
|
8
6
|
};
|
|
9
|
-
export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input:
|
|
10
|
-
url: string;
|
|
11
|
-
}) => Promise<WebFetchResponse>;
|
|
7
|
+
export declare function createSpecialContentResolver({ readers, fallback }: ResolverDeps): (input: ResearchFetchInput) => Promise<WebFetchResponse>;
|
|
12
8
|
export {};
|
package/dist/search/brave.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createBraveSearchTool({ apiKey, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
fetchImpl?: typeof fetch;
|
|
5
4
|
}): ({ query }: {
|
|
6
5
|
query: string;
|
|
7
|
-
}) => Promise<WebSearchResponse>;
|
|
6
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/brave.js
CHANGED
|
@@ -1,83 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
|
|
3
|
-
function resultWithPresentation(result) {
|
|
4
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
-
}
|
|
6
|
-
function normalizeResults(response) {
|
|
7
|
-
return (response.web?.results ?? []).flatMap((item) => {
|
|
8
|
-
if (typeof item.title !== 'string' || typeof item.url !== 'string') {
|
|
9
|
-
return [];
|
|
10
|
-
}
|
|
11
|
-
return [
|
|
12
|
-
{
|
|
13
|
-
title: item.title,
|
|
14
|
-
url: item.url,
|
|
15
|
-
snippet: typeof item.description === 'string' ? item.description : ''
|
|
16
|
-
}
|
|
17
|
-
];
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
function buildBraveUrl(query) {
|
|
21
|
-
const url = new URL(BRAVE_WEB_SEARCH_URL);
|
|
22
|
-
url.searchParams.set('q', query);
|
|
23
|
-
return url.toString();
|
|
24
|
-
}
|
|
25
3
|
export function createBraveSearchTool({ apiKey, fetchImpl = fetch }) {
|
|
26
|
-
return
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
|
|
49
|
-
headers: {
|
|
50
|
-
Accept: 'application/json',
|
|
51
|
-
'X-Subscription-Token': apiKey
|
|
52
|
-
}
|
|
53
|
-
});
|
|
54
|
-
if (!response.ok) {
|
|
55
|
-
throw new Error(`HTTP ${response.status}`);
|
|
56
|
-
}
|
|
57
|
-
const parsed = (await response.json());
|
|
58
|
-
const results = normalizeResults(parsed);
|
|
59
|
-
if (results.length === 0) {
|
|
60
|
-
return resultWithPresentation({
|
|
61
|
-
status: 'error',
|
|
62
|
-
results: [],
|
|
63
|
-
metadata: { backend: 'brave', cacheHit: false },
|
|
64
|
-
error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
return resultWithPresentation({
|
|
68
|
-
status: 'ok',
|
|
69
|
-
results,
|
|
70
|
-
metadata: { backend: 'brave', cacheHit: false }
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
catch (error) {
|
|
74
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
75
|
-
return resultWithPresentation({
|
|
76
|
-
status: 'error',
|
|
77
|
-
results: [],
|
|
78
|
-
metadata: { backend: 'brave', cacheHit: false },
|
|
79
|
-
error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
};
|
|
4
|
+
return createJsonSearchProvider({
|
|
5
|
+
name: 'brave',
|
|
6
|
+
label: 'Brave',
|
|
7
|
+
apiKey,
|
|
8
|
+
missingKeyMessage: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.',
|
|
9
|
+
fetchImpl,
|
|
10
|
+
request: (query) => {
|
|
11
|
+
const url = new URL(BRAVE_WEB_SEARCH_URL);
|
|
12
|
+
url.searchParams.set('q', query);
|
|
13
|
+
return { url: url.toString(), init: { headers: { Accept: 'application/json', 'X-Subscription-Token': apiKey ?? '' } } };
|
|
14
|
+
},
|
|
15
|
+
// A search response (`type: 'search'`) with no `web` block, or a `web` block with no `results`,
|
|
16
|
+
// is Brave's valid empty response. Anything else without `web` (an error-shaped or partial 200)
|
|
17
|
+
// is not trusted as empty.
|
|
18
|
+
normalize: (json) => normalizeResultsArray(json, (body) => {
|
|
19
|
+
if (body.web === undefined)
|
|
20
|
+
return body.type === 'search' ? [] : undefined;
|
|
21
|
+
if (!body.web || typeof body.web !== 'object')
|
|
22
|
+
return undefined;
|
|
23
|
+
return body.web.results === undefined ? [] : body.web.results;
|
|
24
|
+
}, 'description')
|
|
25
|
+
});
|
|
83
26
|
}
|
|
@@ -10,9 +10,13 @@ export declare const DUCKDUCKGO_HEADERS: {
|
|
|
10
10
|
readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
11
11
|
readonly 'Accept-Language': "en-US,en;q=0.9";
|
|
12
12
|
};
|
|
13
|
-
export declare
|
|
13
|
+
export declare class DuckDuckGoHttpError extends Error {
|
|
14
|
+
readonly status: number;
|
|
15
|
+
readonly headers: Headers;
|
|
16
|
+
constructor(status: number, headers: Headers);
|
|
17
|
+
}
|
|
18
|
+
/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
|
|
19
|
+
export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl }?: {
|
|
14
20
|
fetchImpl?: typeof fetch;
|
|
15
|
-
retries?: number;
|
|
16
|
-
sleep?: (ms: number) => Promise<void>;
|
|
17
21
|
}): Promise<string>;
|
|
18
22
|
export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;
|
|
@@ -26,25 +26,24 @@ export const DUCKDUCKGO_HEADERS = {
|
|
|
26
26
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
27
27
|
'Accept-Language': 'en-US,en;q=0.9'
|
|
28
28
|
};
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
29
|
+
export class DuckDuckGoHttpError extends Error {
|
|
30
|
+
status;
|
|
31
|
+
headers;
|
|
32
|
+
constructor(status, headers) {
|
|
33
|
+
super(`DuckDuckGo request failed with ${status}`);
|
|
34
|
+
this.status = status;
|
|
35
|
+
this.headers = headers;
|
|
36
|
+
this.name = 'DuckDuckGoHttpError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
|
|
40
|
+
export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch } = {}) {
|
|
41
|
+
const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
await response.body?.cancel().catch(() => undefined);
|
|
44
|
+
throw new DuckDuckGoHttpError(response.status, response.headers);
|
|
46
45
|
}
|
|
47
|
-
|
|
46
|
+
return response.text();
|
|
48
47
|
}
|
|
49
48
|
export function parseDuckDuckGoResults(html) {
|
|
50
49
|
const $ = cheerio.load(html);
|
package/dist/search/exa.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createExaSearchTool({ apiKey, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
fetchImpl?: typeof fetch;
|
|
5
4
|
}): ({ query }: {
|
|
6
5
|
query: string;
|
|
7
|
-
}) => Promise<WebSearchResponse>;
|
|
6
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/exa.js
CHANGED
|
@@ -1,81 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
const EXA_SEARCH_URL = 'https://api.exa.ai/search';
|
|
3
|
-
function resultWithPresentation(result) {
|
|
4
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
-
}
|
|
6
|
-
function normalizeResults(response) {
|
|
7
|
-
return (response.results ?? []).flatMap((item) => {
|
|
8
|
-
if (typeof item.title !== 'string' || typeof item.url !== 'string') {
|
|
9
|
-
return [];
|
|
10
|
-
}
|
|
11
|
-
return [
|
|
12
|
-
{
|
|
13
|
-
title: item.title,
|
|
14
|
-
url: item.url,
|
|
15
|
-
snippet: typeof item.text === 'string' ? item.text : ''
|
|
16
|
-
}
|
|
17
|
-
];
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
3
|
export function createExaSearchTool({ apiKey, fetchImpl = fetch }) {
|
|
21
|
-
return
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
if (!apiKey?.trim()) {
|
|
32
|
-
return resultWithPresentation({
|
|
33
|
-
status: 'error',
|
|
34
|
-
results: [],
|
|
35
|
-
metadata: { backend: 'exa', cacheHit: false },
|
|
36
|
-
error: {
|
|
37
|
-
code: 'BACKEND_CONFIG_INVALID',
|
|
38
|
-
message: 'Exa search requires EXA_API_KEY.'
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
try {
|
|
43
|
-
const response = await fetchImpl(EXA_SEARCH_URL, {
|
|
4
|
+
return createJsonSearchProvider({
|
|
5
|
+
name: 'exa',
|
|
6
|
+
label: 'Exa',
|
|
7
|
+
apiKey,
|
|
8
|
+
missingKeyMessage: 'Exa search requires EXA_API_KEY.',
|
|
9
|
+
fetchImpl,
|
|
10
|
+
request: (query) => ({
|
|
11
|
+
url: EXA_SEARCH_URL,
|
|
12
|
+
init: {
|
|
44
13
|
method: 'POST',
|
|
45
|
-
headers: {
|
|
46
|
-
|
|
47
|
-
'Content-Type': 'application/json',
|
|
48
|
-
'x-api-key': apiKey
|
|
49
|
-
},
|
|
50
|
-
body: JSON.stringify({ query: normalizedQuery, numResults: 10 })
|
|
51
|
-
});
|
|
52
|
-
if (!response.ok) {
|
|
53
|
-
throw new Error(`HTTP ${response.status}`);
|
|
54
|
-
}
|
|
55
|
-
const parsed = (await response.json());
|
|
56
|
-
const results = normalizeResults(parsed);
|
|
57
|
-
if (results.length === 0) {
|
|
58
|
-
return resultWithPresentation({
|
|
59
|
-
status: 'error',
|
|
60
|
-
results: [],
|
|
61
|
-
metadata: { backend: 'exa', cacheHit: false },
|
|
62
|
-
error: { code: 'NO_RESULTS', message: 'Exa returned no usable results for this query.' }
|
|
63
|
-
});
|
|
14
|
+
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'x-api-key': apiKey ?? '' },
|
|
15
|
+
body: JSON.stringify({ query, numResults: 10 })
|
|
64
16
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
metadata: { backend: 'exa', cacheHit: false }
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
catch (error) {
|
|
72
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
73
|
-
return resultWithPresentation({
|
|
74
|
-
status: 'error',
|
|
75
|
-
results: [],
|
|
76
|
-
metadata: { backend: 'exa', cacheHit: false },
|
|
77
|
-
error: { code: 'FETCH_FAILED', message: `Exa search request failed: ${rawMessage}` }
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
};
|
|
17
|
+
}),
|
|
18
|
+
normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'text')
|
|
19
|
+
});
|
|
81
20
|
}
|
package/dist/search/fanout.d.ts
CHANGED
|
@@ -5,10 +5,22 @@ export type FanoutProvider = {
|
|
|
5
5
|
query: string;
|
|
6
6
|
}) => Promise<WebSearchResponse>;
|
|
7
7
|
};
|
|
8
|
+
export declare const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
|
|
9
|
+
/** Longest a policy-wrapped provider can legitimately take: two timed-out calls and the retry sleep. */
|
|
10
|
+
export declare function fanoutBackstopMs(timeoutMs: number): number;
|
|
11
|
+
type SearchFn = FanoutProvider['search'];
|
|
12
|
+
/**
|
|
13
|
+
* Per-call timeout. Wrap it INSIDE the retry policy so a stalled call is a transient
|
|
14
|
+
* failure the policy can retry once (#55).
|
|
15
|
+
*/
|
|
16
|
+
export declare function withCallTimeout(search: SearchFn, timeoutMs: number, name: SearchProviderName): SearchFn;
|
|
8
17
|
export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
|
|
9
18
|
providers: FanoutProvider[];
|
|
10
19
|
mode: Exclude<FanoutMode, 'off'>;
|
|
20
|
+
/** Per-call timeout the providers apply themselves (see withCallTimeout). Fanout only keeps a
|
|
21
|
+
* backstop that can't cut a retry short. */
|
|
11
22
|
timeoutMs?: number;
|
|
12
23
|
}): ({ query }: {
|
|
13
24
|
query: string;
|
|
14
25
|
}) => Promise<WebSearchResponse>;
|
|
26
|
+
export {};
|