@demigodmode/pi-web-agent 1.11.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/backends/config.d.ts +13 -0
- package/dist/backends/config.js +44 -1
- package/dist/backends/factory.d.ts +15 -0
- package/dist/backends/factory.js +118 -91
- package/dist/backends/failure.d.ts +11 -0
- package/dist/backends/failure.js +34 -0
- package/dist/backends/fallback-policy.d.ts +33 -0
- package/dist/backends/fallback-policy.js +239 -0
- package/dist/backends/provider-failure.d.ts +21 -0
- package/dist/backends/provider-failure.js +111 -0
- package/dist/backends/provider-health.d.ts +29 -0
- package/dist/backends/provider-health.js +49 -0
- package/dist/commands/web-agent-config.d.ts +14 -1
- package/dist/commands/web-agent-config.js +75 -3
- package/dist/extension.js +47 -3
- package/dist/fetch/destination-policy.d.ts +32 -0
- package/dist/fetch/destination-policy.js +24 -0
- package/dist/fetch/firecrawl-fetch.js +64 -45
- package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
- package/dist/fetch/guard-proxy-fetch.js +82 -0
- package/dist/fetch/guard-proxy.d.ts +58 -0
- package/dist/fetch/guard-proxy.js +420 -0
- package/dist/fetch/guarded-fetch.d.ts +7 -0
- package/dist/fetch/guarded-fetch.js +75 -0
- package/dist/fetch/headless-fetch.d.ts +10 -2
- package/dist/fetch/headless-fetch.js +181 -9
- package/dist/fetch/http-fetch.js +16 -1
- package/dist/fetch/network-guard.d.ts +82 -0
- package/dist/fetch/network-guard.js +275 -0
- package/dist/orchestration/answer-synthesizer.js +2 -0
- package/dist/orchestration/evidence-quality.d.ts +3 -2
- package/dist/orchestration/evidence-quality.js +2 -1
- package/dist/orchestration/index.d.ts +23 -0
- package/dist/orchestration/index.js +9 -2
- package/dist/orchestration/research-orchestrator.d.ts +21 -1
- package/dist/orchestration/research-orchestrator.js +40 -7
- package/dist/orchestration/research-types.d.ts +13 -1
- package/dist/orchestration/research-worker.js +38 -3
- package/dist/orchestration/stop-decider.js +3 -1
- package/dist/presentation/config-store.js +6 -0
- package/dist/presentation/explore-presentation.js +3 -1
- package/dist/presentation/fetch-presentation.js +16 -9
- package/dist/presentation/search-presentation.d.ts +2 -1
- package/dist/presentation/search-presentation.js +13 -1
- package/dist/search/brave.d.ts +1 -2
- package/dist/search/brave.js +23 -80
- package/dist/search/duckduckgo.d.ts +7 -3
- package/dist/search/duckduckgo.js +17 -18
- package/dist/search/exa.d.ts +1 -2
- package/dist/search/exa.js +15 -76
- package/dist/search/fanout.d.ts +12 -0
- package/dist/search/fanout.js +86 -47
- package/dist/search/json-provider.d.ts +32 -0
- package/dist/search/json-provider.js +76 -0
- package/dist/search/searxng.d.ts +1 -2
- package/dist/search/searxng.js +15 -57
- package/dist/search/tavily.d.ts +1 -2
- package/dist/search/tavily.js +17 -74
- package/dist/search/youcom.d.ts +1 -2
- package/dist/search/youcom.js +15 -76
- package/dist/tools/web-explore.d.ts +9 -0
- package/dist/tools/web-explore.js +16 -2
- package/dist/tools/web-search.js +41 -103
- package/dist/types.d.ts +40 -0
- package/package.json +3 -3
|
@@ -19,6 +19,7 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
|
|
|
19
19
|
evidence: import("./research-types.js").ResearchEvidence[];
|
|
20
20
|
workerPass: import("./research-types.js").ResearchWorkerResult;
|
|
21
21
|
metadata: {
|
|
22
|
+
attempts?: import("../types.js").Attempt[] | undefined;
|
|
22
23
|
searchPasses: number;
|
|
23
24
|
fetchedPages: number;
|
|
24
25
|
headlessAttempts: number;
|
|
@@ -27,5 +28,27 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
|
|
|
27
28
|
fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
|
|
28
29
|
fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
|
|
29
30
|
};
|
|
31
|
+
terminalFailure?: undefined;
|
|
32
|
+
} | {
|
|
33
|
+
decision: import("./research-types.js").ResearchOrchestratorDecision;
|
|
34
|
+
evidence: never[];
|
|
35
|
+
workerPass: import("./research-types.js").ResearchWorkerResult;
|
|
36
|
+
metadata: {
|
|
37
|
+
attempts?: import("../types.js").Attempt[] | undefined;
|
|
38
|
+
searchPasses: number;
|
|
39
|
+
fetchedPages: number;
|
|
40
|
+
headlessAttempts: number;
|
|
41
|
+
exhaustedBudget: boolean;
|
|
42
|
+
caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
|
|
43
|
+
fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
|
|
44
|
+
fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
|
|
45
|
+
};
|
|
46
|
+
terminalFailure: {
|
|
47
|
+
code: string;
|
|
48
|
+
message: string;
|
|
49
|
+
};
|
|
30
50
|
}>;
|
|
51
|
+
} & {
|
|
52
|
+
/** Releases the backend set this workflow created. Injected capabilities are left alone. */
|
|
53
|
+
close(): Promise<void>;
|
|
31
54
|
};
|
|
@@ -2,14 +2,21 @@ import { createBackendSet } from '../backends/factory.js';
|
|
|
2
2
|
import { createResearchOrchestrator } from './research-orchestrator.js';
|
|
3
3
|
import { createResearchWorker } from './research-worker.js';
|
|
4
4
|
export function createResearchWorkflow({ backendConfig, search, fetchPage, headlessFetch } = {}) {
|
|
5
|
-
|
|
5
|
+
// Only build (and own) a backend set when something wasn't injected.
|
|
6
|
+
const backends = search && fetchPage && headlessFetch ? undefined : createBackendSet(backendConfig);
|
|
6
7
|
const resolvedSearch = search ?? backends.search;
|
|
7
8
|
const resolvedFetchPage = fetchPage ?? backends.fetchPage;
|
|
8
9
|
const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
|
|
9
10
|
const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
|
|
10
|
-
|
|
11
|
+
const orchestrator = createResearchOrchestrator({
|
|
11
12
|
worker,
|
|
12
13
|
fetchDirect: resolvedFetchPage,
|
|
13
14
|
headlessFetch: resolvedHeadlessFetch
|
|
14
15
|
});
|
|
16
|
+
return Object.assign(orchestrator, {
|
|
17
|
+
/** Releases the backend set this workflow created. Injected capabilities are left alone. */
|
|
18
|
+
async close() {
|
|
19
|
+
await backends?.close?.();
|
|
20
|
+
}
|
|
21
|
+
});
|
|
15
22
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
1
|
+
import type { Attempt, SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
2
2
|
import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
|
|
3
3
|
import { type EvidenceCaveatReason } from './evidence-quality.js';
|
|
4
4
|
export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
|
|
@@ -23,6 +23,7 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
|
|
|
23
23
|
evidence: ResearchEvidence[];
|
|
24
24
|
workerPass: ResearchWorkerResult;
|
|
25
25
|
metadata: {
|
|
26
|
+
attempts?: Attempt[] | undefined;
|
|
26
27
|
searchPasses: number;
|
|
27
28
|
fetchedPages: number;
|
|
28
29
|
headlessAttempts: number;
|
|
@@ -31,5 +32,24 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
|
|
|
31
32
|
fanoutProviders: SearchProviderName[] | undefined;
|
|
32
33
|
fanoutSkipped: SearchProviderName[] | undefined;
|
|
33
34
|
};
|
|
35
|
+
terminalFailure?: undefined;
|
|
36
|
+
} | {
|
|
37
|
+
decision: ResearchOrchestratorDecision;
|
|
38
|
+
evidence: never[];
|
|
39
|
+
workerPass: ResearchWorkerResult;
|
|
40
|
+
metadata: {
|
|
41
|
+
attempts?: Attempt[] | undefined;
|
|
42
|
+
searchPasses: number;
|
|
43
|
+
fetchedPages: number;
|
|
44
|
+
headlessAttempts: number;
|
|
45
|
+
exhaustedBudget: boolean;
|
|
46
|
+
caveatReasons: EvidenceCaveatReason[];
|
|
47
|
+
fanoutProviders: SearchProviderName[] | undefined;
|
|
48
|
+
fanoutSkipped: SearchProviderName[] | undefined;
|
|
49
|
+
};
|
|
50
|
+
terminalFailure: {
|
|
51
|
+
code: string;
|
|
52
|
+
message: string;
|
|
53
|
+
};
|
|
34
54
|
}>;
|
|
35
55
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { failureOf, isTerminalFailure } from '../backends/failure.js';
|
|
1
2
|
import { rankEvidence } from './evidence-ranker.js';
|
|
2
3
|
import { planSearchQueries } from './query-planner.js';
|
|
3
4
|
import { classifySourceProfile } from './source-profile.js';
|
|
@@ -79,7 +80,7 @@ function shouldRetryDirectWithHeadless(result, evidence) {
|
|
|
79
80
|
return false;
|
|
80
81
|
return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
|
|
81
82
|
}
|
|
82
|
-
function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
|
|
83
|
+
function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped, attempts }) {
|
|
83
84
|
return {
|
|
84
85
|
searchPasses: previousQueries.length,
|
|
85
86
|
fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
|
|
@@ -87,7 +88,8 @@ function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutco
|
|
|
87
88
|
exhaustedBudget,
|
|
88
89
|
caveatReasons,
|
|
89
90
|
fanoutProviders,
|
|
90
|
-
fanoutSkipped
|
|
91
|
+
fanoutSkipped,
|
|
92
|
+
...(attempts && attempts.length > 0 ? { attempts } : {})
|
|
91
93
|
};
|
|
92
94
|
}
|
|
93
95
|
function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
|
|
@@ -114,21 +116,33 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
114
116
|
const suggestedHeadlessUrls = [];
|
|
115
117
|
let headlessAttempts = 0;
|
|
116
118
|
let lastPass;
|
|
119
|
+
let searchCoveragePartial = false;
|
|
117
120
|
const fanoutProvidersSeen = new Set();
|
|
118
121
|
const fanoutSkippedSeen = new Set();
|
|
122
|
+
// Search and fetch attempts from every pass and direct URL, for verbose provenance.
|
|
123
|
+
const runAttempts = [];
|
|
119
124
|
function fanoutSnapshot() {
|
|
120
125
|
const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
|
|
121
126
|
const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
|
|
122
|
-
return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
|
|
127
|
+
return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined, attempts: [...runAttempts] };
|
|
123
128
|
}
|
|
124
129
|
if (fetchDirect) {
|
|
125
130
|
for (const url of extractDirectUrls(query).slice(0, 3)) {
|
|
126
131
|
const directResult = await fetchDirect({ url });
|
|
132
|
+
if (directResult.metadata.attempts)
|
|
133
|
+
runAttempts.push(...directResult.metadata.attempts);
|
|
127
134
|
const directEvidence = evidenceFromFetch(directResult);
|
|
128
135
|
if (directEvidence) {
|
|
129
136
|
allEvidence.push(directEvidence);
|
|
130
137
|
continue;
|
|
131
138
|
}
|
|
139
|
+
if (isTerminalFailure(failureOf(directResult))) {
|
|
140
|
+
allGaps.push({
|
|
141
|
+
kind: 'fetch-failed',
|
|
142
|
+
message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
|
|
143
|
+
});
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
132
146
|
if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
|
|
133
147
|
if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
|
|
134
148
|
headlessAttempts++;
|
|
@@ -161,7 +175,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
161
175
|
const quality = analyzeEvidenceQuality({
|
|
162
176
|
evidence: ranked,
|
|
163
177
|
gaps: allGaps,
|
|
164
|
-
lowValueOutcomes: allLowValueOutcomes
|
|
178
|
+
lowValueOutcomes: allLowValueOutcomes,
|
|
179
|
+
partialSearchCoverage: searchCoveragePartial
|
|
165
180
|
});
|
|
166
181
|
return {
|
|
167
182
|
decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
|
|
@@ -200,6 +215,21 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
200
215
|
maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
|
|
201
216
|
});
|
|
202
217
|
lastPass = pass;
|
|
218
|
+
if (pass.searchAttempts)
|
|
219
|
+
runAttempts.push(...pass.searchAttempts);
|
|
220
|
+
if (pass.fetchAttempts)
|
|
221
|
+
runAttempts.push(...pass.fetchAttempts);
|
|
222
|
+
if (pass.searchCoveragePartial)
|
|
223
|
+
searchCoveragePartial = true;
|
|
224
|
+
if (pass.terminalFailure) {
|
|
225
|
+
return {
|
|
226
|
+
decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked: [], exhaustedBudget: false }),
|
|
227
|
+
evidence: [],
|
|
228
|
+
workerPass: combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget: false }),
|
|
229
|
+
metadata: buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget: false, ...fanoutSnapshot() }),
|
|
230
|
+
terminalFailure: pass.terminalFailure
|
|
231
|
+
};
|
|
232
|
+
}
|
|
203
233
|
pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
|
|
204
234
|
pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
|
|
205
235
|
allEvidence.push(...pass.evidence);
|
|
@@ -211,7 +241,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
211
241
|
const quality = analyzeEvidenceQuality({
|
|
212
242
|
evidence: ranked,
|
|
213
243
|
gaps: allGaps,
|
|
214
|
-
lowValueOutcomes: allLowValueOutcomes
|
|
244
|
+
lowValueOutcomes: allLowValueOutcomes,
|
|
245
|
+
partialSearchCoverage: searchCoveragePartial
|
|
215
246
|
});
|
|
216
247
|
const decision = decideNextResearchStep({
|
|
217
248
|
evidence: ranked,
|
|
@@ -232,7 +263,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
232
263
|
const updatedQuality = analyzeEvidenceQuality({
|
|
233
264
|
evidence: updatedRanked,
|
|
234
265
|
gaps: allGaps,
|
|
235
|
-
lowValueOutcomes: allLowValueOutcomes
|
|
266
|
+
lowValueOutcomes: allLowValueOutcomes,
|
|
267
|
+
partialSearchCoverage: searchCoveragePartial
|
|
236
268
|
});
|
|
237
269
|
const updatedDecision = decideNextResearchStep({
|
|
238
270
|
evidence: updatedRanked,
|
|
@@ -328,7 +360,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
328
360
|
const quality = analyzeEvidenceQuality({
|
|
329
361
|
evidence: ranked,
|
|
330
362
|
gaps: allGaps,
|
|
331
|
-
lowValueOutcomes: allLowValueOutcomes
|
|
363
|
+
lowValueOutcomes: allLowValueOutcomes,
|
|
364
|
+
partialSearchCoverage: searchCoveragePartial
|
|
332
365
|
});
|
|
333
366
|
return {
|
|
334
367
|
decision: decisionForAnswer({ action: 'answer-with-caveat', query, ranked, exhaustedBudget: true }),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SearchProviderName } from '../types.js';
|
|
1
|
+
import type { Attempt, SearchProviderName } from '../types.js';
|
|
2
2
|
export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
|
|
3
3
|
export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
|
|
4
4
|
export type ResearchEvidence = {
|
|
@@ -27,6 +27,13 @@ export type ResearchWorkerResult = {
|
|
|
27
27
|
exhaustedBudget: boolean;
|
|
28
28
|
fanoutProviders?: SearchProviderName[];
|
|
29
29
|
fanoutSkipped?: SearchProviderName[];
|
|
30
|
+
searchCoveragePartial?: boolean;
|
|
31
|
+
searchAttempts?: Attempt[];
|
|
32
|
+
fetchAttempts?: Attempt[];
|
|
33
|
+
terminalFailure?: {
|
|
34
|
+
code: string;
|
|
35
|
+
message: string;
|
|
36
|
+
};
|
|
30
37
|
};
|
|
31
38
|
export type ResearchRunMetadata = {
|
|
32
39
|
searchPasses: number;
|
|
@@ -35,6 +42,11 @@ export type ResearchRunMetadata = {
|
|
|
35
42
|
exhaustedBudget: boolean;
|
|
36
43
|
fanoutProviders?: SearchProviderName[];
|
|
37
44
|
fanoutSkipped?: SearchProviderName[];
|
|
45
|
+
searchCoveragePartial?: boolean;
|
|
46
|
+
terminalFailure?: {
|
|
47
|
+
code: string;
|
|
48
|
+
message: string;
|
|
49
|
+
};
|
|
38
50
|
};
|
|
39
51
|
export type ResearchOrchestratorDecision = {
|
|
40
52
|
action: 'answer';
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { failureOf, isTerminalFailure } from '../backends/failure.js';
|
|
1
2
|
import { selectCandidates } from './candidate-selector.js';
|
|
2
3
|
import { classifySourceProfile } from './source-profile.js';
|
|
3
4
|
function classifySource(url) {
|
|
@@ -80,6 +81,22 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
80
81
|
};
|
|
81
82
|
}
|
|
82
83
|
const searchResult = await search({ query });
|
|
84
|
+
const searchCoveragePartial = searchResult.metadata.coverage?.partial === true;
|
|
85
|
+
const searchAttempts = searchResult.metadata.attempts;
|
|
86
|
+
if (isTerminalFailure(failureOf(searchResult))) {
|
|
87
|
+
return {
|
|
88
|
+
searchQueries,
|
|
89
|
+
evidence,
|
|
90
|
+
gaps: [],
|
|
91
|
+
lowValueOutcomes,
|
|
92
|
+
exhaustedBudget: false,
|
|
93
|
+
searchAttempts,
|
|
94
|
+
terminalFailure: {
|
|
95
|
+
code: searchResult.error?.code ?? 'SEARCH_FAILED',
|
|
96
|
+
message: `${searchResult.error?.message ?? 'Search failed.'} (${searchResult.error?.failure?.kind})`
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
}
|
|
83
100
|
const fanoutProviders = searchResult.metadata.fanout?.providers;
|
|
84
101
|
const fanoutSkipped = searchResult.metadata.fanout?.skipped;
|
|
85
102
|
if (searchResult.status !== 'ok') {
|
|
@@ -96,7 +113,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
96
113
|
suggestedHeadlessUrl,
|
|
97
114
|
exhaustedBudget: false,
|
|
98
115
|
fanoutProviders,
|
|
99
|
-
fanoutSkipped
|
|
116
|
+
fanoutSkipped,
|
|
117
|
+
searchCoveragePartial,
|
|
118
|
+
searchAttempts
|
|
100
119
|
};
|
|
101
120
|
}
|
|
102
121
|
if (searchResult.results.length === 0) {
|
|
@@ -113,7 +132,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
113
132
|
suggestedHeadlessUrl,
|
|
114
133
|
exhaustedBudget: false,
|
|
115
134
|
fanoutProviders,
|
|
116
|
-
fanoutSkipped
|
|
135
|
+
fanoutSkipped,
|
|
136
|
+
searchCoveragePartial,
|
|
137
|
+
searchAttempts
|
|
117
138
|
};
|
|
118
139
|
}
|
|
119
140
|
const candidates = selectCandidates({
|
|
@@ -122,8 +143,11 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
122
143
|
seenUrls: new Set(evidence.map((item) => item.url)),
|
|
123
144
|
maxCandidates: maxFetches
|
|
124
145
|
});
|
|
146
|
+
const fetchAttempts = [];
|
|
125
147
|
for (const candidate of candidates) {
|
|
126
148
|
const fetched = await fetchPage({ url: candidate.url });
|
|
149
|
+
if (fetched.metadata.attempts)
|
|
150
|
+
fetchAttempts.push(...fetched.metadata.attempts);
|
|
127
151
|
if (fetched.status === 'ok') {
|
|
128
152
|
const parsedEvidence = evidenceFromFetch(fetched, candidate.title);
|
|
129
153
|
if (parsedEvidence) {
|
|
@@ -136,6 +160,14 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
136
160
|
}
|
|
137
161
|
continue;
|
|
138
162
|
}
|
|
163
|
+
// guard_refused and friends are final; never hand them to headless.
|
|
164
|
+
if (isTerminalFailure(failureOf(fetched))) {
|
|
165
|
+
gaps.push({
|
|
166
|
+
kind: 'fetch-failed',
|
|
167
|
+
message: fetched.error?.message ?? `Fetch failed for ${candidate.url}`
|
|
168
|
+
});
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
139
171
|
if (fetched.status === 'needs_headless') {
|
|
140
172
|
if (!suggestedHeadlessUrl) {
|
|
141
173
|
suggestedHeadlessUrl = fetched.url;
|
|
@@ -156,7 +188,10 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
156
188
|
suggestedHeadlessUrl,
|
|
157
189
|
exhaustedBudget: false,
|
|
158
190
|
fanoutProviders,
|
|
159
|
-
fanoutSkipped
|
|
191
|
+
fanoutSkipped,
|
|
192
|
+
searchCoveragePartial,
|
|
193
|
+
searchAttempts,
|
|
194
|
+
fetchAttempts
|
|
160
195
|
};
|
|
161
196
|
}
|
|
162
197
|
};
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
|
|
2
2
|
function activeCaveatReasons(evidence, quality) {
|
|
3
|
-
|
|
3
|
+
// Partial search coverage is reported as a caveat but never changes the decision (#55):
|
|
4
|
+
// a strong answer stays an answer, and web-explore adds just the coverage sentence.
|
|
5
|
+
const reasons = (quality?.caveatReasons ?? []).filter((reason) => reason !== 'partial-search-coverage');
|
|
4
6
|
if (!hasOfficialDocsAndApi(evidence))
|
|
5
7
|
return reasons;
|
|
6
8
|
return reasons.filter((reason) => reason !== 'low-diversity');
|
|
@@ -65,6 +65,12 @@ function serializeBackendConfigOverride(config) {
|
|
|
65
65
|
const { password: _password, ...proxy } = config.proxy;
|
|
66
66
|
backends.proxy = { ...proxy };
|
|
67
67
|
}
|
|
68
|
+
if (config.network) {
|
|
69
|
+
backends.network = {
|
|
70
|
+
...config.network,
|
|
71
|
+
...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
|
|
72
|
+
};
|
|
73
|
+
}
|
|
68
74
|
return { backends };
|
|
69
75
|
}
|
|
70
76
|
async function readConfigFileForWrite(filePath) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attemptLines } from './search-presentation.js';
|
|
1
2
|
function internalReaderLabel(method) {
|
|
2
3
|
if (method === 'headless')
|
|
3
4
|
return 'web_fetch_headless';
|
|
@@ -49,7 +50,8 @@ export function buildExplorePresentation(result) {
|
|
|
49
50
|
'Sources',
|
|
50
51
|
...result.sources.map((source) => `- [${internalReaderLabel(source.method)}] ${source.title}: ${source.url}`),
|
|
51
52
|
internalSummary ? `\nInternal tools\n${internalSummary}` : undefined,
|
|
52
|
-
result.caveat ? `\nCaveat\n${result.caveat}` : undefined
|
|
53
|
+
result.caveat ? `\nCaveat\n${result.caveat}` : undefined,
|
|
54
|
+
attemptLines(result.metadata?.attempts)
|
|
53
55
|
]
|
|
54
56
|
.filter((line) => line !== undefined)
|
|
55
57
|
.join('\n');
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { attemptLines } from './search-presentation.js';
|
|
1
2
|
function countWords(text) {
|
|
2
3
|
return text?.trim() ? text.trim().split(/\s+/).length : undefined;
|
|
3
4
|
}
|
|
@@ -24,15 +25,21 @@ export function buildFetchPresentation(result) {
|
|
|
24
25
|
preview: result.content?.title
|
|
25
26
|
? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
|
|
26
27
|
: firstExcerpt(result.content?.text),
|
|
27
|
-
verbose:
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
verbose: [
|
|
29
|
+
...(result.status === 'ok'
|
|
30
|
+
? [
|
|
31
|
+
`URL: ${result.url}`,
|
|
32
|
+
result.content?.title ? `Title: ${result.content.title}` : undefined,
|
|
33
|
+
firstExcerpt(result.content?.text, 500),
|
|
34
|
+
result.metadata.blockedSubresources
|
|
35
|
+
? `Blocked private-address requests: ${result.metadata.blockedSubresources}`
|
|
36
|
+
: undefined
|
|
37
|
+
]
|
|
38
|
+
: []),
|
|
39
|
+
attemptLines(result.metadata.attempts)
|
|
40
|
+
]
|
|
41
|
+
.filter(Boolean)
|
|
42
|
+
.join('\n') || undefined
|
|
36
43
|
},
|
|
37
44
|
metrics: {
|
|
38
45
|
wordCount,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
1
|
+
import type { Attempt, WebSearchResponse } from '../types.js';
|
|
2
2
|
import type { PresentationEnvelope } from './types.js';
|
|
3
|
+
export declare function attemptLines(attempts: Attempt[] | undefined): string | undefined;
|
|
3
4
|
export declare function buildSearchPresentation(result: WebSearchResponse): PresentationEnvelope;
|
|
@@ -10,6 +10,18 @@ function fanoutNote(result) {
|
|
|
10
10
|
}
|
|
11
11
|
return '';
|
|
12
12
|
}
|
|
13
|
+
export function attemptLines(attempts) {
|
|
14
|
+
const interesting = (attempts ?? []).filter((a) => a.outcome !== 'results' && a.outcome !== 'empty');
|
|
15
|
+
if (interesting.length === 0)
|
|
16
|
+
return undefined;
|
|
17
|
+
return interesting
|
|
18
|
+
.map((a) => {
|
|
19
|
+
const kind = a.failure?.kind ? ` (${a.failure.kind})` : '';
|
|
20
|
+
const until = a.cooldownUntil !== undefined ? `, cooling down until ${new Date(a.cooldownUntil).toISOString()}` : '';
|
|
21
|
+
return `${a.backend}: ${a.outcome}${a.skipReason ? ` [${a.skipReason}]` : ''}${kind}${until}`;
|
|
22
|
+
})
|
|
23
|
+
.join('\n');
|
|
24
|
+
}
|
|
13
25
|
function formatCompact(result) {
|
|
14
26
|
const fallbackPrefix = result.metadata.fallbackFrom
|
|
15
27
|
? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
|
|
@@ -34,7 +46,7 @@ export function buildSearchPresentation(result) {
|
|
|
34
46
|
views: {
|
|
35
47
|
compact: formatCompact(result),
|
|
36
48
|
preview: preview || undefined,
|
|
37
|
-
verbose: verbose || undefined
|
|
49
|
+
verbose: [verbose, attemptLines(result.metadata.attempts)].filter(Boolean).join('\n') || undefined
|
|
38
50
|
},
|
|
39
51
|
metrics: {
|
|
40
52
|
resultCount: result.results.length,
|
package/dist/search/brave.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createBraveSearchTool({ apiKey, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
fetchImpl?: typeof fetch;
|
|
5
4
|
}): ({ query }: {
|
|
6
5
|
query: string;
|
|
7
|
-
}) => Promise<WebSearchResponse>;
|
|
6
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/brave.js
CHANGED
|
@@ -1,83 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
|
|
3
|
-
function resultWithPresentation(result) {
|
|
4
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
-
}
|
|
6
|
-
function normalizeResults(response) {
|
|
7
|
-
return (response.web?.results ?? []).flatMap((item) => {
|
|
8
|
-
if (typeof item.title !== 'string' || typeof item.url !== 'string') {
|
|
9
|
-
return [];
|
|
10
|
-
}
|
|
11
|
-
return [
|
|
12
|
-
{
|
|
13
|
-
title: item.title,
|
|
14
|
-
url: item.url,
|
|
15
|
-
snippet: typeof item.description === 'string' ? item.description : ''
|
|
16
|
-
}
|
|
17
|
-
];
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
function buildBraveUrl(query) {
|
|
21
|
-
const url = new URL(BRAVE_WEB_SEARCH_URL);
|
|
22
|
-
url.searchParams.set('q', query);
|
|
23
|
-
return url.toString();
|
|
24
|
-
}
|
|
25
3
|
export function createBraveSearchTool({ apiKey, fetchImpl = fetch }) {
|
|
26
|
-
return
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
|
|
49
|
-
headers: {
|
|
50
|
-
Accept: 'application/json',
|
|
51
|
-
'X-Subscription-Token': apiKey
|
|
52
|
-
}
|
|
53
|
-
});
|
|
54
|
-
if (!response.ok) {
|
|
55
|
-
throw new Error(`HTTP ${response.status}`);
|
|
56
|
-
}
|
|
57
|
-
const parsed = (await response.json());
|
|
58
|
-
const results = normalizeResults(parsed);
|
|
59
|
-
if (results.length === 0) {
|
|
60
|
-
return resultWithPresentation({
|
|
61
|
-
status: 'error',
|
|
62
|
-
results: [],
|
|
63
|
-
metadata: { backend: 'brave', cacheHit: false },
|
|
64
|
-
error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
return resultWithPresentation({
|
|
68
|
-
status: 'ok',
|
|
69
|
-
results,
|
|
70
|
-
metadata: { backend: 'brave', cacheHit: false }
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
catch (error) {
|
|
74
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
75
|
-
return resultWithPresentation({
|
|
76
|
-
status: 'error',
|
|
77
|
-
results: [],
|
|
78
|
-
metadata: { backend: 'brave', cacheHit: false },
|
|
79
|
-
error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
};
|
|
4
|
+
return createJsonSearchProvider({
|
|
5
|
+
name: 'brave',
|
|
6
|
+
label: 'Brave',
|
|
7
|
+
apiKey,
|
|
8
|
+
missingKeyMessage: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.',
|
|
9
|
+
fetchImpl,
|
|
10
|
+
request: (query) => {
|
|
11
|
+
const url = new URL(BRAVE_WEB_SEARCH_URL);
|
|
12
|
+
url.searchParams.set('q', query);
|
|
13
|
+
return { url: url.toString(), init: { headers: { Accept: 'application/json', 'X-Subscription-Token': apiKey ?? '' } } };
|
|
14
|
+
},
|
|
15
|
+
// A search response (`type: 'search'`) with no `web` block, or a `web` block with no `results`,
|
|
16
|
+
// is Brave's valid empty response. Anything else without `web` (an error-shaped or partial 200)
|
|
17
|
+
// is not trusted as empty.
|
|
18
|
+
normalize: (json) => normalizeResultsArray(json, (body) => {
|
|
19
|
+
if (body.web === undefined)
|
|
20
|
+
return body.type === 'search' ? [] : undefined;
|
|
21
|
+
if (!body.web || typeof body.web !== 'object')
|
|
22
|
+
return undefined;
|
|
23
|
+
return body.web.results === undefined ? [] : body.web.results;
|
|
24
|
+
}, 'description')
|
|
25
|
+
});
|
|
83
26
|
}
|
|
@@ -10,9 +10,13 @@ export declare const DUCKDUCKGO_HEADERS: {
|
|
|
10
10
|
readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
11
11
|
readonly 'Accept-Language': "en-US,en;q=0.9";
|
|
12
12
|
};
|
|
13
|
-
export declare
|
|
13
|
+
export declare class DuckDuckGoHttpError extends Error {
|
|
14
|
+
readonly status: number;
|
|
15
|
+
readonly headers: Headers;
|
|
16
|
+
constructor(status: number, headers: Headers);
|
|
17
|
+
}
|
|
18
|
+
/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
|
|
19
|
+
export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl }?: {
|
|
14
20
|
fetchImpl?: typeof fetch;
|
|
15
|
-
retries?: number;
|
|
16
|
-
sleep?: (ms: number) => Promise<void>;
|
|
17
21
|
}): Promise<string>;
|
|
18
22
|
export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;
|
|
@@ -26,25 +26,24 @@ export const DUCKDUCKGO_HEADERS = {
|
|
|
26
26
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
27
27
|
'Accept-Language': 'en-US,en;q=0.9'
|
|
28
28
|
};
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
}
|
|
29
|
+
export class DuckDuckGoHttpError extends Error {
|
|
30
|
+
status;
|
|
31
|
+
headers;
|
|
32
|
+
constructor(status, headers) {
|
|
33
|
+
super(`DuckDuckGo request failed with ${status}`);
|
|
34
|
+
this.status = status;
|
|
35
|
+
this.headers = headers;
|
|
36
|
+
this.name = 'DuckDuckGoHttpError';
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/** One request. Retries belong to the fallback policy (#55), which only retries transient failures. */
|
|
40
|
+
export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch } = {}) {
|
|
41
|
+
const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
await response.body?.cancel().catch(() => undefined);
|
|
44
|
+
throw new DuckDuckGoHttpError(response.status, response.headers);
|
|
46
45
|
}
|
|
47
|
-
|
|
46
|
+
return response.text();
|
|
48
47
|
}
|
|
49
48
|
export function parseDuckDuckGoResults(html) {
|
|
50
49
|
const $ = cheerio.load(html);
|
package/dist/search/exa.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createExaSearchTool({ apiKey, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
fetchImpl?: typeof fetch;
|
|
5
4
|
}): ({ query }: {
|
|
6
5
|
query: string;
|
|
7
|
-
}) => Promise<WebSearchResponse>;
|
|
6
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|