@demigodmode/pi-web-agent 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -1
- package/dist/backends/config.d.ts +8 -0
- package/dist/backends/config.js +45 -0
- package/dist/backends/doctor.js +33 -0
- package/dist/backends/factory.js +36 -1
- package/dist/commands/web-agent-config.js +63 -6
- package/dist/extension.js +53 -12
- package/dist/orchestration/direct-url.js +2 -25
- package/dist/orchestration/evidence-quality.d.ts +1 -0
- package/dist/orchestration/evidence-quality.js +4 -2
- package/dist/orchestration/evidence-ranker.js +2 -0
- package/dist/orchestration/index.d.ts +2 -0
- package/dist/orchestration/research-orchestrator.d.ts +3 -1
- package/dist/orchestration/research-orchestrator.js +63 -6
- package/dist/orchestration/research-types.d.ts +6 -1
- package/dist/orchestration/research-worker.js +26 -3
- package/dist/orchestration/url.d.ts +6 -0
- package/dist/orchestration/url.js +34 -0
- package/dist/presentation/explore-presentation.js +12 -4
- package/dist/presentation/search-presentation.js +14 -2
- package/dist/readers/github-reader.js +3 -2
- package/dist/readers/limits.d.ts +3 -0
- package/dist/readers/limits.js +3 -0
- package/dist/readers/pdf-reader.js +3 -2
- package/dist/readers/youtube-reader.js +3 -2
- package/dist/search/fanout.d.ts +14 -0
- package/dist/search/fanout.js +118 -0
- package/dist/tools/web-explore.d.ts +2 -0
- package/dist/types.d.ts +10 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -18,9 +18,24 @@ The format is intentionally simple and release-oriented.
|
|
|
18
18
|
### Breaking
|
|
19
19
|
- None.
|
|
20
20
|
|
|
21
|
+
## [1.9.0] - 2026-08-15
|
|
22
|
+
### Added
|
|
23
|
+
- Search fanout. Ask a hard question and web_explore can now hit several of your configured search backends at once, dedupe the merged results, and rank the pages that more than one provider agreed on to the top. Off by default. Flip it to on or auto in Settings → Backends (auto only fans out when the first provider comes back thin, so easy queries stay cheap). Preview/verbose shows which providers ran and which got skipped. (#35)
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- None.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- web_explore can finally summarize a link you hand it. Paste a GitHub file/issue/PR, a PDF, or a YouTube URL and ask for a summary: it now reads the actual content (raw source, extracted PDF text, the video transcript) and gives the model the whole thing. 1.8.0 added these readers but then squeezed everything down to a ~180-char snippet before the model ever saw it, so summaries were basically guesswork. Now they're grounded in the real text (long content still capped around 24k characters). (#40, #41)
|
|
30
|
+
- Fanout won't stall on a dead provider anymore. Each one gets an 8s timeout, so an unreachable self-hosted SearXNG can't hang a whole research pass. (#35)
|
|
31
|
+
- Fanout only offers and queries the providers you've actually set up (keyless DuckDuckGo, SearXNG when you give it a URL, the hosted ones only when their key is present) instead of acting like all six are on. (#35)
|
|
32
|
+
|
|
33
|
+
### Breaking
|
|
34
|
+
- None.
|
|
35
|
+
|
|
21
36
|
## [1.8.0] - 2026-08-14
|
|
22
37
|
### Added
|
|
23
|
-
- Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript. All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
|
|
38
|
+
- Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript (long content capped around 24k characters). All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
|
|
24
39
|
|
|
25
40
|
### Changed
|
|
26
41
|
- None.
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
+
import type { FanoutMode, SearchProviderName } from '../types.js';
|
|
1
2
|
export type SearxngOptions = {
|
|
2
3
|
categories?: string[];
|
|
3
4
|
language?: string;
|
|
4
5
|
safesearch?: 0 | 1 | 2;
|
|
5
6
|
};
|
|
7
|
+
export type FanoutConfig = {
|
|
8
|
+
mode: FanoutMode;
|
|
9
|
+
providers?: SearchProviderName[];
|
|
10
|
+
};
|
|
6
11
|
export type FirecrawlOptions = {
|
|
7
12
|
formats?: string[];
|
|
8
13
|
onlyMainContent?: boolean;
|
|
@@ -12,6 +17,7 @@ export type SearchBackendConfig = {
|
|
|
12
17
|
baseUrl?: string;
|
|
13
18
|
fallback?: 'duckduckgo';
|
|
14
19
|
options?: SearxngOptions;
|
|
20
|
+
fanout?: FanoutConfig;
|
|
15
21
|
};
|
|
16
22
|
export type FetchBackendConfig = {
|
|
17
23
|
provider: 'http' | 'firecrawl';
|
|
@@ -40,6 +46,7 @@ export type BackendConfigFile = {
|
|
|
40
46
|
baseUrl?: unknown;
|
|
41
47
|
fallback?: unknown;
|
|
42
48
|
options?: unknown;
|
|
49
|
+
fanout?: unknown;
|
|
43
50
|
};
|
|
44
51
|
fetch?: {
|
|
45
52
|
provider?: unknown;
|
|
@@ -54,6 +61,7 @@ export type BackendConfigFile = {
|
|
|
54
61
|
};
|
|
55
62
|
};
|
|
56
63
|
export declare const DEFAULT_BACKEND_CONFIG: BackendConfig;
|
|
64
|
+
export declare function usableSearchProviders(search: SearchBackendConfig, env?: NodeJS.ProcessEnv): SearchProviderName[];
|
|
57
65
|
export declare function extractBackendConfigOverride(file: BackendConfigFile | null | undefined): BackendConfigOverride;
|
|
58
66
|
export declare function validateBackendConfig(config: BackendConfig): string[];
|
|
59
67
|
export declare function mergeBackendConfigLayers(...layers: Array<BackendConfig | BackendConfigOverride | undefined>): BackendConfig;
|
package/dist/backends/config.js
CHANGED
|
@@ -35,6 +35,38 @@ function extractFirecrawlOptions(value) {
|
|
|
35
35
|
options.onlyMainContent = raw.onlyMainContent;
|
|
36
36
|
return Object.keys(options).length > 0 ? options : undefined;
|
|
37
37
|
}
|
|
38
|
+
const PROVIDER_NAMES = ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
|
|
39
|
+
export function usableSearchProviders(search, env = process.env) {
|
|
40
|
+
// Match the provider implementations, which treat a blank/whitespace key as unconfigured.
|
|
41
|
+
const usable = ['duckduckgo']; // keyless, always usable
|
|
42
|
+
if (search.baseUrl?.trim())
|
|
43
|
+
usable.push('searxng');
|
|
44
|
+
if (env.PI_WEB_AGENT_BRAVE_API_KEY?.trim())
|
|
45
|
+
usable.push('brave');
|
|
46
|
+
if (env.YDC_API_KEY?.trim())
|
|
47
|
+
usable.push('youcom');
|
|
48
|
+
if (env.EXA_API_KEY?.trim())
|
|
49
|
+
usable.push('exa');
|
|
50
|
+
if (env.TAVILY_API_KEY?.trim())
|
|
51
|
+
usable.push('tavily');
|
|
52
|
+
return usable;
|
|
53
|
+
}
|
|
54
|
+
function extractFanoutConfig(value) {
|
|
55
|
+
if (!value || typeof value !== 'object')
|
|
56
|
+
return undefined;
|
|
57
|
+
const raw = value;
|
|
58
|
+
if (raw.mode !== 'off' && raw.mode !== 'on' && raw.mode !== 'auto')
|
|
59
|
+
return undefined;
|
|
60
|
+
const config = { mode: raw.mode };
|
|
61
|
+
if (Array.isArray(raw.providers)) {
|
|
62
|
+
const providers = raw.providers.filter((p) => typeof p === 'string' && PROVIDER_NAMES.includes(p));
|
|
63
|
+
if (providers.length !== raw.providers.length)
|
|
64
|
+
return undefined; // fail loud on any invalid entry
|
|
65
|
+
if (providers.length > 0)
|
|
66
|
+
config.providers = providers;
|
|
67
|
+
}
|
|
68
|
+
return config;
|
|
69
|
+
}
|
|
38
70
|
export function extractBackendConfigOverride(file) {
|
|
39
71
|
const backends = file?.backends;
|
|
40
72
|
const override = {};
|
|
@@ -58,6 +90,10 @@ export function extractBackendConfigOverride(file) {
|
|
|
58
90
|
}
|
|
59
91
|
}
|
|
60
92
|
}
|
|
93
|
+
const fanout = extractFanoutConfig(backends?.search?.fanout);
|
|
94
|
+
if (fanout) {
|
|
95
|
+
override.search = { ...(override.search ?? {}), fanout };
|
|
96
|
+
}
|
|
61
97
|
if (backends?.fetch?.provider === 'http' || backends?.fetch?.provider === 'firecrawl') {
|
|
62
98
|
override.fetch = { provider: backends.fetch.provider };
|
|
63
99
|
if (typeof backends.fetch.baseUrl === 'string') {
|
|
@@ -106,6 +142,15 @@ export function validateBackendConfig(config) {
|
|
|
106
142
|
if (config.fetch.options?.formats && config.fetch.options.formats.length === 0) {
|
|
107
143
|
issues.push('fetch options.formats must contain at least one format when provided');
|
|
108
144
|
}
|
|
145
|
+
const fanout = config.search.fanout;
|
|
146
|
+
if (fanout) {
|
|
147
|
+
if (fanout.mode !== 'off' && fanout.mode !== 'on' && fanout.mode !== 'auto') {
|
|
148
|
+
issues.push('search fanout.mode must be off, on, or auto');
|
|
149
|
+
}
|
|
150
|
+
if (fanout.providers?.includes('searxng') && !config.search.baseUrl) {
|
|
151
|
+
issues.push('search fanout with searxng requires backends.search.baseUrl');
|
|
152
|
+
}
|
|
153
|
+
}
|
|
109
154
|
return issues;
|
|
110
155
|
}
|
|
111
156
|
function mergeSearchConfig(current, override) {
|
package/dist/backends/doctor.js
CHANGED
|
@@ -208,6 +208,39 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
|
|
|
208
208
|
if (config.search.fallback) {
|
|
209
209
|
lines.push(`search fallback: ${config.search.fallback}`);
|
|
210
210
|
}
|
|
211
|
+
if (config.search.fanout && config.search.fanout.mode !== 'off') {
|
|
212
|
+
const providers = config.search.fanout.providers || ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
|
|
213
|
+
lines.push(`search fanout: ${config.search.fanout.mode} (${providers.join(', ')})`);
|
|
214
|
+
for (const provider of providers) {
|
|
215
|
+
if (provider === 'searxng' && !config.search.baseUrl) {
|
|
216
|
+
lines.push('search fanout provider searxng warning (missing baseUrl)');
|
|
217
|
+
}
|
|
218
|
+
else if (provider === 'brave') {
|
|
219
|
+
const apiKey = process.env.PI_WEB_AGENT_BRAVE_API_KEY;
|
|
220
|
+
if (!apiKey?.trim()) {
|
|
221
|
+
lines.push('search fanout provider brave warning (missing PI_WEB_AGENT_BRAVE_API_KEY)');
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
else if (provider === 'youcom') {
|
|
225
|
+
const apiKey = process.env.YDC_API_KEY;
|
|
226
|
+
if (!apiKey?.trim()) {
|
|
227
|
+
lines.push('search fanout provider youcom warning (missing YDC_API_KEY)');
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
else if (provider === 'exa') {
|
|
231
|
+
const apiKey = process.env.EXA_API_KEY;
|
|
232
|
+
if (!apiKey?.trim()) {
|
|
233
|
+
lines.push('search fanout provider exa warning (missing EXA_API_KEY)');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
else if (provider === 'tavily') {
|
|
237
|
+
const apiKey = process.env.TAVILY_API_KEY;
|
|
238
|
+
if (!apiKey?.trim()) {
|
|
239
|
+
lines.push('search fanout provider tavily warning (missing TAVILY_API_KEY)');
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
211
244
|
if (config.fetch.provider === 'http') {
|
|
212
245
|
lines.push('fetch backend: http');
|
|
213
246
|
}
|
package/dist/backends/factory.js
CHANGED
|
@@ -4,12 +4,13 @@ import { createYouComSearchTool } from '../search/youcom.js';
|
|
|
4
4
|
import { createExaSearchTool } from '../search/exa.js';
|
|
5
5
|
import { createTavilySearchTool } from '../search/tavily.js';
|
|
6
6
|
import { createSearxngSearchTool } from '../search/searxng.js';
|
|
7
|
+
import { createFanoutSearch } from '../search/fanout.js';
|
|
7
8
|
import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
|
|
8
9
|
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
9
10
|
import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
|
|
10
11
|
import { createWebFetchTool } from '../tools/web-fetch.js';
|
|
11
12
|
import { createWebSearchTool } from '../tools/web-search.js';
|
|
12
|
-
import { DEFAULT_BACKEND_CONFIG } from './config.js';
|
|
13
|
+
import { DEFAULT_BACKEND_CONFIG, usableSearchProviders } from './config.js';
|
|
13
14
|
import { createSpecialContentResolver } from '../readers/resolver.js';
|
|
14
15
|
import { createGithubReader } from '../readers/github-reader.js';
|
|
15
16
|
import { createPdfReader } from '../readers/pdf-reader.js';
|
|
@@ -86,6 +87,25 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
86
87
|
const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
|
|
87
88
|
const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
|
|
88
89
|
const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
|
|
90
|
+
function buildProviderSearch(name) {
|
|
91
|
+
switch (name) {
|
|
92
|
+
case 'searxng':
|
|
93
|
+
return config.search.baseUrl
|
|
94
|
+
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
95
|
+
: invalidSearxngSearch();
|
|
96
|
+
case 'brave':
|
|
97
|
+
return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY });
|
|
98
|
+
case 'youcom':
|
|
99
|
+
return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
|
|
100
|
+
case 'exa':
|
|
101
|
+
return createExaSearch({ apiKey: process.env.EXA_API_KEY });
|
|
102
|
+
case 'tavily':
|
|
103
|
+
return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
|
|
104
|
+
case 'duckduckgo':
|
|
105
|
+
default:
|
|
106
|
+
return createDuckDuckGoSearch();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
89
109
|
let search = config.search.provider === 'searxng'
|
|
90
110
|
? config.search.baseUrl
|
|
91
111
|
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
@@ -114,6 +134,21 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
114
134
|
if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
|
|
115
135
|
search = withSearchFallback(search, createDuckDuckGoSearch(), 'tavily');
|
|
116
136
|
}
|
|
137
|
+
const fanoutConfig = config.search.fanout;
|
|
138
|
+
if (fanoutConfig && fanoutConfig.mode !== 'off') {
|
|
139
|
+
const baseNames = fanoutConfig.providers && fanoutConfig.providers.length > 0
|
|
140
|
+
? fanoutConfig.providers
|
|
141
|
+
: usableSearchProviders(config.search);
|
|
142
|
+
// A configured DuckDuckGo fallback must still be honored under fanout: fold it into the set.
|
|
143
|
+
const providerNames = config.search.fallback === 'duckduckgo' && !baseNames.includes('duckduckgo')
|
|
144
|
+
? [...baseNames, 'duckduckgo']
|
|
145
|
+
: baseNames;
|
|
146
|
+
const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter((n, i, arr) => arr.indexOf(n) === i);
|
|
147
|
+
search = createFanoutSearch({
|
|
148
|
+
providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
|
|
149
|
+
mode: fanoutConfig.mode
|
|
150
|
+
});
|
|
151
|
+
}
|
|
117
152
|
const httpFetch = createHttpFetch();
|
|
118
153
|
let fetchPage = config.fetch.provider === 'firecrawl'
|
|
119
154
|
? config.fetch.baseUrl
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig } from '../backends/config.js';
|
|
1
|
+
import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
|
|
2
2
|
import { checkBackendHealth } from '../backends/doctor.js';
|
|
3
3
|
import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
|
|
4
4
|
import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
|
|
@@ -20,7 +20,10 @@ function cloneBackendConfig(config) {
|
|
|
20
20
|
return {
|
|
21
21
|
search: {
|
|
22
22
|
...config.search,
|
|
23
|
-
options: config.search.options ? { ...config.search.options } : undefined
|
|
23
|
+
options: config.search.options ? { ...config.search.options } : undefined,
|
|
24
|
+
fanout: config.search.fanout
|
|
25
|
+
? { mode: config.search.fanout.mode, providers: config.search.fanout.providers ? [...config.search.fanout.providers] : undefined }
|
|
26
|
+
: undefined
|
|
24
27
|
},
|
|
25
28
|
fetch: {
|
|
26
29
|
...config.fetch,
|
|
@@ -54,12 +57,16 @@ async function defaultCheckTypebox() {
|
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
function formatSearchOptions(config) {
|
|
57
|
-
|
|
60
|
+
const parts = [
|
|
58
61
|
config.fallback ? `fallback ${config.fallback}` : undefined,
|
|
59
62
|
config.options?.categories?.length ? `categories ${config.options.categories.join(',')}` : undefined,
|
|
60
63
|
config.options?.language ? `language ${config.options.language}` : undefined,
|
|
61
|
-
config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined
|
|
62
|
-
|
|
64
|
+
config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined,
|
|
65
|
+
config.fanout && config.fanout.mode !== 'off'
|
|
66
|
+
? `fanout ${config.fanout.mode} (${(config.fanout.providers || usableSearchProviders(config)).join(', ')})`
|
|
67
|
+
: undefined
|
|
68
|
+
].filter(Boolean);
|
|
69
|
+
return parts.join(' ');
|
|
63
70
|
}
|
|
64
71
|
function formatFetchOptions(config) {
|
|
65
72
|
return [
|
|
@@ -155,6 +162,9 @@ export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChang
|
|
|
155
162
|
};
|
|
156
163
|
}
|
|
157
164
|
function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange) {
|
|
165
|
+
const usable = usableSearchProviders(backends.search);
|
|
166
|
+
const fanoutMode = backends.search.fanout?.mode ?? 'off';
|
|
167
|
+
const fanoutProviders = backends.search.fanout?.providers ?? usable;
|
|
158
168
|
return [
|
|
159
169
|
{
|
|
160
170
|
id: 'scope',
|
|
@@ -180,6 +190,18 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
|
|
|
180
190
|
currentValue: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? backends.search.fallback ?? 'off' : 'off',
|
|
181
191
|
values: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? ['off', 'duckduckgo'] : ['off']
|
|
182
192
|
},
|
|
193
|
+
{
|
|
194
|
+
id: 'backend:search:fanout:mode',
|
|
195
|
+
label: 'Search fanout',
|
|
196
|
+
currentValue: fanoutMode,
|
|
197
|
+
values: ['off', 'on', 'auto']
|
|
198
|
+
},
|
|
199
|
+
...(fanoutMode !== 'off' ? usable.map((provider) => ({
|
|
200
|
+
id: `backend:search:fanout:provider:${provider}`,
|
|
201
|
+
label: ` ${provider}`,
|
|
202
|
+
currentValue: fanoutProviders.includes(provider) ? 'included' : 'excluded',
|
|
203
|
+
values: ['included', 'excluded']
|
|
204
|
+
})) : []),
|
|
183
205
|
{
|
|
184
206
|
id: 'backend:secret:brave',
|
|
185
207
|
label: 'Brave API key',
|
|
@@ -333,6 +355,40 @@ export function applySettingsValue(state, id, newValue) {
|
|
|
333
355
|
delete currentBackends.search.fallback;
|
|
334
356
|
}
|
|
335
357
|
}
|
|
358
|
+
if (id === 'backend:search:fanout:mode') {
|
|
359
|
+
if (newValue === 'off') {
|
|
360
|
+
currentBackends.search.fanout = { mode: 'off' };
|
|
361
|
+
}
|
|
362
|
+
else if (newValue === 'on' || newValue === 'auto') {
|
|
363
|
+
currentBackends.search.fanout = {
|
|
364
|
+
mode: newValue,
|
|
365
|
+
providers: undefined
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
if (id.startsWith('backend:search:fanout:provider:')) {
|
|
370
|
+
const providerName = id.slice('backend:search:fanout:provider:'.length);
|
|
371
|
+
const usableProviders = usableSearchProviders(currentBackends.search);
|
|
372
|
+
// Ensure fanout exists
|
|
373
|
+
if (!currentBackends.search.fanout) {
|
|
374
|
+
currentBackends.search.fanout = { mode: 'on', providers: undefined };
|
|
375
|
+
}
|
|
376
|
+
// Materialize the provider list if it's currently undefined; use usable providers instead of all
|
|
377
|
+
const currentProviders = currentBackends.search.fanout.providers ?? usableProviders;
|
|
378
|
+
if (newValue === 'excluded') {
|
|
379
|
+
// Remove the provider from the list
|
|
380
|
+
const filtered = currentProviders.filter((p) => p !== providerName);
|
|
381
|
+
// never allow an empty set: keep at least one provider (empty would be read as "all")
|
|
382
|
+
if (filtered.length > 0) {
|
|
383
|
+
currentBackends.search.fanout.providers = filtered;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
else if (newValue === 'included') {
|
|
387
|
+
// Add the provider back, maintaining canonical order using usable providers
|
|
388
|
+
const result = usableProviders.filter((p) => currentProviders.includes(p) || p === providerName);
|
|
389
|
+
currentBackends.search.fanout.providers = result;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
336
392
|
if (id === 'backend:search:baseUrl') {
|
|
337
393
|
if (newValue.trim()) {
|
|
338
394
|
currentBackends.search.provider = 'searxng';
|
|
@@ -400,7 +456,8 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
|
|
|
400
456
|
: {
|
|
401
457
|
...(config.search.baseUrl !== inheritedConfig.search.baseUrl ? { baseUrl: config.search.baseUrl } : {}),
|
|
402
458
|
...(config.search.fallback !== inheritedConfig.search.fallback ? { fallback: config.search.fallback } : {}),
|
|
403
|
-
...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {})
|
|
459
|
+
...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {}),
|
|
460
|
+
...(!sameJson(config.search.fanout, inheritedConfig.search.fanout) ? { fanout: config.search.fanout } : {})
|
|
404
461
|
};
|
|
405
462
|
if (config.search.provider !== inheritedConfig.search.provider) {
|
|
406
463
|
override.search.provider = config.search.provider;
|
package/dist/extension.js
CHANGED
|
@@ -7,6 +7,30 @@ import { loadPresentationConfigLayers } from './presentation/config-store.js';
|
|
|
7
7
|
import { selectPresentationView } from './presentation/select-view.js';
|
|
8
8
|
import { createWebExploreTool } from './tools/web-explore.js';
|
|
9
9
|
import { getUpdateChangelogNotice } from './changelog-notice.js';
|
|
10
|
+
import { Text } from '@earendil-works/pi-tui';
|
|
11
|
+
/**
|
|
12
|
+
* What the model receives as the tool result: the actual synthesized findings (which for a
|
|
13
|
+
* direct GitHub/PDF/YouTube read is the full extracted content), plus source citations and any
|
|
14
|
+
* caveat. This is separate from the terminal display (see renderResult), so the model always
|
|
15
|
+
* gets the substance regardless of the user's compact/preview/verbose presentation setting.
|
|
16
|
+
*/
|
|
17
|
+
function serializeForModel(result) {
|
|
18
|
+
if (result.status === 'error') {
|
|
19
|
+
return `Research failed: ${result.error?.message ?? 'Unknown research failure.'}`;
|
|
20
|
+
}
|
|
21
|
+
if (result.findings.length === 0 && result.sources.length === 0) {
|
|
22
|
+
return 'No usable evidence found.';
|
|
23
|
+
}
|
|
24
|
+
return [
|
|
25
|
+
result.findings.join('\n\n'),
|
|
26
|
+
result.sources.length
|
|
27
|
+
? `Sources:\n${result.sources.map((source) => `- ${source.title}: ${source.url}`).join('\n')}`
|
|
28
|
+
: undefined,
|
|
29
|
+
result.caveat
|
|
30
|
+
]
|
|
31
|
+
.filter(Boolean)
|
|
32
|
+
.join('\n\n');
|
|
33
|
+
}
|
|
10
34
|
async function loadWebAgentConfig(pi) {
|
|
11
35
|
const store = pi.__presentationConfigStore;
|
|
12
36
|
return store?.load?.() ?? loadPresentationConfigLayers();
|
|
@@ -29,11 +53,6 @@ async function getEffectiveBackendConfig(pi) {
|
|
|
29
53
|
return DEFAULT_BACKEND_CONFIG;
|
|
30
54
|
}
|
|
31
55
|
}
|
|
32
|
-
async function renderToolText(pi, toolName, details) {
|
|
33
|
-
const config = await getEffectivePresentationConfig(pi);
|
|
34
|
-
const mode = resolvePresentationMode(toolName, config);
|
|
35
|
-
return selectPresentationView(details.presentation, mode) ?? JSON.stringify(details, null, 2);
|
|
36
|
-
}
|
|
37
56
|
export default function extension(pi) {
|
|
38
57
|
registerWebAgentConfigCommands(pi);
|
|
39
58
|
const injectedWebExplore = pi.__webExploreTool;
|
|
@@ -79,16 +98,38 @@ export default function extension(pi) {
|
|
|
79
98
|
async execute(_toolCallId, params) {
|
|
80
99
|
const webExplore = await getConfiguredWebExplore();
|
|
81
100
|
const result = await webExplore({ query: params.query });
|
|
101
|
+
// Terminal display honors the user's presentation mode; the model gets the full findings.
|
|
102
|
+
// The fallback must stay terse: never fall back to serializeForModel here, or a missing
|
|
103
|
+
// presentation would dump the full findings into the terminal.
|
|
104
|
+
const mode = resolvePresentationMode('web_explore', await getEffectivePresentationConfig(pi));
|
|
105
|
+
const terseFallback = 'web_explore result';
|
|
106
|
+
const terminalText = selectPresentationView(result.presentation, mode) ?? terseFallback;
|
|
107
|
+
const terminalTextExpanded = selectPresentationView(result.presentation, 'verbose') ?? terminalText;
|
|
82
108
|
return {
|
|
83
|
-
content: [
|
|
84
|
-
|
|
85
|
-
type: 'text',
|
|
86
|
-
text: await renderToolText(pi, 'web_explore', result)
|
|
87
|
-
}
|
|
88
|
-
],
|
|
89
|
-
details: result,
|
|
109
|
+
content: [{ type: 'text', text: serializeForModel(result) }],
|
|
110
|
+
details: { ...result, terminalText, terminalTextExpanded },
|
|
90
111
|
isError: result.status === 'error'
|
|
91
112
|
};
|
|
113
|
+
},
|
|
114
|
+
renderResult(toolResult, options) {
|
|
115
|
+
const details = toolResult.details;
|
|
116
|
+
try {
|
|
117
|
+
// Legacy fallback: results persisted before this change carry `presentation` but no
|
|
118
|
+
// `terminalText`, so old sessions stay readable instead of rendering blank.
|
|
119
|
+
const legacy = options.expanded
|
|
120
|
+
? details?.presentation?.views?.verbose ?? details?.presentation?.views?.compact
|
|
121
|
+
: details?.presentation?.views?.compact;
|
|
122
|
+
const text = (options.expanded ? details?.terminalTextExpanded : details?.terminalText) ??
|
|
123
|
+
details?.terminalText ??
|
|
124
|
+
legacy ??
|
|
125
|
+
'web_explore result';
|
|
126
|
+
return new Text(text, 0, 0);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Never throw: a thrown renderer makes Pi fall back to raw content, which would dump
|
|
130
|
+
// the full model-facing findings into the terminal.
|
|
131
|
+
return new Text('web_explore result', 0, 0);
|
|
132
|
+
}
|
|
92
133
|
}
|
|
93
134
|
});
|
|
94
135
|
}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
'utm_source',
|
|
3
|
-
'utm_medium',
|
|
4
|
-
'utm_campaign',
|
|
5
|
-
'utm_term',
|
|
6
|
-
'utm_content',
|
|
7
|
-
'utm_name',
|
|
8
|
-
'fbclid',
|
|
9
|
-
'gclid'
|
|
10
|
-
]);
|
|
1
|
+
import { canonicalizeUrl } from './url.js';
|
|
11
2
|
function stripTrailingPunctuation(raw) {
|
|
12
3
|
let next = raw.trim();
|
|
13
4
|
while (/[),.;!?\]]$/.test(next)) {
|
|
@@ -19,21 +10,7 @@ function stripTrailingPunctuation(raw) {
|
|
|
19
10
|
return next;
|
|
20
11
|
}
|
|
21
12
|
function normalizeDirectUrl(raw) {
|
|
22
|
-
|
|
23
|
-
const url = new URL(stripTrailingPunctuation(raw));
|
|
24
|
-
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
25
|
-
return undefined;
|
|
26
|
-
for (const key of [...url.searchParams.keys()]) {
|
|
27
|
-
if (TRACKING_PARAMS.has(key.toLowerCase())) {
|
|
28
|
-
url.searchParams.delete(key);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
url.hash = '';
|
|
32
|
-
return url.toString().replace(/\/$/, '');
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
return undefined;
|
|
36
|
-
}
|
|
13
|
+
return canonicalizeUrl(stripTrailingPunctuation(raw), { canonicalizeHost: false });
|
|
37
14
|
}
|
|
38
15
|
export function extractDirectUrls(query) {
|
|
39
16
|
const matches = query.match(/https?:\/\/\S+/gi) ?? [];
|
|
@@ -25,10 +25,11 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
|
|
|
25
25
|
const community = evidence.filter((item) => item.sourceKind === 'community').length;
|
|
26
26
|
const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
|
|
27
27
|
const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
|
|
28
|
+
const primaryContent = evidence.filter((item) => item.sourceKind === 'primary-content').length;
|
|
28
29
|
const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
|
|
29
30
|
const hasOfficialEvidence = official > 0;
|
|
30
|
-
const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
|
|
31
|
-
const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
|
|
31
|
+
const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0 && primaryContent === 0;
|
|
32
|
+
const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1 && primaryContent === 0;
|
|
32
33
|
const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
|
|
33
34
|
const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
|
|
34
35
|
const hasPossibleConflict = hasConflictMarkers(evidence);
|
|
@@ -47,6 +48,7 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
|
|
|
47
48
|
community,
|
|
48
49
|
thread,
|
|
49
50
|
packagePage,
|
|
51
|
+
primaryContent,
|
|
50
52
|
distinctHosts
|
|
51
53
|
},
|
|
52
54
|
flags: {
|
|
@@ -24,6 +24,8 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
|
|
|
24
24
|
headlessAttempts: number;
|
|
25
25
|
exhaustedBudget: boolean;
|
|
26
26
|
caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
|
|
27
|
+
fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
|
|
28
|
+
fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
|
|
27
29
|
};
|
|
28
30
|
}>;
|
|
29
31
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
1
|
+
import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
2
2
|
import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
|
|
3
3
|
import { type EvidenceCaveatReason } from './evidence-quality.js';
|
|
4
4
|
export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
|
|
@@ -28,6 +28,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
|
|
|
28
28
|
headlessAttempts: number;
|
|
29
29
|
exhaustedBudget: boolean;
|
|
30
30
|
caveatReasons: EvidenceCaveatReason[];
|
|
31
|
+
fanoutProviders: SearchProviderName[] | undefined;
|
|
32
|
+
fanoutSkipped: SearchProviderName[] | undefined;
|
|
31
33
|
};
|
|
32
34
|
}>;
|
|
33
35
|
};
|
|
@@ -13,6 +13,9 @@ function classifyEvidenceUrl(url) {
|
|
|
13
13
|
function summarizeText(text, maxLength = 180) {
|
|
14
14
|
return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
|
|
15
15
|
}
|
|
16
|
+
function isReaderMethod(method) {
|
|
17
|
+
return method === 'github' || method === 'pdf' || method === 'youtube';
|
|
18
|
+
}
|
|
16
19
|
function isBotCheckContent({ title = '', text }) {
|
|
17
20
|
return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
|
|
18
21
|
}
|
|
@@ -21,6 +24,16 @@ function evidenceFromFetch(result) {
|
|
|
21
24
|
return null;
|
|
22
25
|
if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
|
|
23
26
|
return null;
|
|
27
|
+
if (isReaderMethod(result.metadata.method)) {
|
|
28
|
+
return {
|
|
29
|
+
title: result.content.title ?? result.url,
|
|
30
|
+
url: result.url,
|
|
31
|
+
sourceKind: 'primary-content',
|
|
32
|
+
method: result.metadata.method,
|
|
33
|
+
summary: result.content.text,
|
|
34
|
+
supports: [result.content.text]
|
|
35
|
+
};
|
|
36
|
+
}
|
|
24
37
|
return {
|
|
25
38
|
title: result.content.title ?? result.url,
|
|
26
39
|
url: result.url,
|
|
@@ -66,13 +79,15 @@ function shouldRetryDirectWithHeadless(result, evidence) {
|
|
|
66
79
|
return false;
|
|
67
80
|
return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
|
|
68
81
|
}
|
|
69
|
-
function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [] }) {
|
|
82
|
+
function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
|
|
70
83
|
return {
|
|
71
84
|
searchPasses: previousQueries.length,
|
|
72
85
|
fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
|
|
73
86
|
headlessAttempts,
|
|
74
87
|
exhaustedBudget,
|
|
75
|
-
caveatReasons
|
|
88
|
+
caveatReasons,
|
|
89
|
+
fanoutProviders,
|
|
90
|
+
fanoutSkipped
|
|
76
91
|
};
|
|
77
92
|
}
|
|
78
93
|
function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
|
|
@@ -99,6 +114,13 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
99
114
|
const suggestedHeadlessUrls = [];
|
|
100
115
|
let headlessAttempts = 0;
|
|
101
116
|
let lastPass;
|
|
117
|
+
const fanoutProvidersSeen = new Set();
|
|
118
|
+
const fanoutSkippedSeen = new Set();
|
|
119
|
+
function fanoutSnapshot() {
|
|
120
|
+
const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
|
|
121
|
+
const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
|
|
122
|
+
return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
|
|
123
|
+
}
|
|
102
124
|
if (fetchDirect) {
|
|
103
125
|
for (const url of extractDirectUrls(query).slice(0, 3)) {
|
|
104
126
|
const directResult = await fetchDirect({ url });
|
|
@@ -134,6 +156,35 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
134
156
|
}
|
|
135
157
|
}
|
|
136
158
|
}
|
|
159
|
+
if (allEvidence.some((item) => item.sourceKind === 'primary-content')) {
|
|
160
|
+
const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
|
|
161
|
+
const quality = analyzeEvidenceQuality({
|
|
162
|
+
evidence: ranked,
|
|
163
|
+
gaps: allGaps,
|
|
164
|
+
lowValueOutcomes: allLowValueOutcomes
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
|
|
168
|
+
evidence: ranked,
|
|
169
|
+
workerPass: combinedWorkerPass({
|
|
170
|
+
lastPass,
|
|
171
|
+
previousQueries,
|
|
172
|
+
allGaps,
|
|
173
|
+
allLowValueOutcomes,
|
|
174
|
+
exhaustedBudget: false
|
|
175
|
+
}),
|
|
176
|
+
metadata: buildMetadata({
|
|
177
|
+
previousQueries,
|
|
178
|
+
allEvidence,
|
|
179
|
+
allGaps,
|
|
180
|
+
allLowValueOutcomes,
|
|
181
|
+
headlessAttempts,
|
|
182
|
+
exhaustedBudget: false,
|
|
183
|
+
caveatReasons: quality.caveatReasons,
|
|
184
|
+
...fanoutSnapshot()
|
|
185
|
+
})
|
|
186
|
+
};
|
|
187
|
+
}
|
|
137
188
|
for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
|
|
138
189
|
const queries = planSearchQueries({
|
|
139
190
|
originalQuery: query,
|
|
@@ -149,6 +200,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
149
200
|
maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
|
|
150
201
|
});
|
|
151
202
|
lastPass = pass;
|
|
203
|
+
pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
|
|
204
|
+
pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
|
|
152
205
|
allEvidence.push(...pass.evidence);
|
|
153
206
|
allGaps.push(...pass.gaps);
|
|
154
207
|
allLowValueOutcomes.push(...pass.lowValueOutcomes);
|
|
@@ -213,7 +266,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
213
266
|
allLowValueOutcomes,
|
|
214
267
|
headlessAttempts,
|
|
215
268
|
exhaustedBudget,
|
|
216
|
-
caveatReasons: updatedQuality.caveatReasons
|
|
269
|
+
caveatReasons: updatedQuality.caveatReasons,
|
|
270
|
+
...fanoutSnapshot()
|
|
217
271
|
})
|
|
218
272
|
};
|
|
219
273
|
}
|
|
@@ -239,7 +293,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
239
293
|
allLowValueOutcomes,
|
|
240
294
|
headlessAttempts,
|
|
241
295
|
exhaustedBudget: false,
|
|
242
|
-
caveatReasons: quality.caveatReasons
|
|
296
|
+
caveatReasons: quality.caveatReasons,
|
|
297
|
+
...fanoutSnapshot()
|
|
243
298
|
})
|
|
244
299
|
};
|
|
245
300
|
}
|
|
@@ -262,7 +317,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
262
317
|
allLowValueOutcomes,
|
|
263
318
|
headlessAttempts,
|
|
264
319
|
exhaustedBudget,
|
|
265
|
-
caveatReasons: quality.caveatReasons
|
|
320
|
+
caveatReasons: quality.caveatReasons,
|
|
321
|
+
...fanoutSnapshot()
|
|
266
322
|
})
|
|
267
323
|
};
|
|
268
324
|
}
|
|
@@ -291,7 +347,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
|
|
|
291
347
|
allLowValueOutcomes,
|
|
292
348
|
headlessAttempts,
|
|
293
349
|
exhaustedBudget: true,
|
|
294
|
-
caveatReasons: quality.caveatReasons
|
|
350
|
+
caveatReasons: quality.caveatReasons,
|
|
351
|
+
...fanoutSnapshot()
|
|
295
352
|
})
|
|
296
353
|
};
|
|
297
354
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import type { SearchProviderName } from '../types.js';
|
|
2
|
+
export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
|
|
2
3
|
export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
|
|
3
4
|
export type ResearchEvidence = {
|
|
4
5
|
title: string;
|
|
@@ -24,12 +25,16 @@ export type ResearchWorkerResult = {
|
|
|
24
25
|
lowValueOutcomes: ResearchLowValueOutcome[];
|
|
25
26
|
suggestedHeadlessUrl?: string;
|
|
26
27
|
exhaustedBudget: boolean;
|
|
28
|
+
fanoutProviders?: SearchProviderName[];
|
|
29
|
+
fanoutSkipped?: SearchProviderName[];
|
|
27
30
|
};
|
|
28
31
|
export type ResearchRunMetadata = {
|
|
29
32
|
searchPasses: number;
|
|
30
33
|
fetchedPages: number;
|
|
31
34
|
headlessAttempts: number;
|
|
32
35
|
exhaustedBudget: boolean;
|
|
36
|
+
fanoutProviders?: SearchProviderName[];
|
|
37
|
+
fanoutSkipped?: SearchProviderName[];
|
|
33
38
|
};
|
|
34
39
|
export type ResearchOrchestratorDecision = {
|
|
35
40
|
action: 'answer';
|
|
@@ -6,6 +6,9 @@ function classifySource(url) {
|
|
|
6
6
|
function summarizeText(text, maxLength = 180) {
|
|
7
7
|
return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
|
|
8
8
|
}
|
|
9
|
+
function isReaderMethod(method) {
|
|
10
|
+
return method === 'github' || method === 'pdf' || method === 'youtube';
|
|
11
|
+
}
|
|
9
12
|
function isBotCheckContent({ title = '', text }) {
|
|
10
13
|
return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
|
|
11
14
|
}
|
|
@@ -15,6 +18,18 @@ function evidenceFromFetch(fetched, fallbackTitle) {
|
|
|
15
18
|
return null;
|
|
16
19
|
if (isBotCheckContent({ title: content.title, text: content.text }))
|
|
17
20
|
return null;
|
|
21
|
+
// A successful reader read with usable text is primary content, exempt from the
|
|
22
|
+
// package-page filter below.
|
|
23
|
+
if (isReaderMethod(fetched.metadata.method) && content.text.trim()) {
|
|
24
|
+
return {
|
|
25
|
+
title: content.title ?? fallbackTitle,
|
|
26
|
+
url: fetched.url,
|
|
27
|
+
sourceKind: 'primary-content',
|
|
28
|
+
method: fetched.metadata.method,
|
|
29
|
+
summary: content.text,
|
|
30
|
+
supports: [content.text]
|
|
31
|
+
};
|
|
32
|
+
}
|
|
18
33
|
const sourceKind = classifySource(fetched.url);
|
|
19
34
|
if (sourceKind === 'package-page') {
|
|
20
35
|
return null;
|
|
@@ -65,6 +80,8 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
65
80
|
};
|
|
66
81
|
}
|
|
67
82
|
const searchResult = await search({ query });
|
|
83
|
+
const fanoutProviders = searchResult.metadata.fanout?.providers;
|
|
84
|
+
const fanoutSkipped = searchResult.metadata.fanout?.skipped;
|
|
68
85
|
if (searchResult.status !== 'ok') {
|
|
69
86
|
return {
|
|
70
87
|
searchQueries,
|
|
@@ -77,7 +94,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
77
94
|
],
|
|
78
95
|
lowValueOutcomes,
|
|
79
96
|
suggestedHeadlessUrl,
|
|
80
|
-
exhaustedBudget: false
|
|
97
|
+
exhaustedBudget: false,
|
|
98
|
+
fanoutProviders,
|
|
99
|
+
fanoutSkipped
|
|
81
100
|
};
|
|
82
101
|
}
|
|
83
102
|
if (searchResult.results.length === 0) {
|
|
@@ -92,7 +111,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
92
111
|
}
|
|
93
112
|
],
|
|
94
113
|
suggestedHeadlessUrl,
|
|
95
|
-
exhaustedBudget: false
|
|
114
|
+
exhaustedBudget: false,
|
|
115
|
+
fanoutProviders,
|
|
116
|
+
fanoutSkipped
|
|
96
117
|
};
|
|
97
118
|
}
|
|
98
119
|
const candidates = selectCandidates({
|
|
@@ -133,7 +154,9 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
133
154
|
gaps,
|
|
134
155
|
lowValueOutcomes,
|
|
135
156
|
suggestedHeadlessUrl,
|
|
136
|
-
exhaustedBudget: false
|
|
157
|
+
exhaustedBudget: false,
|
|
158
|
+
fanoutProviders,
|
|
159
|
+
fanoutSkipped
|
|
137
160
|
};
|
|
138
161
|
}
|
|
139
162
|
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
declare const TRACKING_PARAMS: Set<string>;
|
|
2
|
+
/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
|
|
3
|
+
export declare function canonicalizeUrl(raw: string, options?: {
|
|
4
|
+
canonicalizeHost?: boolean;
|
|
5
|
+
}): string | undefined;
|
|
6
|
+
export { TRACKING_PARAMS };
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
const TRACKING_PARAMS = new Set([
|
|
2
|
+
'utm_source',
|
|
3
|
+
'utm_medium',
|
|
4
|
+
'utm_campaign',
|
|
5
|
+
'utm_term',
|
|
6
|
+
'utm_content',
|
|
7
|
+
'utm_name',
|
|
8
|
+
'fbclid',
|
|
9
|
+
'gclid'
|
|
10
|
+
]);
|
|
11
|
+
/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
|
|
12
|
+
export function canonicalizeUrl(raw, options = {}) {
|
|
13
|
+
const { canonicalizeHost = true } = options;
|
|
14
|
+
let url;
|
|
15
|
+
try {
|
|
16
|
+
url = new URL(raw);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
22
|
+
return undefined;
|
|
23
|
+
if (canonicalizeHost) {
|
|
24
|
+
url.hostname = url.hostname.replace(/^www\./, '');
|
|
25
|
+
}
|
|
26
|
+
for (const key of [...url.searchParams.keys()]) {
|
|
27
|
+
if (TRACKING_PARAMS.has(key.toLowerCase())) {
|
|
28
|
+
url.searchParams.delete(key);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
url.hash = '';
|
|
32
|
+
return url.toString().replace(/\/$/, '');
|
|
33
|
+
}
|
|
34
|
+
export { TRACKING_PARAMS };
|
|
@@ -22,8 +22,15 @@ export function buildExplorePresentation(result) {
|
|
|
22
22
|
}
|
|
23
23
|
};
|
|
24
24
|
}
|
|
25
|
+
const fanoutProviders = result.metadata?.fanoutProviders;
|
|
26
|
+
const fanoutSkipped = result.metadata?.fanoutSkipped;
|
|
27
|
+
const fanoutNote = fanoutProviders?.length
|
|
28
|
+
? ` (fanout: ${fanoutProviders.join(', ')}${fanoutSkipped?.length ? `; skipped: ${fanoutSkipped.join(', ')}` : ''})`
|
|
29
|
+
: fanoutSkipped?.length
|
|
30
|
+
? ` (fanout; skipped: ${fanoutSkipped.join(', ')})`
|
|
31
|
+
: '';
|
|
25
32
|
const internalSummary = result.metadata
|
|
26
|
-
? `Internal research: web_search ×${result.metadata.searchPasses}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
|
|
33
|
+
? `Internal research: web_search ×${result.metadata.searchPasses}${fanoutNote}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
|
|
27
34
|
: undefined;
|
|
28
35
|
const hasEvidence = result.findings.length > 0 || result.sources.length > 0;
|
|
29
36
|
const evidenceLines = hasEvidence
|
|
@@ -46,12 +53,13 @@ export function buildExplorePresentation(result) {
|
|
|
46
53
|
]
|
|
47
54
|
.filter((line) => line !== undefined)
|
|
48
55
|
.join('\n');
|
|
56
|
+
const compact = hasEvidence
|
|
57
|
+
? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
|
|
58
|
+
: 'No usable evidence found';
|
|
49
59
|
return {
|
|
50
60
|
mode: 'compact',
|
|
51
61
|
views: {
|
|
52
|
-
compact
|
|
53
|
-
? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
|
|
54
|
-
: 'No usable evidence found',
|
|
62
|
+
compact,
|
|
55
63
|
preview,
|
|
56
64
|
verbose
|
|
57
65
|
},
|
|
@@ -1,12 +1,24 @@
|
|
|
1
|
+
function fanoutNote(result) {
|
|
2
|
+
const f = result.metadata.fanout;
|
|
3
|
+
if (!f)
|
|
4
|
+
return '';
|
|
5
|
+
if (f.providers.length) {
|
|
6
|
+
return ` (fanout: ${f.providers.join(', ')}${f.skipped?.length ? `; skipped: ${f.skipped.join(', ')}` : ''})`;
|
|
7
|
+
}
|
|
8
|
+
if (f.skipped?.length) {
|
|
9
|
+
return ` (fanout; skipped: ${f.skipped.join(', ')})`;
|
|
10
|
+
}
|
|
11
|
+
return '';
|
|
12
|
+
}
|
|
1
13
|
function formatCompact(result) {
|
|
2
14
|
const fallbackPrefix = result.metadata.fallbackFrom
|
|
3
15
|
? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
|
|
4
16
|
: '';
|
|
5
17
|
if (result.status === 'error') {
|
|
6
|
-
return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
|
|
18
|
+
return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}${fanoutNote(result)}`;
|
|
7
19
|
}
|
|
8
20
|
const suffix = result.results.length === 1 ? 'result' : 'results';
|
|
9
|
-
return `${fallbackPrefix}Found ${result.results.length} ${suffix}`;
|
|
21
|
+
return `${fallbackPrefix}Found ${result.results.length} ${suffix}${fanoutNote(result)}`;
|
|
10
22
|
}
|
|
11
23
|
export function buildSearchPresentation(result) {
|
|
12
24
|
const preview = result.results
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { READER_TEXT_CAP } from './limits.js';
|
|
1
2
|
function parseGithub(url) {
|
|
2
3
|
try {
|
|
3
4
|
return new URL(url);
|
|
@@ -51,12 +52,12 @@ function fail(url, message) {
|
|
|
51
52
|
};
|
|
52
53
|
}
|
|
53
54
|
function okResponse(url, title, text) {
|
|
54
|
-
const capped = text.slice(0,
|
|
55
|
+
const capped = text.slice(0, READER_TEXT_CAP);
|
|
55
56
|
return {
|
|
56
57
|
status: 'ok',
|
|
57
58
|
url,
|
|
58
59
|
content: { title, text: capped },
|
|
59
|
-
metadata: { method: 'github', cacheHit: false, truncated: text.length >=
|
|
60
|
+
metadata: { method: 'github', cacheHit: false, truncated: text.length >= READER_TEXT_CAP }
|
|
60
61
|
};
|
|
61
62
|
}
|
|
62
63
|
export function createGithubReader({ fetchImpl = fetch, token = process.env.GITHUB_TOKEN } = {}) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { extractText, getDocumentProxy, getMeta } from 'unpdf';
|
|
2
|
+
import { READER_TEXT_CAP } from './limits.js';
|
|
2
3
|
async function defaultExtract(bytes) {
|
|
3
4
|
const pdf = await getDocumentProxy(bytes);
|
|
4
5
|
const [{ text }, meta] = await Promise.all([
|
|
@@ -58,8 +59,8 @@ export function createPdfReader({ fetchImpl = fetch, extractPdfText = defaultExt
|
|
|
58
59
|
return {
|
|
59
60
|
status: 'ok',
|
|
60
61
|
url,
|
|
61
|
-
content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0,
|
|
62
|
-
metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >=
|
|
62
|
+
content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, READER_TEXT_CAP) },
|
|
63
|
+
metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= READER_TEXT_CAP }
|
|
63
64
|
};
|
|
64
65
|
}
|
|
65
66
|
catch (err) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getSubtitles, getVideoDetails } from 'youtube-caption-extractor';
|
|
2
|
+
import { READER_TEXT_CAP } from './limits.js';
|
|
2
3
|
function extractVideoId(url) {
|
|
3
4
|
let parsed;
|
|
4
5
|
try {
|
|
@@ -53,8 +54,8 @@ export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetail
|
|
|
53
54
|
return {
|
|
54
55
|
status: 'ok',
|
|
55
56
|
url,
|
|
56
|
-
content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0,
|
|
57
|
-
metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >=
|
|
57
|
+
content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, READER_TEXT_CAP) },
|
|
58
|
+
metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= READER_TEXT_CAP }
|
|
58
59
|
};
|
|
59
60
|
}
|
|
60
61
|
catch (err) {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { FanoutMode, SearchProviderName, WebSearchResponse } from '../types.js';
|
|
2
|
+
export type FanoutProvider = {
|
|
3
|
+
name: SearchProviderName;
|
|
4
|
+
search: (input: {
|
|
5
|
+
query: string;
|
|
6
|
+
}) => Promise<WebSearchResponse>;
|
|
7
|
+
};
|
|
8
|
+
export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
|
|
9
|
+
providers: FanoutProvider[];
|
|
10
|
+
mode: Exclude<FanoutMode, 'off'>;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
}): ({ query }: {
|
|
13
|
+
query: string;
|
|
14
|
+
}) => Promise<WebSearchResponse>;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
2
|
+
import { canonicalizeUrl } from '../orchestration/url.js';
|
|
3
|
+
const FANOUT_MIN_RESULTS = 3;
|
|
4
|
+
function hostOf(url) {
|
|
5
|
+
try {
|
|
6
|
+
return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return undefined;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function primaryLooksWeak(results) {
|
|
13
|
+
if (results.length < FANOUT_MIN_RESULTS)
|
|
14
|
+
return true;
|
|
15
|
+
const hosts = new Set(results.map((r) => hostOf(r.url)).filter(Boolean));
|
|
16
|
+
return hosts.size <= 1;
|
|
17
|
+
}
|
|
18
|
+
function merge(lists) {
|
|
19
|
+
const byKey = new Map();
|
|
20
|
+
for (const { name, results } of lists) {
|
|
21
|
+
const seenThisProvider = new Set();
|
|
22
|
+
results.forEach((result, index) => {
|
|
23
|
+
const key = canonicalizeUrl(result.url) ?? result.url;
|
|
24
|
+
if (seenThisProvider.has(key))
|
|
25
|
+
return; // ignore duplicates within a single provider's own list
|
|
26
|
+
seenThisProvider.add(key);
|
|
27
|
+
const existing = byKey.get(key);
|
|
28
|
+
if (!existing) {
|
|
29
|
+
byKey.set(key, { result: { ...result }, providers: new Set([name]), bestRank: index });
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
existing.providers.add(name);
|
|
33
|
+
existing.bestRank = Math.min(existing.bestRank, index);
|
|
34
|
+
if ((result.title?.length ?? 0) > (existing.result.title?.length ?? 0))
|
|
35
|
+
existing.result.title = result.title;
|
|
36
|
+
if ((result.snippet?.length ?? 0) > (existing.result.snippet?.length ?? 0))
|
|
37
|
+
existing.result.snippet = result.snippet;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
return [...byKey.values()]
|
|
41
|
+
.sort((a, b) => b.providers.size - a.providers.size || a.bestRank - b.bestRank)
|
|
42
|
+
.map((entry) => entry.result);
|
|
43
|
+
}
|
|
44
|
+
function withPresentation(result) {
|
|
45
|
+
return { ...result, presentation: buildSearchPresentation(result) };
|
|
46
|
+
}
|
|
47
|
+
const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
|
|
48
|
+
/** Resolve to undefined if the provider doesn't answer in time, so one slow/unreachable
|
|
49
|
+
* provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout across passes. */
|
|
50
|
+
function withTimeout(promise, ms) {
|
|
51
|
+
return new Promise((resolve) => {
|
|
52
|
+
const timer = setTimeout(() => resolve(undefined), ms);
|
|
53
|
+
promise.then((value) => {
|
|
54
|
+
clearTimeout(timer);
|
|
55
|
+
resolve(value);
|
|
56
|
+
}, () => {
|
|
57
|
+
clearTimeout(timer);
|
|
58
|
+
resolve(undefined);
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
export function createFanoutSearch({ providers, mode, timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS }) {
|
|
63
|
+
return async function fanoutSearch({ query }) {
|
|
64
|
+
const [primary, ...rest] = providers;
|
|
65
|
+
async function runSet(set) {
|
|
66
|
+
const settled = await Promise.all(set.map((p) => withTimeout(p.search({ query }), timeoutMs)));
|
|
67
|
+
const contributing = [];
|
|
68
|
+
const skipped = [];
|
|
69
|
+
set.forEach((provider, i) => {
|
|
70
|
+
const value = settled[i];
|
|
71
|
+
if (value && value.status === 'ok' && value.results.length > 0) {
|
|
72
|
+
contributing.push({ name: provider.name, results: value.results });
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
skipped.push(provider.name);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
return { contributing, skipped };
|
|
79
|
+
}
|
|
80
|
+
function finalize(lists, skipped, resolvedMode) {
|
|
81
|
+
if (lists.length === 0) {
|
|
82
|
+
const fanout = { mode: resolvedMode, providers: [] };
|
|
83
|
+
if (skipped.length > 0)
|
|
84
|
+
fanout.skipped = skipped;
|
|
85
|
+
return withPresentation({
|
|
86
|
+
status: 'error',
|
|
87
|
+
results: [],
|
|
88
|
+
metadata: { backend: primary.name, cacheHit: false, fanout },
|
|
89
|
+
error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
const fanout = { mode: resolvedMode, providers: lists.map((l) => l.name) };
|
|
93
|
+
if (skipped.length > 0)
|
|
94
|
+
fanout.skipped = skipped;
|
|
95
|
+
return withPresentation({
|
|
96
|
+
status: 'ok',
|
|
97
|
+
results: merge(lists),
|
|
98
|
+
metadata: { backend: primary.name, cacheHit: false, fanout }
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
if (mode === 'auto') {
|
|
102
|
+
const primaryOutcome = await withTimeout(primary.search({ query }), timeoutMs);
|
|
103
|
+
const primaryResults = primaryOutcome?.status === 'ok' ? primaryOutcome.results : [];
|
|
104
|
+
if (primaryResults.length > 0 && !primaryLooksWeak(primaryResults)) {
|
|
105
|
+
return withPresentation(primaryOutcome); // strong primary: no fanout
|
|
106
|
+
}
|
|
107
|
+
const { contributing, skipped } = await runSet(rest);
|
|
108
|
+
const lists = [
|
|
109
|
+
...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
|
|
110
|
+
...contributing
|
|
111
|
+
];
|
|
112
|
+
const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
|
|
113
|
+
return finalize(lists, allSkipped, 'auto');
|
|
114
|
+
}
|
|
115
|
+
const { contributing, skipped } = await runSet(providers);
|
|
116
|
+
return finalize(contributing, skipped, 'on');
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -40,6 +40,8 @@ export declare function createWebExploreTool({ explore }?: {
|
|
|
40
40
|
headlessAttempts: number;
|
|
41
41
|
exhaustedBudget: boolean;
|
|
42
42
|
caveatReasons?: string[];
|
|
43
|
+
fanoutProviders?: import("../types.js").SearchProviderName[];
|
|
44
|
+
fanoutSkipped?: import("../types.js").SearchProviderName[];
|
|
43
45
|
};
|
|
44
46
|
error?: import("../types.js").ToolError;
|
|
45
47
|
}>;
|
package/dist/types.d.ts
CHANGED
|
@@ -6,6 +6,13 @@ export type SearchResult = {
|
|
|
6
6
|
url: string;
|
|
7
7
|
snippet: string;
|
|
8
8
|
};
|
|
9
|
+
export type SearchProviderName = 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
|
|
10
|
+
export type FanoutMode = 'off' | 'on' | 'auto';
|
|
11
|
+
export type FanoutMetadata = {
|
|
12
|
+
mode: Exclude<FanoutMode, 'off'>;
|
|
13
|
+
providers: SearchProviderName[];
|
|
14
|
+
skipped?: SearchProviderName[];
|
|
15
|
+
};
|
|
9
16
|
export type ToolError = {
|
|
10
17
|
code: string;
|
|
11
18
|
message: string;
|
|
@@ -15,6 +22,7 @@ export type SearchMetadata = {
|
|
|
15
22
|
cacheHit: boolean;
|
|
16
23
|
fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
|
|
17
24
|
fallbackReason?: string;
|
|
25
|
+
fanout?: FanoutMetadata;
|
|
18
26
|
};
|
|
19
27
|
export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
|
|
20
28
|
export type FetchMetadata = {
|
|
@@ -70,6 +78,8 @@ export type WebExploreResponse = {
|
|
|
70
78
|
headlessAttempts: number;
|
|
71
79
|
exhaustedBudget: boolean;
|
|
72
80
|
caveatReasons?: string[];
|
|
81
|
+
fanoutProviders?: SearchProviderName[];
|
|
82
|
+
fanoutSkipped?: SearchProviderName[];
|
|
73
83
|
};
|
|
74
84
|
presentation?: PresentationEnvelope;
|
|
75
85
|
error?: ToolError;
|
package/package.json
CHANGED