@demigodmode/pi-web-agent 1.7.2 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/backends/config.d.ts +8 -0
- package/dist/backends/config.js +45 -0
- package/dist/backends/doctor.js +33 -0
- package/dist/backends/factory.js +45 -2
- package/dist/commands/web-agent-config.js +63 -6
- package/dist/extension.js +53 -12
- package/dist/orchestration/direct-url.js +2 -25
- package/dist/orchestration/evidence-quality.d.ts +1 -0
- package/dist/orchestration/evidence-quality.js +4 -2
- package/dist/orchestration/evidence-ranker.js +2 -0
- package/dist/orchestration/index.d.ts +2 -0
- package/dist/orchestration/research-orchestrator.d.ts +3 -1
- package/dist/orchestration/research-orchestrator.js +63 -6
- package/dist/orchestration/research-types.d.ts +7 -2
- package/dist/orchestration/research-worker.js +26 -3
- package/dist/orchestration/url.d.ts +6 -0
- package/dist/orchestration/url.js +34 -0
- package/dist/presentation/explore-presentation.js +18 -4
- package/dist/presentation/search-presentation.js +14 -2
- package/dist/readers/github-reader.d.ts +7 -0
- package/dist/readers/github-reader.js +130 -0
- package/dist/readers/limits.d.ts +3 -0
- package/dist/readers/limits.js +3 -0
- package/dist/readers/pdf-reader.d.ts +11 -0
- package/dist/readers/pdf-reader.js +76 -0
- package/dist/readers/resolver.d.ts +12 -0
- package/dist/readers/resolver.js +16 -0
- package/dist/readers/types.d.ts +10 -0
- package/dist/readers/types.js +1 -0
- package/dist/readers/youtube-reader.d.ts +21 -0
- package/dist/readers/youtube-reader.js +71 -0
- package/dist/search/fanout.d.ts +14 -0
- package/dist/search/fanout.js +118 -0
- package/dist/tools/web-explore.d.ts +3 -1
- package/dist/types.d.ts +13 -2
- package/package.json +4 -2
package/CHANGELOG.md
CHANGED
|
@@ -18,6 +18,34 @@ The format is intentionally simple and release-oriented.
|
|
|
18
18
|
### Breaking
|
|
19
19
|
- None.
|
|
20
20
|
|
|
21
|
+
## [1.9.0] - 2026-08-15
|
|
22
|
+
### Added
|
|
23
|
+
- Search fanout. Ask a hard question and web_explore can now hit several of your configured search backends at once, dedupe the merged results, and rank the pages that more than one provider agreed on to the top. Off by default. Flip it to on or auto in Settings → Backends (auto only fans out when the first provider comes back thin, so easy queries stay cheap). Preview/verbose shows which providers ran and which got skipped. (#35)
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- None.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- web_explore can finally summarize a link you hand it. Paste a GitHub file/issue/PR, a PDF, or a YouTube URL and ask for a summary: it now reads the actual content (raw source, extracted PDF text, the video transcript) and gives the model the whole thing. 1.8.0 added these readers but then squeezed everything down to a ~180-char snippet before the model ever saw it, so summaries were basically guesswork. Now they're grounded in the real text (long content still capped around 24k characters). (#40, #41)
|
|
30
|
+
- Fanout won't stall on a dead provider anymore. Each one gets an 8s timeout, so an unreachable self-hosted SearXNG can't hang a whole research pass. (#35)
|
|
31
|
+
- Fanout only offers and queries the providers you've actually set up (keyless DuckDuckGo, SearXNG when you give it a URL, the hosted ones only when their key is present) instead of acting like all six are on. (#35)
|
|
32
|
+
|
|
33
|
+
### Breaking
|
|
34
|
+
- None.
|
|
35
|
+
|
|
36
|
+
## [1.8.0] - 2026-08-14
|
|
37
|
+
### Added
|
|
38
|
+
- Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript (long content capped around 24k characters). All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
|
|
39
|
+
|
|
40
|
+
### Changed
|
|
41
|
+
- None.
|
|
42
|
+
|
|
43
|
+
### Fixed
|
|
44
|
+
- None.
|
|
45
|
+
|
|
46
|
+
### Breaking
|
|
47
|
+
- None.
|
|
48
|
+
|
|
21
49
|
## [1.7.2] - 2026-08-12
|
|
22
50
|
### Added
|
|
23
51
|
- None.
|
|
@@ -1,8 +1,13 @@
|
|
|
1
|
+
import type { FanoutMode, SearchProviderName } from '../types.js';
|
|
1
2
|
export type SearxngOptions = {
|
|
2
3
|
categories?: string[];
|
|
3
4
|
language?: string;
|
|
4
5
|
safesearch?: 0 | 1 | 2;
|
|
5
6
|
};
|
|
7
|
+
export type FanoutConfig = {
|
|
8
|
+
mode: FanoutMode;
|
|
9
|
+
providers?: SearchProviderName[];
|
|
10
|
+
};
|
|
6
11
|
export type FirecrawlOptions = {
|
|
7
12
|
formats?: string[];
|
|
8
13
|
onlyMainContent?: boolean;
|
|
@@ -12,6 +17,7 @@ export type SearchBackendConfig = {
|
|
|
12
17
|
baseUrl?: string;
|
|
13
18
|
fallback?: 'duckduckgo';
|
|
14
19
|
options?: SearxngOptions;
|
|
20
|
+
fanout?: FanoutConfig;
|
|
15
21
|
};
|
|
16
22
|
export type FetchBackendConfig = {
|
|
17
23
|
provider: 'http' | 'firecrawl';
|
|
@@ -40,6 +46,7 @@ export type BackendConfigFile = {
|
|
|
40
46
|
baseUrl?: unknown;
|
|
41
47
|
fallback?: unknown;
|
|
42
48
|
options?: unknown;
|
|
49
|
+
fanout?: unknown;
|
|
43
50
|
};
|
|
44
51
|
fetch?: {
|
|
45
52
|
provider?: unknown;
|
|
@@ -54,6 +61,7 @@ export type BackendConfigFile = {
|
|
|
54
61
|
};
|
|
55
62
|
};
|
|
56
63
|
export declare const DEFAULT_BACKEND_CONFIG: BackendConfig;
|
|
64
|
+
export declare function usableSearchProviders(search: SearchBackendConfig, env?: NodeJS.ProcessEnv): SearchProviderName[];
|
|
57
65
|
export declare function extractBackendConfigOverride(file: BackendConfigFile | null | undefined): BackendConfigOverride;
|
|
58
66
|
export declare function validateBackendConfig(config: BackendConfig): string[];
|
|
59
67
|
export declare function mergeBackendConfigLayers(...layers: Array<BackendConfig | BackendConfigOverride | undefined>): BackendConfig;
|
package/dist/backends/config.js
CHANGED
|
@@ -35,6 +35,38 @@ function extractFirecrawlOptions(value) {
|
|
|
35
35
|
options.onlyMainContent = raw.onlyMainContent;
|
|
36
36
|
return Object.keys(options).length > 0 ? options : undefined;
|
|
37
37
|
}
|
|
38
|
+
const PROVIDER_NAMES = ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
|
|
39
|
+
export function usableSearchProviders(search, env = process.env) {
|
|
40
|
+
// Match the provider implementations, which treat a blank/whitespace key as unconfigured.
|
|
41
|
+
const usable = ['duckduckgo']; // keyless, always usable
|
|
42
|
+
if (search.baseUrl?.trim())
|
|
43
|
+
usable.push('searxng');
|
|
44
|
+
if (env.PI_WEB_AGENT_BRAVE_API_KEY?.trim())
|
|
45
|
+
usable.push('brave');
|
|
46
|
+
if (env.YDC_API_KEY?.trim())
|
|
47
|
+
usable.push('youcom');
|
|
48
|
+
if (env.EXA_API_KEY?.trim())
|
|
49
|
+
usable.push('exa');
|
|
50
|
+
if (env.TAVILY_API_KEY?.trim())
|
|
51
|
+
usable.push('tavily');
|
|
52
|
+
return usable;
|
|
53
|
+
}
|
|
54
|
+
function extractFanoutConfig(value) {
|
|
55
|
+
if (!value || typeof value !== 'object')
|
|
56
|
+
return undefined;
|
|
57
|
+
const raw = value;
|
|
58
|
+
if (raw.mode !== 'off' && raw.mode !== 'on' && raw.mode !== 'auto')
|
|
59
|
+
return undefined;
|
|
60
|
+
const config = { mode: raw.mode };
|
|
61
|
+
if (Array.isArray(raw.providers)) {
|
|
62
|
+
const providers = raw.providers.filter((p) => typeof p === 'string' && PROVIDER_NAMES.includes(p));
|
|
63
|
+
if (providers.length !== raw.providers.length)
|
|
64
|
+
return undefined; // fail loud on any invalid entry
|
|
65
|
+
if (providers.length > 0)
|
|
66
|
+
config.providers = providers;
|
|
67
|
+
}
|
|
68
|
+
return config;
|
|
69
|
+
}
|
|
38
70
|
export function extractBackendConfigOverride(file) {
|
|
39
71
|
const backends = file?.backends;
|
|
40
72
|
const override = {};
|
|
@@ -58,6 +90,10 @@ export function extractBackendConfigOverride(file) {
|
|
|
58
90
|
}
|
|
59
91
|
}
|
|
60
92
|
}
|
|
93
|
+
const fanout = extractFanoutConfig(backends?.search?.fanout);
|
|
94
|
+
if (fanout) {
|
|
95
|
+
override.search = { ...(override.search ?? {}), fanout };
|
|
96
|
+
}
|
|
61
97
|
if (backends?.fetch?.provider === 'http' || backends?.fetch?.provider === 'firecrawl') {
|
|
62
98
|
override.fetch = { provider: backends.fetch.provider };
|
|
63
99
|
if (typeof backends.fetch.baseUrl === 'string') {
|
|
@@ -106,6 +142,15 @@ export function validateBackendConfig(config) {
|
|
|
106
142
|
if (config.fetch.options?.formats && config.fetch.options.formats.length === 0) {
|
|
107
143
|
issues.push('fetch options.formats must contain at least one format when provided');
|
|
108
144
|
}
|
|
145
|
+
const fanout = config.search.fanout;
|
|
146
|
+
if (fanout) {
|
|
147
|
+
if (fanout.mode !== 'off' && fanout.mode !== 'on' && fanout.mode !== 'auto') {
|
|
148
|
+
issues.push('search fanout.mode must be off, on, or auto');
|
|
149
|
+
}
|
|
150
|
+
if (fanout.providers?.includes('searxng') && !config.search.baseUrl) {
|
|
151
|
+
issues.push('search fanout with searxng requires backends.search.baseUrl');
|
|
152
|
+
}
|
|
153
|
+
}
|
|
109
154
|
return issues;
|
|
110
155
|
}
|
|
111
156
|
function mergeSearchConfig(current, override) {
|
package/dist/backends/doctor.js
CHANGED
|
@@ -208,6 +208,39 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
|
|
|
208
208
|
if (config.search.fallback) {
|
|
209
209
|
lines.push(`search fallback: ${config.search.fallback}`);
|
|
210
210
|
}
|
|
211
|
+
if (config.search.fanout && config.search.fanout.mode !== 'off') {
|
|
212
|
+
const providers = config.search.fanout.providers || ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
|
|
213
|
+
lines.push(`search fanout: ${config.search.fanout.mode} (${providers.join(', ')})`);
|
|
214
|
+
for (const provider of providers) {
|
|
215
|
+
if (provider === 'searxng' && !config.search.baseUrl) {
|
|
216
|
+
lines.push('search fanout provider searxng warning (missing baseUrl)');
|
|
217
|
+
}
|
|
218
|
+
else if (provider === 'brave') {
|
|
219
|
+
const apiKey = process.env.PI_WEB_AGENT_BRAVE_API_KEY;
|
|
220
|
+
if (!apiKey?.trim()) {
|
|
221
|
+
lines.push('search fanout provider brave warning (missing PI_WEB_AGENT_BRAVE_API_KEY)');
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
else if (provider === 'youcom') {
|
|
225
|
+
const apiKey = process.env.YDC_API_KEY;
|
|
226
|
+
if (!apiKey?.trim()) {
|
|
227
|
+
lines.push('search fanout provider youcom warning (missing YDC_API_KEY)');
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
else if (provider === 'exa') {
|
|
231
|
+
const apiKey = process.env.EXA_API_KEY;
|
|
232
|
+
if (!apiKey?.trim()) {
|
|
233
|
+
lines.push('search fanout provider exa warning (missing EXA_API_KEY)');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
else if (provider === 'tavily') {
|
|
237
|
+
const apiKey = process.env.TAVILY_API_KEY;
|
|
238
|
+
if (!apiKey?.trim()) {
|
|
239
|
+
lines.push('search fanout provider tavily warning (missing TAVILY_API_KEY)');
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
211
244
|
if (config.fetch.provider === 'http') {
|
|
212
245
|
lines.push('fetch backend: http');
|
|
213
246
|
}
|
package/dist/backends/factory.js
CHANGED
|
@@ -4,12 +4,17 @@ import { createYouComSearchTool } from '../search/youcom.js';
|
|
|
4
4
|
import { createExaSearchTool } from '../search/exa.js';
|
|
5
5
|
import { createTavilySearchTool } from '../search/tavily.js';
|
|
6
6
|
import { createSearxngSearchTool } from '../search/searxng.js';
|
|
7
|
+
import { createFanoutSearch } from '../search/fanout.js';
|
|
7
8
|
import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
|
|
8
9
|
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
9
10
|
import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
|
|
10
11
|
import { createWebFetchTool } from '../tools/web-fetch.js';
|
|
11
12
|
import { createWebSearchTool } from '../tools/web-search.js';
|
|
12
|
-
import { DEFAULT_BACKEND_CONFIG } from './config.js';
|
|
13
|
+
import { DEFAULT_BACKEND_CONFIG, usableSearchProviders } from './config.js';
|
|
14
|
+
import { createSpecialContentResolver } from '../readers/resolver.js';
|
|
15
|
+
import { createGithubReader } from '../readers/github-reader.js';
|
|
16
|
+
import { createPdfReader } from '../readers/pdf-reader.js';
|
|
17
|
+
import { createYoutubeReader } from '../readers/youtube-reader.js';
|
|
13
18
|
function invalidSearxngSearch() {
|
|
14
19
|
return async function search() {
|
|
15
20
|
const result = {
|
|
@@ -82,6 +87,25 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
82
87
|
const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
|
|
83
88
|
const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
|
|
84
89
|
const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
|
|
90
|
+
function buildProviderSearch(name) {
|
|
91
|
+
switch (name) {
|
|
92
|
+
case 'searxng':
|
|
93
|
+
return config.search.baseUrl
|
|
94
|
+
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
95
|
+
: invalidSearxngSearch();
|
|
96
|
+
case 'brave':
|
|
97
|
+
return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY });
|
|
98
|
+
case 'youcom':
|
|
99
|
+
return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
|
|
100
|
+
case 'exa':
|
|
101
|
+
return createExaSearch({ apiKey: process.env.EXA_API_KEY });
|
|
102
|
+
case 'tavily':
|
|
103
|
+
return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
|
|
104
|
+
case 'duckduckgo':
|
|
105
|
+
default:
|
|
106
|
+
return createDuckDuckGoSearch();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
85
109
|
let search = config.search.provider === 'searxng'
|
|
86
110
|
? config.search.baseUrl
|
|
87
111
|
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
@@ -110,6 +134,21 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
110
134
|
if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
|
|
111
135
|
search = withSearchFallback(search, createDuckDuckGoSearch(), 'tavily');
|
|
112
136
|
}
|
|
137
|
+
const fanoutConfig = config.search.fanout;
|
|
138
|
+
if (fanoutConfig && fanoutConfig.mode !== 'off') {
|
|
139
|
+
const baseNames = fanoutConfig.providers && fanoutConfig.providers.length > 0
|
|
140
|
+
? fanoutConfig.providers
|
|
141
|
+
: usableSearchProviders(config.search);
|
|
142
|
+
// A configured DuckDuckGo fallback must still be honored under fanout: fold it into the set.
|
|
143
|
+
const providerNames = config.search.fallback === 'duckduckgo' && !baseNames.includes('duckduckgo')
|
|
144
|
+
? [...baseNames, 'duckduckgo']
|
|
145
|
+
: baseNames;
|
|
146
|
+
const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter((n, i, arr) => arr.indexOf(n) === i);
|
|
147
|
+
search = createFanoutSearch({
|
|
148
|
+
providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
|
|
149
|
+
mode: fanoutConfig.mode
|
|
150
|
+
});
|
|
151
|
+
}
|
|
113
152
|
const httpFetch = createHttpFetch();
|
|
114
153
|
let fetchPage = config.fetch.provider === 'firecrawl'
|
|
115
154
|
? config.fetch.baseUrl
|
|
@@ -125,9 +164,13 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
125
164
|
if (config.fetch.provider === 'firecrawl' && config.fetch.fallback === 'http') {
|
|
126
165
|
fetchPage = withFetchFallback(fetchPage, httpFetch);
|
|
127
166
|
}
|
|
167
|
+
const fetchPageWithReaders = createSpecialContentResolver({
|
|
168
|
+
readers: [createGithubReader(), createPdfReader(), createYoutubeReader()],
|
|
169
|
+
fallback: fetchPage
|
|
170
|
+
});
|
|
128
171
|
return {
|
|
129
172
|
search,
|
|
130
|
-
fetchPage,
|
|
173
|
+
fetchPage: fetchPageWithReaders,
|
|
131
174
|
headlessFetch: createHeadlessFetch()
|
|
132
175
|
};
|
|
133
176
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig } from '../backends/config.js';
|
|
1
|
+
import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
|
|
2
2
|
import { checkBackendHealth } from '../backends/doctor.js';
|
|
3
3
|
import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
|
|
4
4
|
import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
|
|
@@ -20,7 +20,10 @@ function cloneBackendConfig(config) {
|
|
|
20
20
|
return {
|
|
21
21
|
search: {
|
|
22
22
|
...config.search,
|
|
23
|
-
options: config.search.options ? { ...config.search.options } : undefined
|
|
23
|
+
options: config.search.options ? { ...config.search.options } : undefined,
|
|
24
|
+
fanout: config.search.fanout
|
|
25
|
+
? { mode: config.search.fanout.mode, providers: config.search.fanout.providers ? [...config.search.fanout.providers] : undefined }
|
|
26
|
+
: undefined
|
|
24
27
|
},
|
|
25
28
|
fetch: {
|
|
26
29
|
...config.fetch,
|
|
@@ -54,12 +57,16 @@ async function defaultCheckTypebox() {
|
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
function formatSearchOptions(config) {
|
|
57
|
-
|
|
60
|
+
const parts = [
|
|
58
61
|
config.fallback ? `fallback ${config.fallback}` : undefined,
|
|
59
62
|
config.options?.categories?.length ? `categories ${config.options.categories.join(',')}` : undefined,
|
|
60
63
|
config.options?.language ? `language ${config.options.language}` : undefined,
|
|
61
|
-
config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined
|
|
62
|
-
|
|
64
|
+
config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined,
|
|
65
|
+
config.fanout && config.fanout.mode !== 'off'
|
|
66
|
+
? `fanout ${config.fanout.mode} (${(config.fanout.providers || usableSearchProviders(config)).join(', ')})`
|
|
67
|
+
: undefined
|
|
68
|
+
].filter(Boolean);
|
|
69
|
+
return parts.join(' ');
|
|
63
70
|
}
|
|
64
71
|
function formatFetchOptions(config) {
|
|
65
72
|
return [
|
|
@@ -155,6 +162,9 @@ export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChang
|
|
|
155
162
|
};
|
|
156
163
|
}
|
|
157
164
|
function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange) {
|
|
165
|
+
const usable = usableSearchProviders(backends.search);
|
|
166
|
+
const fanoutMode = backends.search.fanout?.mode ?? 'off';
|
|
167
|
+
const fanoutProviders = backends.search.fanout?.providers ?? usable;
|
|
158
168
|
return [
|
|
159
169
|
{
|
|
160
170
|
id: 'scope',
|
|
@@ -180,6 +190,18 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
|
|
|
180
190
|
currentValue: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? backends.search.fallback ?? 'off' : 'off',
|
|
181
191
|
values: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? ['off', 'duckduckgo'] : ['off']
|
|
182
192
|
},
|
|
193
|
+
{
|
|
194
|
+
id: 'backend:search:fanout:mode',
|
|
195
|
+
label: 'Search fanout',
|
|
196
|
+
currentValue: fanoutMode,
|
|
197
|
+
values: ['off', 'on', 'auto']
|
|
198
|
+
},
|
|
199
|
+
...(fanoutMode !== 'off' ? usable.map((provider) => ({
|
|
200
|
+
id: `backend:search:fanout:provider:${provider}`,
|
|
201
|
+
label: ` ${provider}`,
|
|
202
|
+
currentValue: fanoutProviders.includes(provider) ? 'included' : 'excluded',
|
|
203
|
+
values: ['included', 'excluded']
|
|
204
|
+
})) : []),
|
|
183
205
|
{
|
|
184
206
|
id: 'backend:secret:brave',
|
|
185
207
|
label: 'Brave API key',
|
|
@@ -333,6 +355,40 @@ export function applySettingsValue(state, id, newValue) {
|
|
|
333
355
|
delete currentBackends.search.fallback;
|
|
334
356
|
}
|
|
335
357
|
}
|
|
358
|
+
if (id === 'backend:search:fanout:mode') {
|
|
359
|
+
if (newValue === 'off') {
|
|
360
|
+
currentBackends.search.fanout = { mode: 'off' };
|
|
361
|
+
}
|
|
362
|
+
else if (newValue === 'on' || newValue === 'auto') {
|
|
363
|
+
currentBackends.search.fanout = {
|
|
364
|
+
mode: newValue,
|
|
365
|
+
providers: undefined
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
if (id.startsWith('backend:search:fanout:provider:')) {
|
|
370
|
+
const providerName = id.slice('backend:search:fanout:provider:'.length);
|
|
371
|
+
const usableProviders = usableSearchProviders(currentBackends.search);
|
|
372
|
+
// Ensure fanout exists
|
|
373
|
+
if (!currentBackends.search.fanout) {
|
|
374
|
+
currentBackends.search.fanout = { mode: 'on', providers: undefined };
|
|
375
|
+
}
|
|
376
|
+
// Materialize the provider list if it's currently undefined; use usable providers instead of all
|
|
377
|
+
const currentProviders = currentBackends.search.fanout.providers ?? usableProviders;
|
|
378
|
+
if (newValue === 'excluded') {
|
|
379
|
+
// Remove the provider from the list
|
|
380
|
+
const filtered = currentProviders.filter((p) => p !== providerName);
|
|
381
|
+
// never allow an empty set: keep at least one provider (empty would be read as "all")
|
|
382
|
+
if (filtered.length > 0) {
|
|
383
|
+
currentBackends.search.fanout.providers = filtered;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
else if (newValue === 'included') {
|
|
387
|
+
// Add the provider back, maintaining canonical order using usable providers
|
|
388
|
+
const result = usableProviders.filter((p) => currentProviders.includes(p) || p === providerName);
|
|
389
|
+
currentBackends.search.fanout.providers = result;
|
|
390
|
+
}
|
|
391
|
+
}
|
|
336
392
|
if (id === 'backend:search:baseUrl') {
|
|
337
393
|
if (newValue.trim()) {
|
|
338
394
|
currentBackends.search.provider = 'searxng';
|
|
@@ -400,7 +456,8 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
|
|
|
400
456
|
: {
|
|
401
457
|
...(config.search.baseUrl !== inheritedConfig.search.baseUrl ? { baseUrl: config.search.baseUrl } : {}),
|
|
402
458
|
...(config.search.fallback !== inheritedConfig.search.fallback ? { fallback: config.search.fallback } : {}),
|
|
403
|
-
...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {})
|
|
459
|
+
...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {}),
|
|
460
|
+
...(!sameJson(config.search.fanout, inheritedConfig.search.fanout) ? { fanout: config.search.fanout } : {})
|
|
404
461
|
};
|
|
405
462
|
if (config.search.provider !== inheritedConfig.search.provider) {
|
|
406
463
|
override.search.provider = config.search.provider;
|
package/dist/extension.js
CHANGED
|
@@ -7,6 +7,30 @@ import { loadPresentationConfigLayers } from './presentation/config-store.js';
|
|
|
7
7
|
import { selectPresentationView } from './presentation/select-view.js';
|
|
8
8
|
import { createWebExploreTool } from './tools/web-explore.js';
|
|
9
9
|
import { getUpdateChangelogNotice } from './changelog-notice.js';
|
|
10
|
+
import { Text } from '@earendil-works/pi-tui';
|
|
11
|
+
/**
|
|
12
|
+
* What the model receives as the tool result: the actual synthesized findings (which for a
|
|
13
|
+
* direct GitHub/PDF/YouTube read is the full extracted content), plus source citations and any
|
|
14
|
+
* caveat. This is separate from the terminal display (see renderResult), so the model always
|
|
15
|
+
* gets the substance regardless of the user's compact/preview/verbose presentation setting.
|
|
16
|
+
*/
|
|
17
|
+
function serializeForModel(result) {
|
|
18
|
+
if (result.status === 'error') {
|
|
19
|
+
return `Research failed: ${result.error?.message ?? 'Unknown research failure.'}`;
|
|
20
|
+
}
|
|
21
|
+
if (result.findings.length === 0 && result.sources.length === 0) {
|
|
22
|
+
return 'No usable evidence found.';
|
|
23
|
+
}
|
|
24
|
+
return [
|
|
25
|
+
result.findings.join('\n\n'),
|
|
26
|
+
result.sources.length
|
|
27
|
+
? `Sources:\n${result.sources.map((source) => `- ${source.title}: ${source.url}`).join('\n')}`
|
|
28
|
+
: undefined,
|
|
29
|
+
result.caveat
|
|
30
|
+
]
|
|
31
|
+
.filter(Boolean)
|
|
32
|
+
.join('\n\n');
|
|
33
|
+
}
|
|
10
34
|
async function loadWebAgentConfig(pi) {
|
|
11
35
|
const store = pi.__presentationConfigStore;
|
|
12
36
|
return store?.load?.() ?? loadPresentationConfigLayers();
|
|
@@ -29,11 +53,6 @@ async function getEffectiveBackendConfig(pi) {
|
|
|
29
53
|
return DEFAULT_BACKEND_CONFIG;
|
|
30
54
|
}
|
|
31
55
|
}
|
|
32
|
-
async function renderToolText(pi, toolName, details) {
|
|
33
|
-
const config = await getEffectivePresentationConfig(pi);
|
|
34
|
-
const mode = resolvePresentationMode(toolName, config);
|
|
35
|
-
return selectPresentationView(details.presentation, mode) ?? JSON.stringify(details, null, 2);
|
|
36
|
-
}
|
|
37
56
|
export default function extension(pi) {
|
|
38
57
|
registerWebAgentConfigCommands(pi);
|
|
39
58
|
const injectedWebExplore = pi.__webExploreTool;
|
|
@@ -79,16 +98,38 @@ export default function extension(pi) {
|
|
|
79
98
|
async execute(_toolCallId, params) {
|
|
80
99
|
const webExplore = await getConfiguredWebExplore();
|
|
81
100
|
const result = await webExplore({ query: params.query });
|
|
101
|
+
// Terminal display honors the user's presentation mode; the model gets the full findings.
|
|
102
|
+
// The fallback must stay terse: never fall back to serializeForModel here, or a missing
|
|
103
|
+
// presentation would dump the full findings into the terminal.
|
|
104
|
+
const mode = resolvePresentationMode('web_explore', await getEffectivePresentationConfig(pi));
|
|
105
|
+
const terseFallback = 'web_explore result';
|
|
106
|
+
const terminalText = selectPresentationView(result.presentation, mode) ?? terseFallback;
|
|
107
|
+
const terminalTextExpanded = selectPresentationView(result.presentation, 'verbose') ?? terminalText;
|
|
82
108
|
return {
|
|
83
|
-
content: [
|
|
84
|
-
|
|
85
|
-
type: 'text',
|
|
86
|
-
text: await renderToolText(pi, 'web_explore', result)
|
|
87
|
-
}
|
|
88
|
-
],
|
|
89
|
-
details: result,
|
|
109
|
+
content: [{ type: 'text', text: serializeForModel(result) }],
|
|
110
|
+
details: { ...result, terminalText, terminalTextExpanded },
|
|
90
111
|
isError: result.status === 'error'
|
|
91
112
|
};
|
|
113
|
+
},
|
|
114
|
+
renderResult(toolResult, options) {
|
|
115
|
+
const details = toolResult.details;
|
|
116
|
+
try {
|
|
117
|
+
// Legacy fallback: results persisted before this change carry `presentation` but no
|
|
118
|
+
// `terminalText`, so old sessions stay readable instead of rendering blank.
|
|
119
|
+
const legacy = options.expanded
|
|
120
|
+
? details?.presentation?.views?.verbose ?? details?.presentation?.views?.compact
|
|
121
|
+
: details?.presentation?.views?.compact;
|
|
122
|
+
const text = (options.expanded ? details?.terminalTextExpanded : details?.terminalText) ??
|
|
123
|
+
details?.terminalText ??
|
|
124
|
+
legacy ??
|
|
125
|
+
'web_explore result';
|
|
126
|
+
return new Text(text, 0, 0);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Never throw: a thrown renderer makes Pi fall back to raw content, which would dump
|
|
130
|
+
// the full model-facing findings into the terminal.
|
|
131
|
+
return new Text('web_explore result', 0, 0);
|
|
132
|
+
}
|
|
92
133
|
}
|
|
93
134
|
});
|
|
94
135
|
}
|
|
@@ -1,13 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
'utm_source',
|
|
3
|
-
'utm_medium',
|
|
4
|
-
'utm_campaign',
|
|
5
|
-
'utm_term',
|
|
6
|
-
'utm_content',
|
|
7
|
-
'utm_name',
|
|
8
|
-
'fbclid',
|
|
9
|
-
'gclid'
|
|
10
|
-
]);
|
|
1
|
+
import { canonicalizeUrl } from './url.js';
|
|
11
2
|
function stripTrailingPunctuation(raw) {
|
|
12
3
|
let next = raw.trim();
|
|
13
4
|
while (/[),.;!?\]]$/.test(next)) {
|
|
@@ -19,21 +10,7 @@ function stripTrailingPunctuation(raw) {
|
|
|
19
10
|
return next;
|
|
20
11
|
}
|
|
21
12
|
function normalizeDirectUrl(raw) {
|
|
22
|
-
|
|
23
|
-
const url = new URL(stripTrailingPunctuation(raw));
|
|
24
|
-
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
25
|
-
return undefined;
|
|
26
|
-
for (const key of [...url.searchParams.keys()]) {
|
|
27
|
-
if (TRACKING_PARAMS.has(key.toLowerCase())) {
|
|
28
|
-
url.searchParams.delete(key);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
url.hash = '';
|
|
32
|
-
return url.toString().replace(/\/$/, '');
|
|
33
|
-
}
|
|
34
|
-
catch {
|
|
35
|
-
return undefined;
|
|
36
|
-
}
|
|
13
|
+
return canonicalizeUrl(stripTrailingPunctuation(raw), { canonicalizeHost: false });
|
|
37
14
|
}
|
|
38
15
|
export function extractDirectUrls(query) {
|
|
39
16
|
const matches = query.match(/https?:\/\/\S+/gi) ?? [];
|
|
@@ -25,10 +25,11 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
|
|
|
25
25
|
const community = evidence.filter((item) => item.sourceKind === 'community').length;
|
|
26
26
|
const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
|
|
27
27
|
const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
|
|
28
|
+
const primaryContent = evidence.filter((item) => item.sourceKind === 'primary-content').length;
|
|
28
29
|
const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
|
|
29
30
|
const hasOfficialEvidence = official > 0;
|
|
30
|
-
const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
|
|
31
|
-
const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
|
|
31
|
+
const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0 && primaryContent === 0;
|
|
32
|
+
const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1 && primaryContent === 0;
|
|
32
33
|
const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
|
|
33
34
|
const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
|
|
34
35
|
const hasPossibleConflict = hasConflictMarkers(evidence);
|
|
@@ -47,6 +48,7 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
|
|
|
47
48
|
community,
|
|
48
49
|
thread,
|
|
49
50
|
packagePage,
|
|
51
|
+
primaryContent,
|
|
50
52
|
distinctHosts
|
|
51
53
|
},
|
|
52
54
|
flags: {
|
|
@@ -24,6 +24,8 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
|
|
|
24
24
|
headlessAttempts: number;
|
|
25
25
|
exhaustedBudget: boolean;
|
|
26
26
|
caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
|
|
27
|
+
fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
|
|
28
|
+
fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
|
|
27
29
|
};
|
|
28
30
|
}>;
|
|
29
31
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
1
|
+
import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
2
2
|
import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
|
|
3
3
|
import { type EvidenceCaveatReason } from './evidence-quality.js';
|
|
4
4
|
export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
|
|
@@ -28,6 +28,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
|
|
|
28
28
|
headlessAttempts: number;
|
|
29
29
|
exhaustedBudget: boolean;
|
|
30
30
|
caveatReasons: EvidenceCaveatReason[];
|
|
31
|
+
fanoutProviders: SearchProviderName[] | undefined;
|
|
32
|
+
fanoutSkipped: SearchProviderName[] | undefined;
|
|
31
33
|
};
|
|
32
34
|
}>;
|
|
33
35
|
};
|