@demigodmode/pi-web-agent 1.8.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -18,9 +18,24 @@ The format is intentionally simple and release-oriented.
18
18
  ### Breaking
19
19
  - None.
20
20
 
21
+ ## [1.9.0] - 2026-08-15
22
+ ### Added
23
+ - Search fanout. Ask a hard question and web_explore can now hit several of your configured search backends at once, dedupe the merged results, and rank the pages that more than one provider agreed on to the top. Off by default. Flip it to on or auto in Settings → Backends (auto only fans out when the first provider comes back thin, so easy queries stay cheap). Preview/verbose shows which providers ran and which got skipped. (#35)
24
+
25
+ ### Changed
26
+ - None.
27
+
28
+ ### Fixed
29
+ - web_explore can finally summarize a link you hand it. Paste a GitHub file/issue/PR, a PDF, or a YouTube URL and ask for a summary: it now reads the actual content (raw source, extracted PDF text, the video transcript) and gives the model the whole thing. 1.8.0 added these readers but then squeezed everything down to a ~180-char snippet before the model ever saw it, so summaries were basically guesswork. Now they're grounded in the real text (long content still capped around 24k characters). (#40, #41)
30
+ - Fanout won't stall on a dead provider anymore. Each one gets an 8s timeout, so an unreachable self-hosted SearXNG can't hang a whole research pass. (#35)
31
+ - Fanout only offers and queries the providers you've actually set up (keyless DuckDuckGo, SearXNG when you give it a URL, the hosted ones only when their key is present) instead of acting like all six are on. (#35)
32
+
33
+ ### Breaking
34
+ - None.
35
+
21
36
  ## [1.8.0] - 2026-08-14
22
37
  ### Added
23
- - Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript. All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
38
+ - Read the content behind GitHub, PDF, and YouTube links directly instead of the page shell. GitHub files, issues and PRs come from the API/raw endpoints (optional `GITHUB_TOKEN` raises the rate limit), PDFs are parsed with unpdf, and YouTube links return the transcript (long content capped around 24k characters). All keyless. Scanned PDFs and caption-less videos are caveated rather than failing. (#39, #40, #41)
24
39
 
25
40
  ### Changed
26
41
  - None.
@@ -1,8 +1,13 @@
1
+ import type { FanoutMode, SearchProviderName } from '../types.js';
1
2
  export type SearxngOptions = {
2
3
  categories?: string[];
3
4
  language?: string;
4
5
  safesearch?: 0 | 1 | 2;
5
6
  };
7
+ export type FanoutConfig = {
8
+ mode: FanoutMode;
9
+ providers?: SearchProviderName[];
10
+ };
6
11
  export type FirecrawlOptions = {
7
12
  formats?: string[];
8
13
  onlyMainContent?: boolean;
@@ -12,6 +17,7 @@ export type SearchBackendConfig = {
12
17
  baseUrl?: string;
13
18
  fallback?: 'duckduckgo';
14
19
  options?: SearxngOptions;
20
+ fanout?: FanoutConfig;
15
21
  };
16
22
  export type FetchBackendConfig = {
17
23
  provider: 'http' | 'firecrawl';
@@ -40,6 +46,7 @@ export type BackendConfigFile = {
40
46
  baseUrl?: unknown;
41
47
  fallback?: unknown;
42
48
  options?: unknown;
49
+ fanout?: unknown;
43
50
  };
44
51
  fetch?: {
45
52
  provider?: unknown;
@@ -54,6 +61,7 @@ export type BackendConfigFile = {
54
61
  };
55
62
  };
56
63
  export declare const DEFAULT_BACKEND_CONFIG: BackendConfig;
64
+ export declare function usableSearchProviders(search: SearchBackendConfig, env?: NodeJS.ProcessEnv): SearchProviderName[];
57
65
  export declare function extractBackendConfigOverride(file: BackendConfigFile | null | undefined): BackendConfigOverride;
58
66
  export declare function validateBackendConfig(config: BackendConfig): string[];
59
67
  export declare function mergeBackendConfigLayers(...layers: Array<BackendConfig | BackendConfigOverride | undefined>): BackendConfig;
@@ -35,6 +35,38 @@ function extractFirecrawlOptions(value) {
35
35
  options.onlyMainContent = raw.onlyMainContent;
36
36
  return Object.keys(options).length > 0 ? options : undefined;
37
37
  }
38
+ const PROVIDER_NAMES = ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
39
+ export function usableSearchProviders(search, env = process.env) {
40
+ // Match the provider implementations, which treat a blank/whitespace key as unconfigured.
41
+ const usable = ['duckduckgo']; // keyless, always usable
42
+ if (search.baseUrl?.trim())
43
+ usable.push('searxng');
44
+ if (env.PI_WEB_AGENT_BRAVE_API_KEY?.trim())
45
+ usable.push('brave');
46
+ if (env.YDC_API_KEY?.trim())
47
+ usable.push('youcom');
48
+ if (env.EXA_API_KEY?.trim())
49
+ usable.push('exa');
50
+ if (env.TAVILY_API_KEY?.trim())
51
+ usable.push('tavily');
52
+ return usable;
53
+ }
54
+ function extractFanoutConfig(value) {
55
+ if (!value || typeof value !== 'object')
56
+ return undefined;
57
+ const raw = value;
58
+ if (raw.mode !== 'off' && raw.mode !== 'on' && raw.mode !== 'auto')
59
+ return undefined;
60
+ const config = { mode: raw.mode };
61
+ if (Array.isArray(raw.providers)) {
62
+ const providers = raw.providers.filter((p) => typeof p === 'string' && PROVIDER_NAMES.includes(p));
63
+ if (providers.length !== raw.providers.length)
64
+ return undefined; // fail loud on any invalid entry
65
+ if (providers.length > 0)
66
+ config.providers = providers;
67
+ }
68
+ return config;
69
+ }
38
70
  export function extractBackendConfigOverride(file) {
39
71
  const backends = file?.backends;
40
72
  const override = {};
@@ -58,6 +90,10 @@ export function extractBackendConfigOverride(file) {
58
90
  }
59
91
  }
60
92
  }
93
+ const fanout = extractFanoutConfig(backends?.search?.fanout);
94
+ if (fanout) {
95
+ override.search = { ...(override.search ?? {}), fanout };
96
+ }
61
97
  if (backends?.fetch?.provider === 'http' || backends?.fetch?.provider === 'firecrawl') {
62
98
  override.fetch = { provider: backends.fetch.provider };
63
99
  if (typeof backends.fetch.baseUrl === 'string') {
@@ -106,6 +142,15 @@ export function validateBackendConfig(config) {
106
142
  if (config.fetch.options?.formats && config.fetch.options.formats.length === 0) {
107
143
  issues.push('fetch options.formats must contain at least one format when provided');
108
144
  }
145
+ const fanout = config.search.fanout;
146
+ if (fanout) {
147
+ if (fanout.mode !== 'off' && fanout.mode !== 'on' && fanout.mode !== 'auto') {
148
+ issues.push('search fanout.mode must be off, on, or auto');
149
+ }
150
+ if (fanout.providers?.includes('searxng') && !config.search.baseUrl) {
151
+ issues.push('search fanout with searxng requires backends.search.baseUrl');
152
+ }
153
+ }
109
154
  return issues;
110
155
  }
111
156
  function mergeSearchConfig(current, override) {
@@ -208,6 +208,39 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
208
208
  if (config.search.fallback) {
209
209
  lines.push(`search fallback: ${config.search.fallback}`);
210
210
  }
211
+ if (config.search.fanout && config.search.fanout.mode !== 'off') {
212
+ const providers = config.search.fanout.providers || ['duckduckgo', 'searxng', 'brave', 'youcom', 'exa', 'tavily'];
213
+ lines.push(`search fanout: ${config.search.fanout.mode} (${providers.join(', ')})`);
214
+ for (const provider of providers) {
215
+ if (provider === 'searxng' && !config.search.baseUrl) {
216
+ lines.push('search fanout provider searxng warning (missing baseUrl)');
217
+ }
218
+ else if (provider === 'brave') {
219
+ const apiKey = process.env.PI_WEB_AGENT_BRAVE_API_KEY;
220
+ if (!apiKey?.trim()) {
221
+ lines.push('search fanout provider brave warning (missing PI_WEB_AGENT_BRAVE_API_KEY)');
222
+ }
223
+ }
224
+ else if (provider === 'youcom') {
225
+ const apiKey = process.env.YDC_API_KEY;
226
+ if (!apiKey?.trim()) {
227
+ lines.push('search fanout provider youcom warning (missing YDC_API_KEY)');
228
+ }
229
+ }
230
+ else if (provider === 'exa') {
231
+ const apiKey = process.env.EXA_API_KEY;
232
+ if (!apiKey?.trim()) {
233
+ lines.push('search fanout provider exa warning (missing EXA_API_KEY)');
234
+ }
235
+ }
236
+ else if (provider === 'tavily') {
237
+ const apiKey = process.env.TAVILY_API_KEY;
238
+ if (!apiKey?.trim()) {
239
+ lines.push('search fanout provider tavily warning (missing TAVILY_API_KEY)');
240
+ }
241
+ }
242
+ }
243
+ }
211
244
  if (config.fetch.provider === 'http') {
212
245
  lines.push('fetch backend: http');
213
246
  }
@@ -4,12 +4,13 @@ import { createYouComSearchTool } from '../search/youcom.js';
4
4
  import { createExaSearchTool } from '../search/exa.js';
5
5
  import { createTavilySearchTool } from '../search/tavily.js';
6
6
  import { createSearxngSearchTool } from '../search/searxng.js';
7
+ import { createFanoutSearch } from '../search/fanout.js';
7
8
  import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
8
9
  import { buildSearchPresentation } from '../presentation/search-presentation.js';
9
10
  import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
10
11
  import { createWebFetchTool } from '../tools/web-fetch.js';
11
12
  import { createWebSearchTool } from '../tools/web-search.js';
12
- import { DEFAULT_BACKEND_CONFIG } from './config.js';
13
+ import { DEFAULT_BACKEND_CONFIG, usableSearchProviders } from './config.js';
13
14
  import { createSpecialContentResolver } from '../readers/resolver.js';
14
15
  import { createGithubReader } from '../readers/github-reader.js';
15
16
  import { createPdfReader } from '../readers/pdf-reader.js';
@@ -86,6 +87,25 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
86
87
  const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
87
88
  const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
88
89
  const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
90
+ function buildProviderSearch(name) {
91
+ switch (name) {
92
+ case 'searxng':
93
+ return config.search.baseUrl
94
+ ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
95
+ : invalidSearxngSearch();
96
+ case 'brave':
97
+ return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY });
98
+ case 'youcom':
99
+ return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
100
+ case 'exa':
101
+ return createExaSearch({ apiKey: process.env.EXA_API_KEY });
102
+ case 'tavily':
103
+ return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
104
+ case 'duckduckgo':
105
+ default:
106
+ return createDuckDuckGoSearch();
107
+ }
108
+ }
89
109
  let search = config.search.provider === 'searxng'
90
110
  ? config.search.baseUrl
91
111
  ? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
@@ -114,6 +134,21 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
114
134
  if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
115
135
  search = withSearchFallback(search, createDuckDuckGoSearch(), 'tavily');
116
136
  }
137
+ const fanoutConfig = config.search.fanout;
138
+ if (fanoutConfig && fanoutConfig.mode !== 'off') {
139
+ const baseNames = fanoutConfig.providers && fanoutConfig.providers.length > 0
140
+ ? fanoutConfig.providers
141
+ : usableSearchProviders(config.search);
142
+ // A configured DuckDuckGo fallback must still be honored under fanout: fold it into the set.
143
+ const providerNames = config.search.fallback === 'duckduckgo' && !baseNames.includes('duckduckgo')
144
+ ? [...baseNames, 'duckduckgo']
145
+ : baseNames;
146
+ const ordered = [config.search.provider, ...providerNames.filter((n) => n !== config.search.provider)].filter((n, i, arr) => arr.indexOf(n) === i);
147
+ search = createFanoutSearch({
148
+ providers: ordered.map((name) => ({ name, search: buildProviderSearch(name) })),
149
+ mode: fanoutConfig.mode
150
+ });
151
+ }
117
152
  const httpFetch = createHttpFetch();
118
153
  let fetchPage = config.fetch.provider === 'firecrawl'
119
154
  ? config.fetch.baseUrl
@@ -1,4 +1,4 @@
1
- import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig } from '../backends/config.js';
1
+ import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
2
2
  import { checkBackendHealth } from '../backends/doctor.js';
3
3
  import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
4
4
  import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
@@ -20,7 +20,10 @@ function cloneBackendConfig(config) {
20
20
  return {
21
21
  search: {
22
22
  ...config.search,
23
- options: config.search.options ? { ...config.search.options } : undefined
23
+ options: config.search.options ? { ...config.search.options } : undefined,
24
+ fanout: config.search.fanout
25
+ ? { mode: config.search.fanout.mode, providers: config.search.fanout.providers ? [...config.search.fanout.providers] : undefined }
26
+ : undefined
24
27
  },
25
28
  fetch: {
26
29
  ...config.fetch,
@@ -54,12 +57,16 @@ async function defaultCheckTypebox() {
54
57
  }
55
58
  }
56
59
  function formatSearchOptions(config) {
57
- return [
60
+ const parts = [
58
61
  config.fallback ? `fallback ${config.fallback}` : undefined,
59
62
  config.options?.categories?.length ? `categories ${config.options.categories.join(',')}` : undefined,
60
63
  config.options?.language ? `language ${config.options.language}` : undefined,
61
- config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined
62
- ].filter(Boolean).join(' ');
64
+ config.options?.safesearch !== undefined ? `safesearch ${config.options.safesearch}` : undefined,
65
+ config.fanout && config.fanout.mode !== 'off'
66
+ ? `fanout ${config.fanout.mode} (${(config.fanout.providers || usableSearchProviders(config)).join(', ')})`
67
+ : undefined
68
+ ].filter(Boolean);
69
+ return parts.join(' ');
63
70
  }
64
71
  function formatFetchOptions(config) {
65
72
  return [
@@ -155,6 +162,9 @@ export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChang
155
162
  };
156
163
  }
157
164
  function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange) {
165
+ const usable = usableSearchProviders(backends.search);
166
+ const fanoutMode = backends.search.fanout?.mode ?? 'off';
167
+ const fanoutProviders = backends.search.fanout?.providers ?? usable;
158
168
  return [
159
169
  {
160
170
  id: 'scope',
@@ -180,6 +190,18 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
180
190
  currentValue: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? backends.search.fallback ?? 'off' : 'off',
181
191
  values: backends.search.provider === 'searxng' || backends.search.provider === 'brave' || backends.search.provider === 'youcom' || backends.search.provider === 'exa' || backends.search.provider === 'tavily' ? ['off', 'duckduckgo'] : ['off']
182
192
  },
193
+ {
194
+ id: 'backend:search:fanout:mode',
195
+ label: 'Search fanout',
196
+ currentValue: fanoutMode,
197
+ values: ['off', 'on', 'auto']
198
+ },
199
+ ...(fanoutMode !== 'off' ? usable.map((provider) => ({
200
+ id: `backend:search:fanout:provider:${provider}`,
201
+ label: ` ${provider}`,
202
+ currentValue: fanoutProviders.includes(provider) ? 'included' : 'excluded',
203
+ values: ['included', 'excluded']
204
+ })) : []),
183
205
  {
184
206
  id: 'backend:secret:brave',
185
207
  label: 'Brave API key',
@@ -333,6 +355,40 @@ export function applySettingsValue(state, id, newValue) {
333
355
  delete currentBackends.search.fallback;
334
356
  }
335
357
  }
358
+ if (id === 'backend:search:fanout:mode') {
359
+ if (newValue === 'off') {
360
+ currentBackends.search.fanout = { mode: 'off' };
361
+ }
362
+ else if (newValue === 'on' || newValue === 'auto') {
363
+ currentBackends.search.fanout = {
364
+ mode: newValue,
365
+ providers: undefined
366
+ };
367
+ }
368
+ }
369
+ if (id.startsWith('backend:search:fanout:provider:')) {
370
+ const providerName = id.slice('backend:search:fanout:provider:'.length);
371
+ const usableProviders = usableSearchProviders(currentBackends.search);
372
+ // Ensure fanout exists
373
+ if (!currentBackends.search.fanout) {
374
+ currentBackends.search.fanout = { mode: 'on', providers: undefined };
375
+ }
376
+ // Materialize the provider list if it's currently undefined; use usable providers instead of all
377
+ const currentProviders = currentBackends.search.fanout.providers ?? usableProviders;
378
+ if (newValue === 'excluded') {
379
+ // Remove the provider from the list
380
+ const filtered = currentProviders.filter((p) => p !== providerName);
381
+ // never allow an empty set: keep at least one provider (empty would be read as "all")
382
+ if (filtered.length > 0) {
383
+ currentBackends.search.fanout.providers = filtered;
384
+ }
385
+ }
386
+ else if (newValue === 'included') {
387
+ // Add the provider back, maintaining canonical order using usable providers
388
+ const result = usableProviders.filter((p) => currentProviders.includes(p) || p === providerName);
389
+ currentBackends.search.fanout.providers = result;
390
+ }
391
+ }
336
392
  if (id === 'backend:search:baseUrl') {
337
393
  if (newValue.trim()) {
338
394
  currentBackends.search.provider = 'searxng';
@@ -400,7 +456,8 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
400
456
  : {
401
457
  ...(config.search.baseUrl !== inheritedConfig.search.baseUrl ? { baseUrl: config.search.baseUrl } : {}),
402
458
  ...(config.search.fallback !== inheritedConfig.search.fallback ? { fallback: config.search.fallback } : {}),
403
- ...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {})
459
+ ...(!sameJson(config.search.options, inheritedConfig.search.options) ? { options: config.search.options } : {}),
460
+ ...(!sameJson(config.search.fanout, inheritedConfig.search.fanout) ? { fanout: config.search.fanout } : {})
404
461
  };
405
462
  if (config.search.provider !== inheritedConfig.search.provider) {
406
463
  override.search.provider = config.search.provider;
package/dist/extension.js CHANGED
@@ -7,6 +7,30 @@ import { loadPresentationConfigLayers } from './presentation/config-store.js';
7
7
  import { selectPresentationView } from './presentation/select-view.js';
8
8
  import { createWebExploreTool } from './tools/web-explore.js';
9
9
  import { getUpdateChangelogNotice } from './changelog-notice.js';
10
+ import { Text } from '@earendil-works/pi-tui';
11
+ /**
12
+ * What the model receives as the tool result: the actual synthesized findings (which for a
13
+ * direct GitHub/PDF/YouTube read is the full extracted content), plus source citations and any
14
+ * caveat. This is separate from the terminal display (see renderResult), so the model always
15
+ * gets the substance regardless of the user's compact/preview/verbose presentation setting.
16
+ */
17
+ function serializeForModel(result) {
18
+ if (result.status === 'error') {
19
+ return `Research failed: ${result.error?.message ?? 'Unknown research failure.'}`;
20
+ }
21
+ if (result.findings.length === 0 && result.sources.length === 0) {
22
+ return 'No usable evidence found.';
23
+ }
24
+ return [
25
+ result.findings.join('\n\n'),
26
+ result.sources.length
27
+ ? `Sources:\n${result.sources.map((source) => `- ${source.title}: ${source.url}`).join('\n')}`
28
+ : undefined,
29
+ result.caveat
30
+ ]
31
+ .filter(Boolean)
32
+ .join('\n\n');
33
+ }
10
34
  async function loadWebAgentConfig(pi) {
11
35
  const store = pi.__presentationConfigStore;
12
36
  return store?.load?.() ?? loadPresentationConfigLayers();
@@ -29,11 +53,6 @@ async function getEffectiveBackendConfig(pi) {
29
53
  return DEFAULT_BACKEND_CONFIG;
30
54
  }
31
55
  }
32
- async function renderToolText(pi, toolName, details) {
33
- const config = await getEffectivePresentationConfig(pi);
34
- const mode = resolvePresentationMode(toolName, config);
35
- return selectPresentationView(details.presentation, mode) ?? JSON.stringify(details, null, 2);
36
- }
37
56
  export default function extension(pi) {
38
57
  registerWebAgentConfigCommands(pi);
39
58
  const injectedWebExplore = pi.__webExploreTool;
@@ -79,16 +98,38 @@ export default function extension(pi) {
79
98
  async execute(_toolCallId, params) {
80
99
  const webExplore = await getConfiguredWebExplore();
81
100
  const result = await webExplore({ query: params.query });
101
+ // Terminal display honors the user's presentation mode; the model gets the full findings.
102
+ // The fallback must stay terse: never fall back to serializeForModel here, or a missing
103
+ // presentation would dump the full findings into the terminal.
104
+ const mode = resolvePresentationMode('web_explore', await getEffectivePresentationConfig(pi));
105
+ const terseFallback = 'web_explore result';
106
+ const terminalText = selectPresentationView(result.presentation, mode) ?? terseFallback;
107
+ const terminalTextExpanded = selectPresentationView(result.presentation, 'verbose') ?? terminalText;
82
108
  return {
83
- content: [
84
- {
85
- type: 'text',
86
- text: await renderToolText(pi, 'web_explore', result)
87
- }
88
- ],
89
- details: result,
109
+ content: [{ type: 'text', text: serializeForModel(result) }],
110
+ details: { ...result, terminalText, terminalTextExpanded },
90
111
  isError: result.status === 'error'
91
112
  };
113
+ },
114
+ renderResult(toolResult, options) {
115
+ const details = toolResult.details;
116
+ try {
117
+ // Legacy fallback: results persisted before this change carry `presentation` but no
118
+ // `terminalText`, so old sessions stay readable instead of rendering blank.
119
+ const legacy = options.expanded
120
+ ? details?.presentation?.views?.verbose ?? details?.presentation?.views?.compact
121
+ : details?.presentation?.views?.compact;
122
+ const text = (options.expanded ? details?.terminalTextExpanded : details?.terminalText) ??
123
+ details?.terminalText ??
124
+ legacy ??
125
+ 'web_explore result';
126
+ return new Text(text, 0, 0);
127
+ }
128
+ catch {
129
+ // Never throw: a thrown renderer makes Pi fall back to raw content, which would dump
130
+ // the full model-facing findings into the terminal.
131
+ return new Text('web_explore result', 0, 0);
132
+ }
92
133
  }
93
134
  });
94
135
  }
@@ -1,13 +1,4 @@
1
- const TRACKING_PARAMS = new Set([
2
- 'utm_source',
3
- 'utm_medium',
4
- 'utm_campaign',
5
- 'utm_term',
6
- 'utm_content',
7
- 'utm_name',
8
- 'fbclid',
9
- 'gclid'
10
- ]);
1
+ import { canonicalizeUrl } from './url.js';
11
2
  function stripTrailingPunctuation(raw) {
12
3
  let next = raw.trim();
13
4
  while (/[),.;!?\]]$/.test(next)) {
@@ -19,21 +10,7 @@ function stripTrailingPunctuation(raw) {
19
10
  return next;
20
11
  }
21
12
  function normalizeDirectUrl(raw) {
22
- try {
23
- const url = new URL(stripTrailingPunctuation(raw));
24
- if (url.protocol !== 'http:' && url.protocol !== 'https:')
25
- return undefined;
26
- for (const key of [...url.searchParams.keys()]) {
27
- if (TRACKING_PARAMS.has(key.toLowerCase())) {
28
- url.searchParams.delete(key);
29
- }
30
- }
31
- url.hash = '';
32
- return url.toString().replace(/\/$/, '');
33
- }
34
- catch {
35
- return undefined;
36
- }
13
+ return canonicalizeUrl(stripTrailingPunctuation(raw), { canonicalizeHost: false });
37
14
  }
38
15
  export function extractDirectUrls(query) {
39
16
  const matches = query.match(/https?:\/\/\S+/gi) ?? [];
@@ -7,6 +7,7 @@ export type EvidenceQualityReport = {
7
7
  community: number;
8
8
  thread: number;
9
9
  packagePage: number;
10
+ primaryContent: number;
10
11
  distinctHosts: number;
11
12
  };
12
13
  flags: {
@@ -25,10 +25,11 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
25
25
  const community = evidence.filter((item) => item.sourceKind === 'community').length;
26
26
  const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
27
27
  const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
28
+ const primaryContent = evidence.filter((item) => item.sourceKind === 'primary-content').length;
28
29
  const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
29
30
  const hasOfficialEvidence = official > 0;
30
- const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
31
- const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
31
+ const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0 && primaryContent === 0;
32
+ const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1 && primaryContent === 0;
32
33
  const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
33
34
  const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
34
35
  const hasPossibleConflict = hasConflictMarkers(evidence);
@@ -47,6 +48,7 @@ export function analyzeEvidenceQuality({ evidence, gaps, lowValueOutcomes }) {
47
48
  community,
48
49
  thread,
49
50
  packagePage,
51
+ primaryContent,
50
52
  distinctHosts
51
53
  },
52
54
  flags: {
@@ -1,5 +1,7 @@
1
1
  function sourceRank(sourceKind) {
2
2
  switch (sourceKind) {
3
+ case 'primary-content':
4
+ return -1;
3
5
  case 'official-docs':
4
6
  return 0;
5
7
  case 'official-api':
@@ -24,6 +24,8 @@ export declare function createResearchWorkflow({ backendConfig, search, fetchPag
24
24
  headlessAttempts: number;
25
25
  exhaustedBudget: boolean;
26
26
  caveatReasons: import("./evidence-quality.js").EvidenceCaveatReason[];
27
+ fanoutProviders: import("../types.js").SearchProviderName[] | undefined;
28
+ fanoutSkipped: import("../types.js").SearchProviderName[] | undefined;
27
29
  };
28
30
  }>;
29
31
  };
@@ -1,4 +1,4 @@
1
- import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
1
+ import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
2
2
  import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
3
3
  import { type EvidenceCaveatReason } from './evidence-quality.js';
4
4
  export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
@@ -28,6 +28,8 @@ export declare function createResearchOrchestrator({ worker, fetchDirect, headle
28
28
  headlessAttempts: number;
29
29
  exhaustedBudget: boolean;
30
30
  caveatReasons: EvidenceCaveatReason[];
31
+ fanoutProviders: SearchProviderName[] | undefined;
32
+ fanoutSkipped: SearchProviderName[] | undefined;
31
33
  };
32
34
  }>;
33
35
  };
@@ -13,6 +13,9 @@ function classifyEvidenceUrl(url) {
13
13
  function summarizeText(text, maxLength = 180) {
14
14
  return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
15
15
  }
16
+ function isReaderMethod(method) {
17
+ return method === 'github' || method === 'pdf' || method === 'youtube';
18
+ }
16
19
  function isBotCheckContent({ title = '', text }) {
17
20
  return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
18
21
  }
@@ -21,6 +24,16 @@ function evidenceFromFetch(result) {
21
24
  return null;
22
25
  if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
23
26
  return null;
27
+ if (isReaderMethod(result.metadata.method)) {
28
+ return {
29
+ title: result.content.title ?? result.url,
30
+ url: result.url,
31
+ sourceKind: 'primary-content',
32
+ method: result.metadata.method,
33
+ summary: result.content.text,
34
+ supports: [result.content.text]
35
+ };
36
+ }
24
37
  return {
25
38
  title: result.content.title ?? result.url,
26
39
  url: result.url,
@@ -66,13 +79,15 @@ function shouldRetryDirectWithHeadless(result, evidence) {
66
79
  return false;
67
80
  return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
68
81
  }
69
- function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [] }) {
82
+ function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget, caveatReasons = [], fanoutProviders, fanoutSkipped }) {
70
83
  return {
71
84
  searchPasses: previousQueries.length,
72
85
  fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
73
86
  headlessAttempts,
74
87
  exhaustedBudget,
75
- caveatReasons
88
+ caveatReasons,
89
+ fanoutProviders,
90
+ fanoutSkipped
76
91
  };
77
92
  }
78
93
  function decisionForAnswer({ action, query, ranked, exhaustedBudget }) {
@@ -99,6 +114,13 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
99
114
  const suggestedHeadlessUrls = [];
100
115
  let headlessAttempts = 0;
101
116
  let lastPass;
117
+ const fanoutProvidersSeen = new Set();
118
+ const fanoutSkippedSeen = new Set();
119
+ function fanoutSnapshot() {
120
+ const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
121
+ const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
122
+ return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
123
+ }
102
124
  if (fetchDirect) {
103
125
  for (const url of extractDirectUrls(query).slice(0, 3)) {
104
126
  const directResult = await fetchDirect({ url });
@@ -134,6 +156,35 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
134
156
  }
135
157
  }
136
158
  }
159
+ if (allEvidence.some((item) => item.sourceKind === 'primary-content')) {
160
+ const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
161
+ const quality = analyzeEvidenceQuality({
162
+ evidence: ranked,
163
+ gaps: allGaps,
164
+ lowValueOutcomes: allLowValueOutcomes
165
+ });
166
+ return {
167
+ decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
168
+ evidence: ranked,
169
+ workerPass: combinedWorkerPass({
170
+ lastPass,
171
+ previousQueries,
172
+ allGaps,
173
+ allLowValueOutcomes,
174
+ exhaustedBudget: false
175
+ }),
176
+ metadata: buildMetadata({
177
+ previousQueries,
178
+ allEvidence,
179
+ allGaps,
180
+ allLowValueOutcomes,
181
+ headlessAttempts,
182
+ exhaustedBudget: false,
183
+ caveatReasons: quality.caveatReasons,
184
+ ...fanoutSnapshot()
185
+ })
186
+ };
187
+ }
137
188
  for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
138
189
  const queries = planSearchQueries({
139
190
  originalQuery: query,
@@ -149,6 +200,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
149
200
  maxFetches: DEFAULT_MAX_FETCHES_PER_PASS
150
201
  });
151
202
  lastPass = pass;
203
+ pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
204
+ pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
152
205
  allEvidence.push(...pass.evidence);
153
206
  allGaps.push(...pass.gaps);
154
207
  allLowValueOutcomes.push(...pass.lowValueOutcomes);
@@ -213,7 +266,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
213
266
  allLowValueOutcomes,
214
267
  headlessAttempts,
215
268
  exhaustedBudget,
216
- caveatReasons: updatedQuality.caveatReasons
269
+ caveatReasons: updatedQuality.caveatReasons,
270
+ ...fanoutSnapshot()
217
271
  })
218
272
  };
219
273
  }
@@ -239,7 +293,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
239
293
  allLowValueOutcomes,
240
294
  headlessAttempts,
241
295
  exhaustedBudget: false,
242
- caveatReasons: quality.caveatReasons
296
+ caveatReasons: quality.caveatReasons,
297
+ ...fanoutSnapshot()
243
298
  })
244
299
  };
245
300
  }
@@ -262,7 +317,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
262
317
  allLowValueOutcomes,
263
318
  headlessAttempts,
264
319
  exhaustedBudget,
265
- caveatReasons: quality.caveatReasons
320
+ caveatReasons: quality.caveatReasons,
321
+ ...fanoutSnapshot()
266
322
  })
267
323
  };
268
324
  }
@@ -291,7 +347,8 @@ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch
291
347
  allLowValueOutcomes,
292
348
  headlessAttempts,
293
349
  exhaustedBudget: true,
294
- caveatReasons: quality.caveatReasons
350
+ caveatReasons: quality.caveatReasons,
351
+ ...fanoutSnapshot()
295
352
  })
296
353
  };
297
354
  }
@@ -1,4 +1,5 @@
1
- export type ResearchSourceKind = 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
1
+ import type { SearchProviderName } from '../types.js';
2
+ export type ResearchSourceKind = 'primary-content' | 'official-docs' | 'official-api' | 'official-discussion' | 'community' | 'issue-thread' | 'package-page' | 'other';
2
3
  export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
3
4
  export type ResearchEvidence = {
4
5
  title: string;
@@ -24,12 +25,16 @@ export type ResearchWorkerResult = {
24
25
  lowValueOutcomes: ResearchLowValueOutcome[];
25
26
  suggestedHeadlessUrl?: string;
26
27
  exhaustedBudget: boolean;
28
+ fanoutProviders?: SearchProviderName[];
29
+ fanoutSkipped?: SearchProviderName[];
27
30
  };
28
31
  export type ResearchRunMetadata = {
29
32
  searchPasses: number;
30
33
  fetchedPages: number;
31
34
  headlessAttempts: number;
32
35
  exhaustedBudget: boolean;
36
+ fanoutProviders?: SearchProviderName[];
37
+ fanoutSkipped?: SearchProviderName[];
33
38
  };
34
39
  export type ResearchOrchestratorDecision = {
35
40
  action: 'answer';
@@ -6,6 +6,9 @@ function classifySource(url) {
6
6
  function summarizeText(text, maxLength = 180) {
7
7
  return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
8
8
  }
9
+ function isReaderMethod(method) {
10
+ return method === 'github' || method === 'pdf' || method === 'youtube';
11
+ }
9
12
  function isBotCheckContent({ title = '', text }) {
10
13
  return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
11
14
  }
@@ -15,6 +18,18 @@ function evidenceFromFetch(fetched, fallbackTitle) {
15
18
  return null;
16
19
  if (isBotCheckContent({ title: content.title, text: content.text }))
17
20
  return null;
21
+ // A successful reader read with usable text is primary content, exempt from the
22
+ // package-page filter below.
23
+ if (isReaderMethod(fetched.metadata.method) && content.text.trim()) {
24
+ return {
25
+ title: content.title ?? fallbackTitle,
26
+ url: fetched.url,
27
+ sourceKind: 'primary-content',
28
+ method: fetched.metadata.method,
29
+ summary: content.text,
30
+ supports: [content.text]
31
+ };
32
+ }
18
33
  const sourceKind = classifySource(fetched.url);
19
34
  if (sourceKind === 'package-page') {
20
35
  return null;
@@ -65,6 +80,8 @@ export function createResearchWorker({ search, fetchPage }) {
65
80
  };
66
81
  }
67
82
  const searchResult = await search({ query });
83
+ const fanoutProviders = searchResult.metadata.fanout?.providers;
84
+ const fanoutSkipped = searchResult.metadata.fanout?.skipped;
68
85
  if (searchResult.status !== 'ok') {
69
86
  return {
70
87
  searchQueries,
@@ -77,7 +94,9 @@ export function createResearchWorker({ search, fetchPage }) {
77
94
  ],
78
95
  lowValueOutcomes,
79
96
  suggestedHeadlessUrl,
80
- exhaustedBudget: false
97
+ exhaustedBudget: false,
98
+ fanoutProviders,
99
+ fanoutSkipped
81
100
  };
82
101
  }
83
102
  if (searchResult.results.length === 0) {
@@ -92,7 +111,9 @@ export function createResearchWorker({ search, fetchPage }) {
92
111
  }
93
112
  ],
94
113
  suggestedHeadlessUrl,
95
- exhaustedBudget: false
114
+ exhaustedBudget: false,
115
+ fanoutProviders,
116
+ fanoutSkipped
96
117
  };
97
118
  }
98
119
  const candidates = selectCandidates({
@@ -133,7 +154,9 @@ export function createResearchWorker({ search, fetchPage }) {
133
154
  gaps,
134
155
  lowValueOutcomes,
135
156
  suggestedHeadlessUrl,
136
- exhaustedBudget: false
157
+ exhaustedBudget: false,
158
+ fanoutProviders,
159
+ fanoutSkipped
137
160
  };
138
161
  }
139
162
  };
@@ -0,0 +1,6 @@
1
+ declare const TRACKING_PARAMS: Set<string>;
2
+ /** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
3
+ export declare function canonicalizeUrl(raw: string, options?: {
4
+ canonicalizeHost?: boolean;
5
+ }): string | undefined;
6
+ export { TRACKING_PARAMS };
@@ -0,0 +1,34 @@
1
+ const TRACKING_PARAMS = new Set([
2
+ 'utm_source',
3
+ 'utm_medium',
4
+ 'utm_campaign',
5
+ 'utm_term',
6
+ 'utm_content',
7
+ 'utm_name',
8
+ 'fbclid',
9
+ 'gclid'
10
+ ]);
11
+ /** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
12
+ export function canonicalizeUrl(raw, options = {}) {
13
+ const { canonicalizeHost = true } = options;
14
+ let url;
15
+ try {
16
+ url = new URL(raw);
17
+ }
18
+ catch {
19
+ return undefined;
20
+ }
21
+ if (url.protocol !== 'http:' && url.protocol !== 'https:')
22
+ return undefined;
23
+ if (canonicalizeHost) {
24
+ url.hostname = url.hostname.replace(/^www\./, '');
25
+ }
26
+ for (const key of [...url.searchParams.keys()]) {
27
+ if (TRACKING_PARAMS.has(key.toLowerCase())) {
28
+ url.searchParams.delete(key);
29
+ }
30
+ }
31
+ url.hash = '';
32
+ return url.toString().replace(/\/$/, '');
33
+ }
34
+ export { TRACKING_PARAMS };
@@ -22,8 +22,15 @@ export function buildExplorePresentation(result) {
22
22
  }
23
23
  };
24
24
  }
25
+ const fanoutProviders = result.metadata?.fanoutProviders;
26
+ const fanoutSkipped = result.metadata?.fanoutSkipped;
27
+ const fanoutNote = fanoutProviders?.length
28
+ ? ` (fanout: ${fanoutProviders.join(', ')}${fanoutSkipped?.length ? `; skipped: ${fanoutSkipped.join(', ')}` : ''})`
29
+ : fanoutSkipped?.length
30
+ ? ` (fanout; skipped: ${fanoutSkipped.join(', ')})`
31
+ : '';
25
32
  const internalSummary = result.metadata
26
- ? `Internal research: web_search ×${result.metadata.searchPasses}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
33
+ ? `Internal research: web_search ×${result.metadata.searchPasses}${fanoutNote}, web_fetch ×${result.metadata.fetchedPages}, web_fetch_headless ×${result.metadata.headlessAttempts}`
27
34
  : undefined;
28
35
  const hasEvidence = result.findings.length > 0 || result.sources.length > 0;
29
36
  const evidenceLines = hasEvidence
@@ -46,12 +53,13 @@ export function buildExplorePresentation(result) {
46
53
  ]
47
54
  .filter((line) => line !== undefined)
48
55
  .join('\n');
56
+ const compact = hasEvidence
57
+ ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
58
+ : 'No usable evidence found';
49
59
  return {
50
60
  mode: 'compact',
51
61
  views: {
52
- compact: hasEvidence
53
- ? `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`
54
- : 'No usable evidence found',
62
+ compact,
55
63
  preview,
56
64
  verbose
57
65
  },
@@ -1,12 +1,24 @@
1
+ function fanoutNote(result) {
2
+ const f = result.metadata.fanout;
3
+ if (!f)
4
+ return '';
5
+ if (f.providers.length) {
6
+ return ` (fanout: ${f.providers.join(', ')}${f.skipped?.length ? `; skipped: ${f.skipped.join(', ')}` : ''})`;
7
+ }
8
+ if (f.skipped?.length) {
9
+ return ` (fanout; skipped: ${f.skipped.join(', ')})`;
10
+ }
11
+ return '';
12
+ }
1
13
  function formatCompact(result) {
2
14
  const fallbackPrefix = result.metadata.fallbackFrom
3
15
  ? `${result.metadata.fallbackFrom} failed; used ${result.metadata.backend} fallback. `
4
16
  : '';
5
17
  if (result.status === 'error') {
6
- return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
18
+ return `${fallbackPrefix}Search failed: ${result.error?.message ?? 'Unknown search failure.'}${fanoutNote(result)}`;
7
19
  }
8
20
  const suffix = result.results.length === 1 ? 'result' : 'results';
9
- return `${fallbackPrefix}Found ${result.results.length} ${suffix}`;
21
+ return `${fallbackPrefix}Found ${result.results.length} ${suffix}${fanoutNote(result)}`;
10
22
  }
11
23
  export function buildSearchPresentation(result) {
12
24
  const preview = result.results
@@ -1,3 +1,4 @@
1
+ import { READER_TEXT_CAP } from './limits.js';
1
2
  function parseGithub(url) {
2
3
  try {
3
4
  return new URL(url);
@@ -51,12 +52,12 @@ function fail(url, message) {
51
52
  };
52
53
  }
53
54
  function okResponse(url, title, text) {
54
- const capped = text.slice(0, 4000);
55
+ const capped = text.slice(0, READER_TEXT_CAP);
55
56
  return {
56
57
  status: 'ok',
57
58
  url,
58
59
  content: { title, text: capped },
59
- metadata: { method: 'github', cacheHit: false, truncated: text.length >= 4000 }
60
+ metadata: { method: 'github', cacheHit: false, truncated: text.length >= READER_TEXT_CAP }
60
61
  };
61
62
  }
62
63
  export function createGithubReader({ fetchImpl = fetch, token = process.env.GITHUB_TOKEN } = {}) {
@@ -0,0 +1,3 @@
1
+ /** Max characters a special-content reader returns. Large enough for a typical transcript
2
+ * or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
3
+ export declare const READER_TEXT_CAP = 24000;
@@ -0,0 +1,3 @@
1
+ /** Max characters a special-content reader returns. Large enough for a typical transcript
2
+ * or a medium PDF; very long documents still truncate (section-aware chunking is future work). */
3
+ export const READER_TEXT_CAP = 24000;
@@ -1,4 +1,5 @@
1
1
  import { extractText, getDocumentProxy, getMeta } from 'unpdf';
2
+ import { READER_TEXT_CAP } from './limits.js';
2
3
  async function defaultExtract(bytes) {
3
4
  const pdf = await getDocumentProxy(bytes);
4
5
  const [{ text }, meta] = await Promise.all([
@@ -58,8 +59,8 @@ export function createPdfReader({ fetchImpl = fetch, extractPdfText = defaultExt
58
59
  return {
59
60
  status: 'ok',
60
61
  url,
61
- content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, 4000) },
62
- metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= 4000 }
62
+ content: { title: extracted.title ?? filenameFromUrl(url), text: text.slice(0, READER_TEXT_CAP) },
63
+ metadata: { method: 'pdf', cacheHit: false, contentType: 'application/pdf', truncated: text.length >= READER_TEXT_CAP }
63
64
  };
64
65
  }
65
66
  catch (err) {
@@ -1,4 +1,5 @@
1
1
  import { getSubtitles, getVideoDetails } from 'youtube-caption-extractor';
2
+ import { READER_TEXT_CAP } from './limits.js';
2
3
  function extractVideoId(url) {
3
4
  let parsed;
4
5
  try {
@@ -53,8 +54,8 @@ export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetail
53
54
  return {
54
55
  status: 'ok',
55
56
  url,
56
- content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, 4000) },
57
- metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= 4000 }
57
+ content: { title: details.title ?? `YouTube ${videoID}`, text: transcript.slice(0, READER_TEXT_CAP) },
58
+ metadata: { method: 'youtube', cacheHit: false, truncated: transcript.length >= READER_TEXT_CAP }
58
59
  };
59
60
  }
60
61
  catch (err) {
@@ -0,0 +1,14 @@
1
+ import type { FanoutMode, SearchProviderName, WebSearchResponse } from '../types.js';
2
+ export type FanoutProvider = {
3
+ name: SearchProviderName;
4
+ search: (input: {
5
+ query: string;
6
+ }) => Promise<WebSearchResponse>;
7
+ };
8
+ export declare function createFanoutSearch({ providers, mode, timeoutMs }: {
9
+ providers: FanoutProvider[];
10
+ mode: Exclude<FanoutMode, 'off'>;
11
+ timeoutMs?: number;
12
+ }): ({ query }: {
13
+ query: string;
14
+ }) => Promise<WebSearchResponse>;
@@ -0,0 +1,118 @@
1
+ import { buildSearchPresentation } from '../presentation/search-presentation.js';
2
+ import { canonicalizeUrl } from '../orchestration/url.js';
3
+ const FANOUT_MIN_RESULTS = 3;
4
+ function hostOf(url) {
5
+ try {
6
+ return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
7
+ }
8
+ catch {
9
+ return undefined;
10
+ }
11
+ }
12
+ function primaryLooksWeak(results) {
13
+ if (results.length < FANOUT_MIN_RESULTS)
14
+ return true;
15
+ const hosts = new Set(results.map((r) => hostOf(r.url)).filter(Boolean));
16
+ return hosts.size <= 1;
17
+ }
18
+ function merge(lists) {
19
+ const byKey = new Map();
20
+ for (const { name, results } of lists) {
21
+ const seenThisProvider = new Set();
22
+ results.forEach((result, index) => {
23
+ const key = canonicalizeUrl(result.url) ?? result.url;
24
+ if (seenThisProvider.has(key))
25
+ return; // ignore duplicates within a single provider's own list
26
+ seenThisProvider.add(key);
27
+ const existing = byKey.get(key);
28
+ if (!existing) {
29
+ byKey.set(key, { result: { ...result }, providers: new Set([name]), bestRank: index });
30
+ return;
31
+ }
32
+ existing.providers.add(name);
33
+ existing.bestRank = Math.min(existing.bestRank, index);
34
+ if ((result.title?.length ?? 0) > (existing.result.title?.length ?? 0))
35
+ existing.result.title = result.title;
36
+ if ((result.snippet?.length ?? 0) > (existing.result.snippet?.length ?? 0))
37
+ existing.result.snippet = result.snippet;
38
+ });
39
+ }
40
+ return [...byKey.values()]
41
+ .sort((a, b) => b.providers.size - a.providers.size || a.bestRank - b.bestRank)
42
+ .map((entry) => entry.result);
43
+ }
44
+ function withPresentation(result) {
45
+ return { ...result, presentation: buildSearchPresentation(result) };
46
+ }
47
+ const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
48
+ /** Resolve to undefined if the provider doesn't answer in time, so one slow/unreachable
49
+ * provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout across passes. */
50
+ function withTimeout(promise, ms) {
51
+ return new Promise((resolve) => {
52
+ const timer = setTimeout(() => resolve(undefined), ms);
53
+ promise.then((value) => {
54
+ clearTimeout(timer);
55
+ resolve(value);
56
+ }, () => {
57
+ clearTimeout(timer);
58
+ resolve(undefined);
59
+ });
60
+ });
61
+ }
62
+ export function createFanoutSearch({ providers, mode, timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS }) {
63
+ return async function fanoutSearch({ query }) {
64
+ const [primary, ...rest] = providers;
65
+ async function runSet(set) {
66
+ const settled = await Promise.all(set.map((p) => withTimeout(p.search({ query }), timeoutMs)));
67
+ const contributing = [];
68
+ const skipped = [];
69
+ set.forEach((provider, i) => {
70
+ const value = settled[i];
71
+ if (value && value.status === 'ok' && value.results.length > 0) {
72
+ contributing.push({ name: provider.name, results: value.results });
73
+ }
74
+ else {
75
+ skipped.push(provider.name);
76
+ }
77
+ });
78
+ return { contributing, skipped };
79
+ }
80
+ function finalize(lists, skipped, resolvedMode) {
81
+ if (lists.length === 0) {
82
+ const fanout = { mode: resolvedMode, providers: [] };
83
+ if (skipped.length > 0)
84
+ fanout.skipped = skipped;
85
+ return withPresentation({
86
+ status: 'error',
87
+ results: [],
88
+ metadata: { backend: primary.name, cacheHit: false, fanout },
89
+ error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
90
+ });
91
+ }
92
+ const fanout = { mode: resolvedMode, providers: lists.map((l) => l.name) };
93
+ if (skipped.length > 0)
94
+ fanout.skipped = skipped;
95
+ return withPresentation({
96
+ status: 'ok',
97
+ results: merge(lists),
98
+ metadata: { backend: primary.name, cacheHit: false, fanout }
99
+ });
100
+ }
101
+ if (mode === 'auto') {
102
+ const primaryOutcome = await withTimeout(primary.search({ query }), timeoutMs);
103
+ const primaryResults = primaryOutcome?.status === 'ok' ? primaryOutcome.results : [];
104
+ if (primaryResults.length > 0 && !primaryLooksWeak(primaryResults)) {
105
+ return withPresentation(primaryOutcome); // strong primary: no fanout
106
+ }
107
+ const { contributing, skipped } = await runSet(rest);
108
+ const lists = [
109
+ ...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
110
+ ...contributing
111
+ ];
112
+ const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
113
+ return finalize(lists, allSkipped, 'auto');
114
+ }
115
+ const { contributing, skipped } = await runSet(providers);
116
+ return finalize(contributing, skipped, 'on');
117
+ };
118
+ }
@@ -40,6 +40,8 @@ export declare function createWebExploreTool({ explore }?: {
40
40
  headlessAttempts: number;
41
41
  exhaustedBudget: boolean;
42
42
  caveatReasons?: string[];
43
+ fanoutProviders?: import("../types.js").SearchProviderName[];
44
+ fanoutSkipped?: import("../types.js").SearchProviderName[];
43
45
  };
44
46
  error?: import("../types.js").ToolError;
45
47
  }>;
package/dist/types.d.ts CHANGED
@@ -6,6 +6,13 @@ export type SearchResult = {
6
6
  url: string;
7
7
  snippet: string;
8
8
  };
9
+ export type SearchProviderName = 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
10
+ export type FanoutMode = 'off' | 'on' | 'auto';
11
+ export type FanoutMetadata = {
12
+ mode: Exclude<FanoutMode, 'off'>;
13
+ providers: SearchProviderName[];
14
+ skipped?: SearchProviderName[];
15
+ };
9
16
  export type ToolError = {
10
17
  code: string;
11
18
  message: string;
@@ -15,6 +22,7 @@ export type SearchMetadata = {
15
22
  cacheHit: boolean;
16
23
  fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
17
24
  fallbackReason?: string;
25
+ fanout?: FanoutMetadata;
18
26
  };
19
27
  export type FetchMethod = 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
20
28
  export type FetchMetadata = {
@@ -70,6 +78,8 @@ export type WebExploreResponse = {
70
78
  headlessAttempts: number;
71
79
  exhaustedBudget: boolean;
72
80
  caveatReasons?: string[];
81
+ fanoutProviders?: SearchProviderName[];
82
+ fanoutSkipped?: SearchProviderName[];
73
83
  };
74
84
  presentation?: PresentationEnvelope;
75
85
  error?: ToolError;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@demigodmode/pi-web-agent",
3
- "version": "1.8.0",
3
+ "version": "1.9.0",
4
4
  "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
5
5
  "type": "module",
6
6
  "main": "./dist/extension.js",