@demigodmode/pi-web-agent 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +19 -4
- package/dist/backends/config.d.ts +40 -0
- package/dist/backends/config.js +140 -1
- package/dist/backends/factory.d.ts +17 -1
- package/dist/backends/factory.js +170 -85
- package/dist/backends/failure.d.ts +11 -0
- package/dist/backends/failure.js +34 -0
- package/dist/backends/fallback-policy.d.ts +33 -0
- package/dist/backends/fallback-policy.js +239 -0
- package/dist/backends/provider-failure.d.ts +21 -0
- package/dist/backends/provider-failure.js +111 -0
- package/dist/backends/provider-health.d.ts +29 -0
- package/dist/backends/provider-health.js +49 -0
- package/dist/commands/web-agent-config.d.ts +17 -1
- package/dist/commands/web-agent-config.js +131 -7
- package/dist/extension.d.ts +1 -0
- package/dist/extension.js +49 -3
- package/dist/fetch/destination-policy.d.ts +32 -0
- package/dist/fetch/destination-policy.js +24 -0
- package/dist/fetch/firecrawl-fetch.js +64 -45
- package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
- package/dist/fetch/guard-proxy-fetch.js +82 -0
- package/dist/fetch/guard-proxy.d.ts +58 -0
- package/dist/fetch/guard-proxy.js +420 -0
- package/dist/fetch/guarded-fetch.d.ts +7 -0
- package/dist/fetch/guarded-fetch.js +75 -0
- package/dist/fetch/headless-fetch.d.ts +17 -2
- package/dist/fetch/headless-fetch.js +181 -9
- package/dist/fetch/http-fetch.js +16 -1
- package/dist/fetch/network-guard.d.ts +82 -0
- package/dist/fetch/network-guard.js +275 -0
- package/dist/fetch/proxy-fetch.d.ts +22 -0
- package/dist/fetch/proxy-fetch.js +46 -0
- package/dist/jiti-compat-run.d.ts +1 -0
- package/dist/jiti-compat-run.js +9 -0
- package/dist/jiti-compat.d.ts +32 -0
- package/dist/jiti-compat.js +215 -0
- package/dist/orchestration/answer-synthesizer.js +2 -0
- package/dist/orchestration/evidence-quality.d.ts +3 -2
- package/dist/orchestration/evidence-quality.js +2 -1
- package/dist/orchestration/index.d.ts +23 -0
- package/dist/orchestration/index.js +9 -2
- package/dist/orchestration/research-orchestrator.d.ts +21 -1
- package/dist/orchestration/research-orchestrator.js +40 -7
- package/dist/orchestration/research-types.d.ts +13 -1
- package/dist/orchestration/research-worker.js +38 -3
- package/dist/orchestration/stop-decider.js +3 -1
- package/dist/presentation/config-store.js +10 -0
- package/dist/presentation/explore-presentation.js +3 -1
- package/dist/presentation/fetch-presentation.js +16 -9
- package/dist/presentation/search-presentation.d.ts +2 -1
- package/dist/presentation/search-presentation.js +13 -1
- package/dist/readers/youtube-reader.d.ts +3 -1
- package/dist/readers/youtube-reader.js +11 -3
- package/dist/search/brave.d.ts +1 -2
- package/dist/search/brave.js +23 -80
- package/dist/search/duckduckgo.d.ts +7 -3
- package/dist/search/duckduckgo.js +17 -18
- package/dist/search/exa.d.ts +1 -2
- package/dist/search/exa.js +15 -76
- package/dist/search/fanout.d.ts +12 -0
- package/dist/search/fanout.js +86 -47
- package/dist/search/json-provider.d.ts +32 -0
- package/dist/search/json-provider.js +76 -0
- package/dist/search/searxng.d.ts +1 -2
- package/dist/search/searxng.js +15 -57
- package/dist/search/tavily.d.ts +1 -2
- package/dist/search/tavily.js +17 -74
- package/dist/search/youcom.d.ts +1 -2
- package/dist/search/youcom.js +15 -76
- package/dist/tools/web-explore.d.ts +9 -0
- package/dist/tools/web-explore.js +16 -2
- package/dist/tools/web-search.js +41 -103
- package/dist/types.d.ts +40 -0
- package/package.json +4 -3
- package/scripts/patch-jiti-compat.mjs +52 -8
package/dist/search/fanout.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
2
2
|
import { canonicalizeUrl } from '../orchestration/url.js';
|
|
3
|
+
import { failureOf, isTerminalFailure } from '../backends/failure.js';
|
|
4
|
+
import { RETRY_BASE_MS, RETRY_JITTER_MS } from '../backends/fallback-policy.js';
|
|
3
5
|
const FANOUT_MIN_RESULTS = 3;
|
|
4
6
|
function hostOf(url) {
|
|
5
7
|
try {
|
|
@@ -44,75 +46,112 @@ function merge(lists) {
|
|
|
44
46
|
function withPresentation(result) {
|
|
45
47
|
return { ...result, presentation: buildSearchPresentation(result) };
|
|
46
48
|
}
|
|
47
|
-
const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
function
|
|
49
|
+
export const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
|
|
50
|
+
const BACKSTOP_MARGIN_MS = 250;
|
|
51
|
+
/** Longest a policy-wrapped provider can legitimately take: two timed-out calls and the retry sleep. */
|
|
52
|
+
export function fanoutBackstopMs(timeoutMs) {
|
|
53
|
+
return 2 * timeoutMs + RETRY_BASE_MS + RETRY_JITTER_MS + BACKSTOP_MARGIN_MS;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Per-call timeout. Wrap it INSIDE the retry policy so a stalled call is a transient
|
|
57
|
+
* failure the policy can retry once (#55).
|
|
58
|
+
*/
|
|
59
|
+
export function withCallTimeout(search, timeoutMs, name) {
|
|
60
|
+
return (input) => withTimeout(search(input), timeoutMs, name);
|
|
61
|
+
}
|
|
62
|
+
/** A provider that doesn't answer in time (or throws) counts as a transient failure, so one
|
|
63
|
+
* slow/unreachable provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout. */
|
|
64
|
+
function withTimeout(promise, ms, name) {
|
|
65
|
+
const timedOut = () => ({
|
|
66
|
+
status: 'error',
|
|
67
|
+
results: [],
|
|
68
|
+
metadata: { backend: name, cacheHit: false },
|
|
69
|
+
error: { code: 'FETCH_FAILED', message: `${name} did not answer in time.`, failure: { kind: 'transient' } }
|
|
70
|
+
});
|
|
51
71
|
return new Promise((resolve) => {
|
|
52
|
-
const timer = setTimeout(() => resolve(
|
|
72
|
+
const timer = setTimeout(() => resolve(timedOut()), ms);
|
|
73
|
+
timer.unref?.();
|
|
53
74
|
promise.then((value) => {
|
|
54
75
|
clearTimeout(timer);
|
|
55
76
|
resolve(value);
|
|
56
77
|
}, () => {
|
|
57
78
|
clearTimeout(timer);
|
|
58
|
-
resolve(
|
|
79
|
+
resolve(timedOut());
|
|
59
80
|
});
|
|
60
81
|
});
|
|
61
82
|
}
|
|
83
|
+
function outcomeOf(name, response) {
|
|
84
|
+
const failure = failureOf(response);
|
|
85
|
+
if (!failure) {
|
|
86
|
+
return response.results.length > 0 ? { provider: name, outcome: 'results', count: response.results.length } : { provider: name, outcome: 'empty' };
|
|
87
|
+
}
|
|
88
|
+
const skipped = response.metadata.attempts?.find((a) => a.outcome === 'skipped');
|
|
89
|
+
return skipped
|
|
90
|
+
? { provider: name, outcome: 'skipped', failure, ...(skipped.skipReason ? { skipReason: skipped.skipReason } : {}) }
|
|
91
|
+
: { provider: name, outcome: 'failed', failure };
|
|
92
|
+
}
|
|
62
93
|
export function createFanoutSearch({ providers, mode, timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS }) {
|
|
94
|
+
const backstopMs = fanoutBackstopMs(timeoutMs);
|
|
63
95
|
return async function fanoutSearch({ query }) {
|
|
64
96
|
const [primary, ...rest] = providers;
|
|
65
97
|
async function runSet(set) {
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
const skipped = [];
|
|
69
|
-
set.forEach((provider, i) => {
|
|
70
|
-
const value = settled[i];
|
|
71
|
-
if (value && value.status === 'ok' && value.results.length > 0) {
|
|
72
|
-
contributing.push({ name: provider.name, results: value.results });
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
skipped.push(provider.name);
|
|
76
|
-
}
|
|
77
|
-
});
|
|
78
|
-
return { contributing, skipped };
|
|
98
|
+
const responses = await Promise.all(set.map((p) => withTimeout(p.search({ query }), backstopMs, p.name)));
|
|
99
|
+
return set.map((provider, i) => ({ provider, response: responses[i] }));
|
|
79
100
|
}
|
|
80
|
-
function finalize(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
101
|
+
function finalize(entries, resolvedMode) {
|
|
102
|
+
const outcomes = entries.map(({ provider, response }) => outcomeOf(provider.name, response));
|
|
103
|
+
const attempts = entries.flatMap(({ response }) => response.metadata.attempts ?? []);
|
|
104
|
+
const contributing = entries.filter((_, i) => outcomes[i].outcome === 'results');
|
|
105
|
+
const fanout = {
|
|
106
|
+
mode: resolvedMode,
|
|
107
|
+
providers: contributing.map(({ provider }) => provider.name),
|
|
108
|
+
outcomes
|
|
109
|
+
};
|
|
110
|
+
const skippedNames = outcomes.filter((o) => o.outcome !== 'results').map((o) => o.provider);
|
|
111
|
+
if (skippedNames.length > 0)
|
|
112
|
+
fanout.skipped = skippedNames;
|
|
113
|
+
// 1. terminal
|
|
114
|
+
const terminal = entries.find(({ response }) => isTerminalFailure(failureOf(response)));
|
|
115
|
+
if (terminal) {
|
|
116
|
+
return withPresentation({ ...terminal.response, metadata: { ...terminal.response.metadata, backend: primary.name, fanout, attempts } });
|
|
117
|
+
}
|
|
118
|
+
const unavailable = outcomes
|
|
119
|
+
.filter((o) => o.outcome === 'failed' || o.outcome === 'skipped')
|
|
120
|
+
.map((o) => ({ provider: o.provider, kind: o.failure.kind }));
|
|
121
|
+
const coverage = unavailable.length > 0 ? { coverage: { partial: true, unavailable } } : {};
|
|
122
|
+
// 2. results
|
|
123
|
+
if (contributing.length > 0) {
|
|
85
124
|
return withPresentation({
|
|
86
|
-
status: '
|
|
87
|
-
results:
|
|
88
|
-
metadata: { backend: primary.name, cacheHit: false, fanout }
|
|
89
|
-
error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
|
|
125
|
+
status: 'ok',
|
|
126
|
+
results: merge(contributing.map(({ provider, response }) => ({ name: provider.name, results: response.results }))),
|
|
127
|
+
metadata: { backend: primary.name, cacheHit: false, fanout, attempts, ...coverage }
|
|
90
128
|
});
|
|
91
129
|
}
|
|
92
|
-
|
|
93
|
-
if (
|
|
94
|
-
|
|
130
|
+
// 3. valid empty
|
|
131
|
+
if (outcomes.some((o) => o.outcome === 'empty')) {
|
|
132
|
+
return withPresentation({ status: 'ok', results: [], metadata: { backend: primary.name, cacheHit: false, fanout, attempts, ...coverage } });
|
|
133
|
+
}
|
|
134
|
+
// 4. all failed or skipped: non-terminal. A bad_request anywhere wins so an outer
|
|
135
|
+
// chain never falls back on it, whatever order the providers ran in.
|
|
136
|
+
const badRequest = outcomes.find((o) => o.failure?.kind === 'bad_request')?.failure;
|
|
137
|
+
const lastFailure = badRequest ?? [...outcomes].reverse().find((o) => o.failure)?.failure ?? { kind: 'bad_response' };
|
|
95
138
|
return withPresentation({
|
|
96
|
-
status: '
|
|
97
|
-
results:
|
|
98
|
-
metadata: { backend: primary.name, cacheHit: false, fanout }
|
|
139
|
+
status: 'error',
|
|
140
|
+
results: [],
|
|
141
|
+
metadata: { backend: primary.name, cacheHit: false, fanout, attempts },
|
|
142
|
+
error: { code: 'FANOUT_ALL_FAILED', message: 'Every fanout provider failed or was unavailable.', failure: lastFailure }
|
|
99
143
|
});
|
|
100
144
|
}
|
|
101
145
|
if (mode === 'auto') {
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
146
|
+
const primaryResponse = await withTimeout(primary.search({ query }), backstopMs, primary.name);
|
|
147
|
+
if (isTerminalFailure(failureOf(primaryResponse))) {
|
|
148
|
+
return finalize([{ provider: primary, response: primaryResponse }], 'auto');
|
|
149
|
+
}
|
|
150
|
+
if (primaryResponse.status === 'ok' && primaryResponse.results.length > 0 && !primaryLooksWeak(primaryResponse.results)) {
|
|
151
|
+
return withPresentation(primaryResponse); // strong primary: no fanout
|
|
106
152
|
}
|
|
107
|
-
|
|
108
|
-
const lists = [
|
|
109
|
-
...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
|
|
110
|
-
...contributing
|
|
111
|
-
];
|
|
112
|
-
const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
|
|
113
|
-
return finalize(lists, allSkipped, 'auto');
|
|
153
|
+
return finalize([{ provider: primary, response: primaryResponse }, ...(await runSet(rest))], 'auto');
|
|
114
154
|
}
|
|
115
|
-
|
|
116
|
-
return finalize(contributing, skipped, 'on');
|
|
155
|
+
return finalize(await runSet(providers), 'on');
|
|
117
156
|
};
|
|
118
157
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { SearchProviderName, SearchResult, WebSearchResponse } from '../types.js';
|
|
2
|
+
export type Normalized = {
|
|
3
|
+
rawCount: number;
|
|
4
|
+
results: SearchResult[];
|
|
5
|
+
};
|
|
6
|
+
export type JsonSearchProviderOptions = {
|
|
7
|
+
name: Exclude<SearchProviderName, 'duckduckgo'>;
|
|
8
|
+
label: string;
|
|
9
|
+
/** Omit for providers that need no key (SearXNG, keyless Tavily). */
|
|
10
|
+
apiKey?: string;
|
|
11
|
+
requiresKey?: boolean;
|
|
12
|
+
missingKeyMessage?: string;
|
|
13
|
+
fetchImpl?: typeof fetch;
|
|
14
|
+
now?: () => number;
|
|
15
|
+
request: (query: string) => {
|
|
16
|
+
url: string;
|
|
17
|
+
init?: RequestInit;
|
|
18
|
+
};
|
|
19
|
+
/** Returns undefined when the body does not have the documented shape. */
|
|
20
|
+
normalize: (json: unknown) => Normalized | undefined;
|
|
21
|
+
/** A 200 that is empty only because the provider degraded (e.g. SearXNG engines suspended). */
|
|
22
|
+
isDegradedEmpty?: (json: unknown) => boolean;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Classifies only; the fallback policy decides what to do (#55).
|
|
26
|
+
* Valid empty responses are `ok` with `[]`; anything malformed or degraded is `bad_response`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createJsonSearchProvider(options: JsonSearchProviderOptions): ({ query }: {
|
|
29
|
+
query: string;
|
|
30
|
+
}) => Promise<WebSearchResponse>;
|
|
31
|
+
/** Shared normalizer for `{ results: [{ title, url, <snippetField> }] }` bodies. */
|
|
32
|
+
export declare function normalizeResultsArray(json: unknown, arrayPath: (body: any) => unknown, snippetField: string): Normalized | undefined;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { classifyHttpFailure, readResponseParts } from '../backends/provider-failure.js';
|
|
2
|
+
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
3
|
+
function respond(result) {
|
|
4
|
+
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Classifies only; the fallback policy decides what to do (#55).
|
|
8
|
+
* Valid empty responses are `ok` with `[]`; anything malformed or degraded is `bad_response`.
|
|
9
|
+
*/
|
|
10
|
+
export function createJsonSearchProvider(options) {
|
|
11
|
+
const { name, label, fetchImpl = fetch, now = Date.now } = options;
|
|
12
|
+
const requiresKey = options.requiresKey ?? true;
|
|
13
|
+
const error = (code, message, failure) => respond({ status: 'error', results: [], metadata: { backend: name, cacheHit: false }, error: { code, message, failure } });
|
|
14
|
+
return async function search({ query }) {
|
|
15
|
+
const normalizedQuery = query.trim();
|
|
16
|
+
if (!normalizedQuery) {
|
|
17
|
+
return error('INVALID_QUERY', 'Query must not be empty.', { kind: 'bad_request' });
|
|
18
|
+
}
|
|
19
|
+
if (requiresKey && !options.apiKey?.trim()) {
|
|
20
|
+
return error('BACKEND_CONFIG_INVALID', options.missingKeyMessage ?? `${label} search is not configured.`, {
|
|
21
|
+
kind: 'not_configured'
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
const { url, init } = options.request(normalizedQuery);
|
|
25
|
+
let response;
|
|
26
|
+
try {
|
|
27
|
+
// No init for plain GETs, so callers and tests see fetch(url) exactly.
|
|
28
|
+
response = await (init ? fetchImpl(url, init) : fetchImpl(url));
|
|
29
|
+
}
|
|
30
|
+
catch (thrown) {
|
|
31
|
+
const message = thrown instanceof Error ? thrown.message : String(thrown);
|
|
32
|
+
return error('FETCH_FAILED', `${label} search request failed: ${message}`, { kind: 'transient' });
|
|
33
|
+
}
|
|
34
|
+
let parts;
|
|
35
|
+
try {
|
|
36
|
+
parts = await readResponseParts(response);
|
|
37
|
+
}
|
|
38
|
+
catch (thrown) {
|
|
39
|
+
const message = thrown instanceof Error ? thrown.message : String(thrown);
|
|
40
|
+
return error('FETCH_FAILED', `${label} search response could not be read: ${message}`, { kind: 'transient' });
|
|
41
|
+
}
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
return error('FETCH_FAILED', `${label} search request failed: HTTP ${response.status}`, classifyHttpFailure(name, parts, now()));
|
|
44
|
+
}
|
|
45
|
+
const normalized = parts.json === undefined ? undefined : options.normalize(parts.json);
|
|
46
|
+
if (!normalized || (normalized.rawCount > 0 && normalized.results.length === 0)) {
|
|
47
|
+
return error('BAD_RESPONSE', `${label} returned a response that did not match the expected format.`, {
|
|
48
|
+
kind: 'bad_response',
|
|
49
|
+
httpStatus: response.status
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (normalized.results.length === 0 && options.isDegradedEmpty?.(parts.json)) {
|
|
53
|
+
return error('BAD_RESPONSE', `${label} returned no results because some of its sources were unavailable.`, {
|
|
54
|
+
kind: 'bad_response',
|
|
55
|
+
httpStatus: response.status
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return respond({ status: 'ok', results: normalized.results, metadata: { backend: name, cacheHit: false } });
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** Shared normalizer for `{ results: [{ title, url, <snippetField> }] }` bodies. */
|
|
62
|
+
export function normalizeResultsArray(json, arrayPath, snippetField) {
|
|
63
|
+
if (!json || typeof json !== 'object')
|
|
64
|
+
return undefined;
|
|
65
|
+
const raw = arrayPath(json);
|
|
66
|
+
if (raw === undefined)
|
|
67
|
+
return undefined;
|
|
68
|
+
if (!Array.isArray(raw))
|
|
69
|
+
return undefined;
|
|
70
|
+
return {
|
|
71
|
+
rawCount: raw.length,
|
|
72
|
+
results: raw.flatMap((item) => item && typeof item.title === 'string' && typeof item.url === 'string'
|
|
73
|
+
? [{ title: item.title, url: item.url, snippet: typeof item[snippetField] === 'string' ? item[snippetField] : '' }]
|
|
74
|
+
: [])
|
|
75
|
+
};
|
|
76
|
+
}
|
package/dist/search/searxng.d.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import type { SearxngOptions } from '../backends/config.js';
|
|
2
|
-
import type { WebSearchResponse } from '../types.js';
|
|
3
2
|
export declare function createSearxngSearchTool({ baseUrl, options, fetchImpl }: {
|
|
4
3
|
baseUrl: string;
|
|
5
4
|
options?: SearxngOptions;
|
|
6
5
|
fetchImpl?: typeof fetch;
|
|
7
6
|
}): ({ query }: {
|
|
8
7
|
query: string;
|
|
9
|
-
}) => Promise<WebSearchResponse>;
|
|
8
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/searxng.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
function buildSearchUrl(baseUrl, query, options = {}) {
|
|
3
3
|
const url = new URL('/search', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`);
|
|
4
4
|
url.searchParams.set('q', query);
|
|
@@ -11,62 +11,20 @@ function buildSearchUrl(baseUrl, query, options = {}) {
|
|
|
11
11
|
url.searchParams.set('safesearch', String(options.safesearch));
|
|
12
12
|
return url.toString();
|
|
13
13
|
}
|
|
14
|
-
function normalizeResults(response) {
|
|
15
|
-
return (response.results ?? []).flatMap((result) => {
|
|
16
|
-
if (typeof result.title !== 'string' || typeof result.url !== 'string') {
|
|
17
|
-
return [];
|
|
18
|
-
}
|
|
19
|
-
return [
|
|
20
|
-
{
|
|
21
|
-
title: result.title,
|
|
22
|
-
url: result.url,
|
|
23
|
-
snippet: typeof result.content === 'string' ? result.content : ''
|
|
24
|
-
}
|
|
25
|
-
];
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
14
|
export function createSearxngSearchTool({ baseUrl, options, fetchImpl = fetch }) {
|
|
29
|
-
return
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
15
|
+
return createJsonSearchProvider({
|
|
16
|
+
name: 'searxng',
|
|
17
|
+
label: 'SearXNG',
|
|
18
|
+
requiresKey: false,
|
|
19
|
+
fetchImpl,
|
|
20
|
+
request: (query) => ({ url: buildSearchUrl(baseUrl, query, options) }),
|
|
21
|
+
normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'content'),
|
|
22
|
+
// Suspended upstream engines still answer 200; version-dependent field (UNVERIFIED everywhere).
|
|
23
|
+
isDegradedEmpty: (json) => {
|
|
24
|
+
const unresponsive = json.unresponsive_engines;
|
|
25
|
+
if (Array.isArray(unresponsive))
|
|
26
|
+
return unresponsive.length > 0;
|
|
27
|
+
return !!unresponsive && typeof unresponsive === 'object' && Object.keys(unresponsive).length > 0;
|
|
39
28
|
}
|
|
40
|
-
|
|
41
|
-
const response = await fetchImpl(buildSearchUrl(baseUrl, normalizedQuery, options));
|
|
42
|
-
if (!response.ok) {
|
|
43
|
-
throw new Error(`HTTP ${response.status}`);
|
|
44
|
-
}
|
|
45
|
-
const parsed = (await response.json());
|
|
46
|
-
const results = normalizeResults(parsed);
|
|
47
|
-
const result = results.length > 0
|
|
48
|
-
? {
|
|
49
|
-
status: 'ok',
|
|
50
|
-
results,
|
|
51
|
-
metadata: { backend: 'searxng', cacheHit: false }
|
|
52
|
-
}
|
|
53
|
-
: {
|
|
54
|
-
status: 'error',
|
|
55
|
-
results: [],
|
|
56
|
-
metadata: { backend: 'searxng', cacheHit: false },
|
|
57
|
-
error: { code: 'NO_RESULTS', message: 'SearXNG returned no usable results for this query.' }
|
|
58
|
-
};
|
|
59
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
60
|
-
}
|
|
61
|
-
catch (error) {
|
|
62
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
63
|
-
const result = {
|
|
64
|
-
status: 'error',
|
|
65
|
-
results: [],
|
|
66
|
-
metadata: { backend: 'searxng', cacheHit: false },
|
|
67
|
-
error: { code: 'FETCH_FAILED', message: `SearXNG search request failed: ${rawMessage}` }
|
|
68
|
-
};
|
|
69
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
70
|
-
}
|
|
71
|
-
};
|
|
29
|
+
});
|
|
72
30
|
}
|
package/dist/search/tavily.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createTavilySearchTool({ apiKey, keyless, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
keyless?: boolean;
|
|
5
4
|
fetchImpl?: typeof fetch;
|
|
6
5
|
}): ({ query }: {
|
|
7
6
|
query: string;
|
|
8
|
-
}) => Promise<WebSearchResponse>;
|
|
7
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/tavily.js
CHANGED
|
@@ -1,83 +1,26 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
const TAVILY_SEARCH_URL = 'https://api.tavily.com/search';
|
|
3
|
-
function resultWithPresentation(result) {
|
|
4
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
-
}
|
|
6
|
-
function normalizeResults(response) {
|
|
7
|
-
return (response.results ?? []).flatMap((item) => {
|
|
8
|
-
if (typeof item.title !== 'string' || typeof item.url !== 'string') {
|
|
9
|
-
return [];
|
|
10
|
-
}
|
|
11
|
-
return [
|
|
12
|
-
{
|
|
13
|
-
title: item.title,
|
|
14
|
-
url: item.url,
|
|
15
|
-
snippet: typeof item.content === 'string' ? item.content : ''
|
|
16
|
-
}
|
|
17
|
-
];
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
3
|
export function createTavilySearchTool({ apiKey, keyless = false, fetchImpl = fetch }) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
return resultWithPresentation({
|
|
33
|
-
status: 'error',
|
|
34
|
-
results: [],
|
|
35
|
-
metadata: { backend: 'tavily', cacheHit: false },
|
|
36
|
-
error: {
|
|
37
|
-
code: 'BACKEND_CONFIG_INVALID',
|
|
38
|
-
message: 'Tavily search requires TAVILY_API_KEY.'
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
try {
|
|
43
|
-
const response = await fetchImpl(TAVILY_SEARCH_URL, {
|
|
4
|
+
const keyed = Boolean(apiKey?.trim());
|
|
5
|
+
return createJsonSearchProvider({
|
|
6
|
+
name: 'tavily',
|
|
7
|
+
label: 'Tavily',
|
|
8
|
+
apiKey,
|
|
9
|
+
requiresKey: !keyless,
|
|
10
|
+
missingKeyMessage: 'Tavily search requires TAVILY_API_KEY.',
|
|
11
|
+
fetchImpl,
|
|
12
|
+
request: (query) => ({
|
|
13
|
+
url: TAVILY_SEARCH_URL,
|
|
14
|
+
init: {
|
|
44
15
|
method: 'POST',
|
|
45
16
|
headers: {
|
|
46
17
|
Accept: 'application/json',
|
|
47
18
|
'Content-Type': 'application/json',
|
|
48
|
-
...(apiKey
|
|
49
|
-
? { Authorization: `Bearer ${apiKey}` }
|
|
50
|
-
: { 'X-Tavily-Access-Mode': 'keyless' })
|
|
19
|
+
...(keyed ? { Authorization: `Bearer ${apiKey}` } : { 'X-Tavily-Access-Mode': 'keyless' })
|
|
51
20
|
},
|
|
52
|
-
body: JSON.stringify({ query
|
|
53
|
-
});
|
|
54
|
-
if (!response.ok) {
|
|
55
|
-
throw new Error(`HTTP ${response.status}`);
|
|
56
|
-
}
|
|
57
|
-
const parsed = (await response.json());
|
|
58
|
-
const results = normalizeResults(parsed);
|
|
59
|
-
if (results.length === 0) {
|
|
60
|
-
return resultWithPresentation({
|
|
61
|
-
status: 'error',
|
|
62
|
-
results: [],
|
|
63
|
-
metadata: { backend: 'tavily', cacheHit: false },
|
|
64
|
-
error: { code: 'NO_RESULTS', message: 'Tavily returned no usable results for this query.' }
|
|
65
|
-
});
|
|
21
|
+
body: JSON.stringify({ query, max_results: 10 })
|
|
66
22
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
metadata: { backend: 'tavily', cacheHit: false }
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
catch (error) {
|
|
74
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
75
|
-
return resultWithPresentation({
|
|
76
|
-
status: 'error',
|
|
77
|
-
results: [],
|
|
78
|
-
metadata: { backend: 'tavily', cacheHit: false },
|
|
79
|
-
error: { code: 'FETCH_FAILED', message: `Tavily search request failed: ${rawMessage}` }
|
|
80
|
-
});
|
|
81
|
-
}
|
|
82
|
-
};
|
|
23
|
+
}),
|
|
24
|
+
normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'content')
|
|
25
|
+
});
|
|
83
26
|
}
|
package/dist/search/youcom.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import type { WebSearchResponse } from '../types.js';
|
|
2
1
|
export declare function createYouComSearchTool({ apiKey, fetchImpl }: {
|
|
3
2
|
apiKey?: string;
|
|
4
3
|
fetchImpl?: typeof fetch;
|
|
5
4
|
}): ({ query }: {
|
|
6
5
|
query: string;
|
|
7
|
-
}) => Promise<WebSearchResponse>;
|
|
6
|
+
}) => Promise<import("../types.js").WebSearchResponse>;
|
package/dist/search/youcom.js
CHANGED
|
@@ -1,81 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createJsonSearchProvider, normalizeResultsArray } from './json-provider.js';
|
|
2
2
|
const YOUCOM_SEARCH_URL = 'https://api.you.com/v1/agents/search';
|
|
3
|
-
function resultWithPresentation(result) {
|
|
4
|
-
return { ...result, presentation: buildSearchPresentation(result) };
|
|
5
|
-
}
|
|
6
|
-
function normalizeResults(response) {
|
|
7
|
-
return (response.results ?? []).flatMap((item) => {
|
|
8
|
-
if (typeof item.title !== 'string' || typeof item.url !== 'string') {
|
|
9
|
-
return [];
|
|
10
|
-
}
|
|
11
|
-
return [
|
|
12
|
-
{
|
|
13
|
-
title: item.title,
|
|
14
|
-
url: item.url,
|
|
15
|
-
snippet: typeof item.snippet === 'string' ? item.snippet : ''
|
|
16
|
-
}
|
|
17
|
-
];
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
3
|
export function createYouComSearchTool({ apiKey, fetchImpl = fetch }) {
|
|
21
|
-
return
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
if (!apiKey?.trim()) {
|
|
32
|
-
return resultWithPresentation({
|
|
33
|
-
status: 'error',
|
|
34
|
-
results: [],
|
|
35
|
-
metadata: { backend: 'youcom', cacheHit: false },
|
|
36
|
-
error: {
|
|
37
|
-
code: 'BACKEND_CONFIG_INVALID',
|
|
38
|
-
message: 'You.com search requires YDC_API_KEY.'
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
}
|
|
42
|
-
try {
|
|
43
|
-
const response = await fetchImpl(YOUCOM_SEARCH_URL, {
|
|
4
|
+
return createJsonSearchProvider({
|
|
5
|
+
name: 'youcom',
|
|
6
|
+
label: 'You.com',
|
|
7
|
+
apiKey,
|
|
8
|
+
missingKeyMessage: 'You.com search requires YDC_API_KEY.',
|
|
9
|
+
fetchImpl,
|
|
10
|
+
request: (query) => ({
|
|
11
|
+
url: YOUCOM_SEARCH_URL,
|
|
12
|
+
init: {
|
|
44
13
|
method: 'POST',
|
|
45
|
-
headers: {
|
|
46
|
-
|
|
47
|
-
'Content-Type': 'application/json',
|
|
48
|
-
'X-API-Key': apiKey
|
|
49
|
-
},
|
|
50
|
-
body: JSON.stringify({ query: normalizedQuery, max_results: 10 })
|
|
51
|
-
});
|
|
52
|
-
if (!response.ok) {
|
|
53
|
-
throw new Error(`HTTP ${response.status}`);
|
|
54
|
-
}
|
|
55
|
-
const parsed = (await response.json());
|
|
56
|
-
const results = normalizeResults(parsed);
|
|
57
|
-
if (results.length === 0) {
|
|
58
|
-
return resultWithPresentation({
|
|
59
|
-
status: 'error',
|
|
60
|
-
results: [],
|
|
61
|
-
metadata: { backend: 'youcom', cacheHit: false },
|
|
62
|
-
error: { code: 'NO_RESULTS', message: 'You.com returned no usable results for this query.' }
|
|
63
|
-
});
|
|
14
|
+
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'X-API-Key': apiKey ?? '' },
|
|
15
|
+
body: JSON.stringify({ query, max_results: 10 })
|
|
64
16
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
metadata: { backend: 'youcom', cacheHit: false }
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
catch (error) {
|
|
72
|
-
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
73
|
-
return resultWithPresentation({
|
|
74
|
-
status: 'error',
|
|
75
|
-
results: [],
|
|
76
|
-
metadata: { backend: 'youcom', cacheHit: false },
|
|
77
|
-
error: { code: 'FETCH_FAILED', message: `You.com search request failed: ${rawMessage}` }
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
};
|
|
17
|
+
}),
|
|
18
|
+
normalize: (json) => normalizeResultsArray(json, (body) => body.results, 'snippet')
|
|
19
|
+
});
|
|
81
20
|
}
|
|
@@ -11,6 +11,10 @@ export declare function createWebExploreTool({ explore }?: {
|
|
|
11
11
|
evidence: ResearchEvidence[];
|
|
12
12
|
workerPass: unknown;
|
|
13
13
|
metadata?: WebExploreResponse['metadata'];
|
|
14
|
+
terminalFailure?: {
|
|
15
|
+
code: string;
|
|
16
|
+
message: string;
|
|
17
|
+
};
|
|
14
18
|
}>;
|
|
15
19
|
} | ((input: {
|
|
16
20
|
query: string;
|
|
@@ -21,6 +25,10 @@ export declare function createWebExploreTool({ explore }?: {
|
|
|
21
25
|
evidence: ResearchEvidence[];
|
|
22
26
|
workerPass: unknown;
|
|
23
27
|
metadata?: WebExploreResponse['metadata'];
|
|
28
|
+
terminalFailure?: {
|
|
29
|
+
code: string;
|
|
30
|
+
message: string;
|
|
31
|
+
};
|
|
24
32
|
}>);
|
|
25
33
|
}): ({ query }: {
|
|
26
34
|
query: string;
|
|
@@ -42,6 +50,7 @@ export declare function createWebExploreTool({ explore }?: {
|
|
|
42
50
|
caveatReasons?: string[];
|
|
43
51
|
fanoutProviders?: import("../types.js").SearchProviderName[];
|
|
44
52
|
fanoutSkipped?: import("../types.js").SearchProviderName[];
|
|
53
|
+
attempts?: import("../types.js").Attempt[];
|
|
45
54
|
};
|
|
46
55
|
error?: import("../types.js").ToolError;
|
|
47
56
|
}>;
|
|
@@ -18,15 +18,29 @@ export function createWebExploreTool({ explore = createResearchWorkflow() } = {}
|
|
|
18
18
|
};
|
|
19
19
|
}
|
|
20
20
|
const result = await runExplore({ query: normalizedQuery });
|
|
21
|
+
if (result.terminalFailure) {
|
|
22
|
+
const failed = {
|
|
23
|
+
status: 'error',
|
|
24
|
+
findings: [],
|
|
25
|
+
sources: [],
|
|
26
|
+
error: result.terminalFailure,
|
|
27
|
+
metadata: result.metadata
|
|
28
|
+
};
|
|
29
|
+
return { ...failed, presentation: buildExplorePresentation(failed) };
|
|
30
|
+
}
|
|
21
31
|
const sources = result.evidence.slice(0, 4).map((item) => ({
|
|
22
32
|
title: item.title,
|
|
23
33
|
url: item.url,
|
|
24
34
|
method: item.method
|
|
25
35
|
}));
|
|
36
|
+
const reasons = (result.metadata?.caveatReasons ?? []);
|
|
37
|
+
const decisionPartial = result.decision.action !== 'answer';
|
|
38
|
+
const coveragePartial = reasons.includes('partial-search-coverage');
|
|
26
39
|
const synthesized = synthesizeAnswer({
|
|
27
40
|
evidence: result.evidence,
|
|
28
|
-
partial:
|
|
29
|
-
|
|
41
|
+
partial: decisionPartial || coveragePartial,
|
|
42
|
+
// A confident answer with partial coverage mentions only the coverage, not unrelated quality notes.
|
|
43
|
+
caveatReasons: decisionPartial ? reasons : ['partial-search-coverage']
|
|
30
44
|
});
|
|
31
45
|
const shaped = {
|
|
32
46
|
status: 'ok',
|