@demigodmode/pi-web-agent 1.11.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/backends/config.d.ts +13 -0
- package/dist/backends/config.js +44 -1
- package/dist/backends/factory.d.ts +15 -0
- package/dist/backends/factory.js +118 -91
- package/dist/backends/failure.d.ts +11 -0
- package/dist/backends/failure.js +34 -0
- package/dist/backends/fallback-policy.d.ts +33 -0
- package/dist/backends/fallback-policy.js +239 -0
- package/dist/backends/provider-failure.d.ts +21 -0
- package/dist/backends/provider-failure.js +111 -0
- package/dist/backends/provider-health.d.ts +29 -0
- package/dist/backends/provider-health.js +49 -0
- package/dist/commands/web-agent-config.d.ts +14 -1
- package/dist/commands/web-agent-config.js +75 -3
- package/dist/extension.js +47 -3
- package/dist/fetch/destination-policy.d.ts +32 -0
- package/dist/fetch/destination-policy.js +24 -0
- package/dist/fetch/firecrawl-fetch.js +64 -45
- package/dist/fetch/guard-proxy-fetch.d.ts +17 -0
- package/dist/fetch/guard-proxy-fetch.js +82 -0
- package/dist/fetch/guard-proxy.d.ts +58 -0
- package/dist/fetch/guard-proxy.js +420 -0
- package/dist/fetch/guarded-fetch.d.ts +7 -0
- package/dist/fetch/guarded-fetch.js +75 -0
- package/dist/fetch/headless-fetch.d.ts +10 -2
- package/dist/fetch/headless-fetch.js +181 -9
- package/dist/fetch/http-fetch.js +16 -1
- package/dist/fetch/network-guard.d.ts +82 -0
- package/dist/fetch/network-guard.js +275 -0
- package/dist/orchestration/answer-synthesizer.js +2 -0
- package/dist/orchestration/evidence-quality.d.ts +3 -2
- package/dist/orchestration/evidence-quality.js +2 -1
- package/dist/orchestration/index.d.ts +23 -0
- package/dist/orchestration/index.js +9 -2
- package/dist/orchestration/research-orchestrator.d.ts +21 -1
- package/dist/orchestration/research-orchestrator.js +40 -7
- package/dist/orchestration/research-types.d.ts +13 -1
- package/dist/orchestration/research-worker.js +38 -3
- package/dist/orchestration/stop-decider.js +3 -1
- package/dist/presentation/config-store.js +6 -0
- package/dist/presentation/explore-presentation.js +3 -1
- package/dist/presentation/fetch-presentation.js +16 -9
- package/dist/presentation/search-presentation.d.ts +2 -1
- package/dist/presentation/search-presentation.js +13 -1
- package/dist/search/brave.d.ts +1 -2
- package/dist/search/brave.js +23 -80
- package/dist/search/duckduckgo.d.ts +7 -3
- package/dist/search/duckduckgo.js +17 -18
- package/dist/search/exa.d.ts +1 -2
- package/dist/search/exa.js +15 -76
- package/dist/search/fanout.d.ts +12 -0
- package/dist/search/fanout.js +86 -47
- package/dist/search/json-provider.d.ts +32 -0
- package/dist/search/json-provider.js +76 -0
- package/dist/search/searxng.d.ts +1 -2
- package/dist/search/searxng.js +15 -57
- package/dist/search/tavily.d.ts +1 -2
- package/dist/search/tavily.js +17 -74
- package/dist/search/youcom.d.ts +1 -2
- package/dist/search/youcom.js +15 -76
- package/dist/tools/web-explore.d.ts +9 -0
- package/dist/tools/web-explore.js +16 -2
- package/dist/tools/web-search.js +41 -103
- package/dist/types.d.ts +40 -0
- package/package.json +3 -3
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
|
|
2
|
+
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
3
|
+
import { failureOf, shouldFallBack } from './failure.js';
|
|
4
|
+
export const RETRY_BASE_MS = 500;
|
|
5
|
+
export const RETRY_JITTER_MS = 250;
|
|
6
|
+
const USER_FIXABLE_KINDS = new Set(['not_configured', 'auth_failed', 'quota_exhausted']);
|
|
7
|
+
/** Only user-fixable failures keep the provider's message; everything else stays message-free. */
|
|
8
|
+
function detailFor(failure, message) {
|
|
9
|
+
return message && USER_FIXABLE_KINDS.has(failure.kind) ? message : undefined;
|
|
10
|
+
}
|
|
11
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
12
|
+
function retryDelay(deps) {
|
|
13
|
+
return RETRY_BASE_MS + Math.floor((deps.random ?? Math.random)() * RETRY_JITTER_MS);
|
|
14
|
+
}
|
|
15
|
+
function skipAttempt(backend, state) {
|
|
16
|
+
return {
|
|
17
|
+
backend,
|
|
18
|
+
outcome: 'skipped',
|
|
19
|
+
failure: state.failure,
|
|
20
|
+
skipReason: state.state,
|
|
21
|
+
...(state.state === 'cooling_down' ? { cooldownUntil: state.until } : {}),
|
|
22
|
+
...(state.detail ? { detail: state.detail } : {})
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function skipMessage(backend, state) {
|
|
26
|
+
const base = state.state === 'cooling_down'
|
|
27
|
+
? `${backend} is cooling down after ${state.failure.kind} until ${new Date(state.until).toISOString()}.`
|
|
28
|
+
: `${backend} is disabled for this session after ${state.failure.kind}.`;
|
|
29
|
+
return state.detail ? `${base} ${state.detail}` : base;
|
|
30
|
+
}
|
|
31
|
+
function failedAttempt(backend, failure, state, detail) {
|
|
32
|
+
return {
|
|
33
|
+
backend,
|
|
34
|
+
outcome: 'failed',
|
|
35
|
+
failure,
|
|
36
|
+
...(state.state === 'cooling_down' ? { cooldownUntil: state.until } : {}),
|
|
37
|
+
...(detail ? { detail } : {})
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* One provider under the #55 policy: skip when cooling down or disabled,
|
|
42
|
+
* retry exactly once on transient, record state. Never falls back itself.
|
|
43
|
+
*/
|
|
44
|
+
export function withSearchPolicy(name, search, deps, healthKey = name) {
|
|
45
|
+
return async (input) => {
|
|
46
|
+
const state = deps.health.get(healthKey);
|
|
47
|
+
if (state.state !== 'available') {
|
|
48
|
+
return {
|
|
49
|
+
status: 'error',
|
|
50
|
+
results: [],
|
|
51
|
+
metadata: { backend: name, cacheHit: false, attempts: [skipAttempt(name, state)] },
|
|
52
|
+
error: { code: 'BACKEND_UNAVAILABLE', message: skipMessage(name, state), failure: state.failure }
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const attempts = [];
|
|
56
|
+
let result = await search(input);
|
|
57
|
+
let failure = failureOf(result);
|
|
58
|
+
if (failure?.kind === 'transient') {
|
|
59
|
+
attempts.push({ backend: name, outcome: 'retried', failure });
|
|
60
|
+
await (deps.sleep ?? defaultSleep)(retryDelay(deps));
|
|
61
|
+
result = await search(input);
|
|
62
|
+
failure = failureOf(result);
|
|
63
|
+
}
|
|
64
|
+
if (failure) {
|
|
65
|
+
const detail = detailFor(failure, result.error?.message);
|
|
66
|
+
attempts.push(failedAttempt(name, failure, deps.health.record(healthKey, failure, detail), detail));
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
attempts.push({ backend: name, outcome: result.results.length > 0 ? 'results' : 'empty' });
|
|
70
|
+
}
|
|
71
|
+
return { ...result, metadata: { ...result.metadata, attempts: [...(result.metadata.attempts ?? []), ...attempts] } };
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function unavailableMessage(attempts, messages = new Map()) {
|
|
75
|
+
const byProvider = new Map();
|
|
76
|
+
for (const attempt of attempts) {
|
|
77
|
+
if (attempt.outcome === 'failed' || attempt.outcome === 'skipped')
|
|
78
|
+
byProvider.set(attempt.backend, attempt);
|
|
79
|
+
}
|
|
80
|
+
const entries = [...byProvider.values()].sort((a, b) => {
|
|
81
|
+
const aUntil = a.cooldownUntil ?? Number.POSITIVE_INFINITY;
|
|
82
|
+
const bUntil = b.cooldownUntil ?? Number.POSITIVE_INFINITY;
|
|
83
|
+
return aUntil - bUntil;
|
|
84
|
+
});
|
|
85
|
+
const parts = entries.map((attempt) => {
|
|
86
|
+
const kind = attempt.failure?.kind ?? 'bad_response';
|
|
87
|
+
if (attempt.cooldownUntil !== undefined) {
|
|
88
|
+
return `${attempt.backend} ${kind} (available again at ${new Date(attempt.cooldownUntil).toISOString()})`;
|
|
89
|
+
}
|
|
90
|
+
// Keep the provider's own hint for problems the user has to fix, e.g. a missing key or base URL.
|
|
91
|
+
const hint = USER_FIXABLE_KINDS.has(kind)
|
|
92
|
+
? attempt.detail ?? (attempt.outcome === 'failed' ? messages.get(attempt.backend) : undefined)
|
|
93
|
+
: undefined;
|
|
94
|
+
return hint ? `${attempt.backend} ${kind} (${hint})` : `${attempt.backend} ${kind}`;
|
|
95
|
+
});
|
|
96
|
+
return `No search backend is available: ${parts.join(', ')}.`;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* A failed link's unavailable providers, from its own attempts: the last failed or skipped
|
|
100
|
+
* attempt per backend with that attempt's kind. A fanout link reports each provider, not the
|
|
101
|
+
* aggregate. Falls back to the aggregate only when the link carries no failed or skipped attempts.
|
|
102
|
+
*/
|
|
103
|
+
function unavailableFor(result, failure) {
|
|
104
|
+
const message = result.error?.message ?? failure.kind;
|
|
105
|
+
const byProvider = new Map();
|
|
106
|
+
for (const entry of result.metadata.coverage?.unavailable ?? []) {
|
|
107
|
+
byProvider.set(entry.provider, { ...entry, message });
|
|
108
|
+
}
|
|
109
|
+
const attempts = result.metadata.attempts ?? [];
|
|
110
|
+
const relevant = attempts.filter((a) => a.outcome === 'failed' || a.outcome === 'skipped');
|
|
111
|
+
if (relevant.length === 0) {
|
|
112
|
+
byProvider.set(result.metadata.backend, { provider: result.metadata.backend, kind: failure.kind, message });
|
|
113
|
+
}
|
|
114
|
+
for (const attempt of relevant) {
|
|
115
|
+
byProvider.delete(attempt.backend); // keep insertion order at the latest attempt
|
|
116
|
+
byProvider.set(attempt.backend, {
|
|
117
|
+
provider: attempt.backend,
|
|
118
|
+
kind: attempt.failure?.kind ?? failure.kind,
|
|
119
|
+
message: attempt.detail ?? message
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return [...byProvider.values()];
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Tries providers in order under the precedence in the #55 spec: terminal and
|
|
126
|
+
* bad_request failures stop the chain; a result or a valid empty response is
|
|
127
|
+
* returned; everything else falls back.
|
|
128
|
+
*/
|
|
129
|
+
export function chainSearch(providers, deps) {
|
|
130
|
+
return async (input) => {
|
|
131
|
+
const attempts = [];
|
|
132
|
+
const unavailable = [];
|
|
133
|
+
let lastFailure;
|
|
134
|
+
let firstBackend;
|
|
135
|
+
// Keep an earlier fanout's provenance (which providers were tried) when a later link answers.
|
|
136
|
+
let fanout;
|
|
137
|
+
for (const provider of providers) {
|
|
138
|
+
const result = await provider(input);
|
|
139
|
+
firstBackend ??= result.metadata.backend;
|
|
140
|
+
attempts.push(...(result.metadata.attempts ?? []));
|
|
141
|
+
fanout ??= result.metadata.fanout;
|
|
142
|
+
const failure = failureOf(result);
|
|
143
|
+
if (!failure) {
|
|
144
|
+
const first = unavailable[0];
|
|
145
|
+
const merged = {
|
|
146
|
+
...result,
|
|
147
|
+
metadata: {
|
|
148
|
+
...result.metadata,
|
|
149
|
+
...(fanout && !result.metadata.fanout ? { fanout } : {}),
|
|
150
|
+
attempts,
|
|
151
|
+
...(first
|
|
152
|
+
? {
|
|
153
|
+
fallbackFrom: first.provider,
|
|
154
|
+
fallbackReason: first.message,
|
|
155
|
+
coverage: { partial: true, unavailable: unavailable.map(({ provider, kind }) => ({ provider, kind })) }
|
|
156
|
+
}
|
|
157
|
+
: {})
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
return { ...merged, presentation: buildSearchPresentation(merged) };
|
|
161
|
+
}
|
|
162
|
+
if (!shouldFallBack(failure.kind)) {
|
|
163
|
+
const stopped = { ...result, metadata: { ...result.metadata, attempts } };
|
|
164
|
+
return { ...stopped, presentation: buildSearchPresentation(stopped) };
|
|
165
|
+
}
|
|
166
|
+
for (const entry of unavailableFor(result, failure)) {
|
|
167
|
+
const existing = unavailable.findIndex((u) => u.provider === entry.provider);
|
|
168
|
+
if (existing >= 0)
|
|
169
|
+
unavailable.splice(existing, 1);
|
|
170
|
+
unavailable.push(entry);
|
|
171
|
+
}
|
|
172
|
+
lastFailure = failure;
|
|
173
|
+
}
|
|
174
|
+
const exhausted = {
|
|
175
|
+
status: 'error',
|
|
176
|
+
results: [],
|
|
177
|
+
metadata: { backend: firstBackend ?? 'duckduckgo', cacheHit: false, attempts },
|
|
178
|
+
error: {
|
|
179
|
+
code: 'SEARCH_BACKENDS_UNAVAILABLE',
|
|
180
|
+
message: unavailableMessage(attempts, new Map(unavailable.map((entry) => [entry.provider, entry.message]))),
|
|
181
|
+
failure: lastFailure ?? { kind: 'bad_response' }
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
return { ...exhausted, presentation: buildSearchPresentation(exhausted) };
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Firecrawl under the policy, with the optional HTTP fallback. The HTTP fetcher
|
|
189
|
+
* talks to the model-chosen site, not a service, so it has no provider health.
|
|
190
|
+
*/
|
|
191
|
+
export function withFetchPolicy(primary, fallback, deps, healthKey = 'firecrawl') {
|
|
192
|
+
const finish = (result) => ({ ...result, presentation: buildFetchPresentation(result) });
|
|
193
|
+
return async (input) => {
|
|
194
|
+
const attempts = [];
|
|
195
|
+
const state = deps.health.get(healthKey);
|
|
196
|
+
let first;
|
|
197
|
+
if (state.state !== 'available') {
|
|
198
|
+
attempts.push(skipAttempt('firecrawl', state));
|
|
199
|
+
first = {
|
|
200
|
+
status: 'error',
|
|
201
|
+
url: input.url,
|
|
202
|
+
metadata: { method: 'firecrawl', cacheHit: false },
|
|
203
|
+
error: { code: 'BACKEND_UNAVAILABLE', message: skipMessage('firecrawl', state), failure: state.failure }
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
else {
|
|
207
|
+
first = await primary(input);
|
|
208
|
+
let failure = failureOf(first);
|
|
209
|
+
if (failure?.kind === 'transient') {
|
|
210
|
+
attempts.push({ backend: 'firecrawl', outcome: 'retried', failure });
|
|
211
|
+
await (deps.sleep ?? defaultSleep)(retryDelay(deps));
|
|
212
|
+
first = await primary(input);
|
|
213
|
+
failure = failureOf(first);
|
|
214
|
+
}
|
|
215
|
+
if (failure) {
|
|
216
|
+
const detail = detailFor(failure, first.error?.message);
|
|
217
|
+
attempts.push(failedAttempt('firecrawl', failure, deps.health.record(healthKey, failure, detail), detail));
|
|
218
|
+
}
|
|
219
|
+
else
|
|
220
|
+
attempts.push({ backend: 'firecrawl', outcome: first.status === 'ok' ? 'results' : 'empty' });
|
|
221
|
+
}
|
|
222
|
+
const failure = failureOf(first);
|
|
223
|
+
const fallBack = first.status === 'needs_headless' || (failure !== undefined && shouldFallBack(failure.kind));
|
|
224
|
+
if (!fallback || !fallBack) {
|
|
225
|
+
return finish({ ...first, metadata: { ...first.metadata, attempts } });
|
|
226
|
+
}
|
|
227
|
+
const second = await fallback(input);
|
|
228
|
+
attempts.push({ backend: 'http', outcome: second.status === 'ok' ? 'results' : 'failed', ...(second.error?.failure ? { failure: second.error.failure } : {}) });
|
|
229
|
+
return finish({
|
|
230
|
+
...second,
|
|
231
|
+
metadata: {
|
|
232
|
+
...second.metadata,
|
|
233
|
+
attempts,
|
|
234
|
+
fallbackFrom: 'firecrawl',
|
|
235
|
+
fallbackReason: first.error?.message ?? 'Firecrawl fetch failed.'
|
|
236
|
+
}
|
|
237
|
+
});
|
|
238
|
+
};
|
|
239
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { FailureInfo, SearchProviderName } from '../types.js';
|
|
2
|
+
export type ClassifiedProvider = SearchProviderName | 'firecrawl';
|
|
3
|
+
export type ResponseParts = {
|
|
4
|
+
status: number;
|
|
5
|
+
headers: Headers;
|
|
6
|
+
/** Parsed JSON body, or undefined when the body was empty or not JSON. */
|
|
7
|
+
json?: unknown;
|
|
8
|
+
};
|
|
9
|
+
export declare class BodyReadError extends Error {
|
|
10
|
+
readonly cause: unknown;
|
|
11
|
+
constructor(cause: unknown);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Reads a response body once, keeping status and headers alongside the parsed JSON.
|
|
15
|
+
* A body that can't be read (connection dropped mid-stream) throws BodyReadError: that's
|
|
16
|
+
* a transport failure, not a malformed response.
|
|
17
|
+
*/
|
|
18
|
+
export declare function readResponseParts(response: Response): Promise<ResponseParts & {
|
|
19
|
+
text: string;
|
|
20
|
+
}>;
|
|
21
|
+
export declare function classifyHttpFailure(provider: ClassifiedProvider, parts: ResponseParts, now?: number): FailureInfo;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import { parseRetryAfter } from './failure.js';
|
|
2
|
+
export class BodyReadError extends Error {
|
|
3
|
+
cause;
|
|
4
|
+
constructor(cause) {
|
|
5
|
+
super(cause instanceof Error ? cause.message : String(cause));
|
|
6
|
+
this.cause = cause;
|
|
7
|
+
this.name = 'BodyReadError';
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Reads a response body once, keeping status and headers alongside the parsed JSON.
|
|
12
|
+
* A body that can't be read (connection dropped mid-stream) throws BodyReadError: that's
|
|
13
|
+
* a transport failure, not a malformed response.
|
|
14
|
+
*/
|
|
15
|
+
export async function readResponseParts(response) {
|
|
16
|
+
let text;
|
|
17
|
+
try {
|
|
18
|
+
text = await response.text();
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
throw new BodyReadError(error);
|
|
22
|
+
}
|
|
23
|
+
let json;
|
|
24
|
+
try {
|
|
25
|
+
json = text ? JSON.parse(text) : undefined;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
json = undefined;
|
|
29
|
+
}
|
|
30
|
+
return { status: response.status, headers: response.headers, json, text };
|
|
31
|
+
}
|
|
32
|
+
// Source: https://exa.ai/docs/reference/error-codes
|
|
33
|
+
const EXA_TAGS = {
|
|
34
|
+
RATE_LIMIT_EXCEEDED: 'rate_limited',
|
|
35
|
+
NO_MORE_CREDITS: 'quota_exhausted',
|
|
36
|
+
API_KEY_BUDGET_EXCEEDED: 'quota_exhausted',
|
|
37
|
+
TEAM_BUDGET_EXCEEDED: 'quota_exhausted',
|
|
38
|
+
INVALID_API_KEY: 'auth_failed',
|
|
39
|
+
FEATURE_DISABLED: 'auth_failed',
|
|
40
|
+
PROHIBITED_CONTENT: 'bad_request',
|
|
41
|
+
CONTENT_FILTER_ERROR: 'bad_request',
|
|
42
|
+
INVALID_REQUEST_BODY: 'bad_request',
|
|
43
|
+
INVALID_REQUEST: 'bad_request',
|
|
44
|
+
INVALID_NUM_RESULTS: 'bad_request'
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* Conservative defaults for undocumented responses. A 401 disables the provider
|
|
48
|
+
* (low risk: resets on config change); an undocumented 403 is treated as a bot
|
|
49
|
+
* wall; an undocumented 402 is not assumed to mean quota.
|
|
50
|
+
*/
|
|
51
|
+
function defaultKind(status) {
|
|
52
|
+
if (status === 429)
|
|
53
|
+
return 'rate_limited';
|
|
54
|
+
if (status === 401)
|
|
55
|
+
return 'auth_failed';
|
|
56
|
+
if (status === 403)
|
|
57
|
+
return 'blocked';
|
|
58
|
+
if (status === 400 || status === 422)
|
|
59
|
+
return 'bad_request';
|
|
60
|
+
if (status === 408 || status >= 500)
|
|
61
|
+
return 'transient';
|
|
62
|
+
return 'bad_response';
|
|
63
|
+
}
|
|
64
|
+
function stringField(json, field) {
|
|
65
|
+
if (!json || typeof json !== 'object')
|
|
66
|
+
return undefined;
|
|
67
|
+
const value = json[field];
|
|
68
|
+
return typeof value === 'string' ? value : undefined;
|
|
69
|
+
}
|
|
70
|
+
export function classifyHttpFailure(provider, parts, now = Date.now()) {
|
|
71
|
+
const { status } = parts;
|
|
72
|
+
let kind;
|
|
73
|
+
let providerCode;
|
|
74
|
+
if (provider === 'exa') {
|
|
75
|
+
const tag = stringField(parts.json, 'tag');
|
|
76
|
+
if (tag && EXA_TAGS[tag]) {
|
|
77
|
+
kind = EXA_TAGS[tag];
|
|
78
|
+
providerCode = tag;
|
|
79
|
+
}
|
|
80
|
+
else if (status === 402) {
|
|
81
|
+
kind = 'quota_exhausted'; // documented 402 meaning for Exa
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
else if (provider === 'firecrawl') {
|
|
85
|
+
// Source: https://docs.firecrawl.dev/api-reference/errors
|
|
86
|
+
if (status === 402)
|
|
87
|
+
kind = 'quota_exhausted';
|
|
88
|
+
}
|
|
89
|
+
else if (provider === 'youcom') {
|
|
90
|
+
// Source: https://you.com/docs/api-reference/search/v1-search (429 UNVERIFIED -> default)
|
|
91
|
+
// Documented for /v1/search; the client currently calls /v1/agents/search (#60).
|
|
92
|
+
if (status === 402)
|
|
93
|
+
kind = 'quota_exhausted';
|
|
94
|
+
else if (status === 403)
|
|
95
|
+
kind = 'auth_failed';
|
|
96
|
+
}
|
|
97
|
+
else if (provider === 'searxng') {
|
|
98
|
+
// Source: https://docs.searxng.org/dev/search_api.html (403 = format=json disabled in settings)
|
|
99
|
+
if (status === 403)
|
|
100
|
+
kind = 'auth_failed';
|
|
101
|
+
}
|
|
102
|
+
// brave, tavily, duckduckgo: defaults only (UNVERIFIED beyond 429; see research gate).
|
|
103
|
+
const resolved = kind ?? defaultKind(status);
|
|
104
|
+
const info = { kind: resolved, httpStatus: status, ...(providerCode ? { providerCode } : {}) };
|
|
105
|
+
if (resolved === 'rate_limited') {
|
|
106
|
+
const retryAfter = parseRetryAfter(parts.headers.get('retry-after'), now);
|
|
107
|
+
if (retryAfter !== undefined)
|
|
108
|
+
info.providerRetryAfterMs = retryAfter;
|
|
109
|
+
}
|
|
110
|
+
return info;
|
|
111
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { FailureInfo } from '../types.js';
|
|
2
|
+
export declare const DEFAULT_COOLDOWN_MS = 60000;
|
|
3
|
+
export declare const MIN_COOLDOWN_MS = 1000;
|
|
4
|
+
export declare const MAX_COOLDOWN_MS: number;
|
|
5
|
+
export type ProviderHealthState = {
|
|
6
|
+
state: 'available';
|
|
7
|
+
} | {
|
|
8
|
+
state: 'cooling_down';
|
|
9
|
+
until: number;
|
|
10
|
+
failure: FailureInfo;
|
|
11
|
+
detail?: string;
|
|
12
|
+
} | {
|
|
13
|
+
state: 'disabled';
|
|
14
|
+
failure: FailureInfo;
|
|
15
|
+
detail?: string;
|
|
16
|
+
};
|
|
17
|
+
export type ProviderHealth = {
|
|
18
|
+
get(key: string): ProviderHealthState;
|
|
19
|
+
record(key: string, failure: FailureInfo, detail?: string): ProviderHealthState;
|
|
20
|
+
};
|
|
21
|
+
/** The applied cooldown. The provider's own value stays in failure.providerRetryAfterMs. */
|
|
22
|
+
export declare function cooldownFor(failure: FailureInfo): number;
|
|
23
|
+
/**
|
|
24
|
+
* Per backend set. The set is rebuilt on any effective config change, so fixed
|
|
25
|
+
* credentials or new endpoints take effect without restarting Pi.
|
|
26
|
+
*/
|
|
27
|
+
export declare function createProviderHealth({ now }?: {
|
|
28
|
+
now?: () => number;
|
|
29
|
+
}): ProviderHealth;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export const DEFAULT_COOLDOWN_MS = 60_000;
|
|
2
|
+
export const MIN_COOLDOWN_MS = 1000;
|
|
3
|
+
export const MAX_COOLDOWN_MS = 15 * 60_000;
|
|
4
|
+
/** The applied cooldown. The provider's own value stays in failure.providerRetryAfterMs. */
|
|
5
|
+
export function cooldownFor(failure) {
|
|
6
|
+
const raw = failure.providerRetryAfterMs;
|
|
7
|
+
if (raw === undefined || Number.isNaN(raw))
|
|
8
|
+
return DEFAULT_COOLDOWN_MS;
|
|
9
|
+
return Math.min(MAX_COOLDOWN_MS, Math.max(MIN_COOLDOWN_MS, raw));
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Per backend set. The set is rebuilt on any effective config change, so fixed
|
|
13
|
+
* credentials or new endpoints take effect without restarting Pi.
|
|
14
|
+
*/
|
|
15
|
+
export function createProviderHealth({ now = Date.now } = {}) {
|
|
16
|
+
const states = new Map();
|
|
17
|
+
function get(key) {
|
|
18
|
+
const current = states.get(key);
|
|
19
|
+
if (!current)
|
|
20
|
+
return { state: 'available' };
|
|
21
|
+
if (current.state === 'cooling_down' && current.until <= now()) {
|
|
22
|
+
states.delete(key);
|
|
23
|
+
return { state: 'available' };
|
|
24
|
+
}
|
|
25
|
+
return current;
|
|
26
|
+
}
|
|
27
|
+
function record(key, failure, detail) {
|
|
28
|
+
const current = get(key);
|
|
29
|
+
if (current.state === 'disabled')
|
|
30
|
+
return current;
|
|
31
|
+
switch (failure.kind) {
|
|
32
|
+
case 'rate_limited': {
|
|
33
|
+
const next = { state: 'cooling_down', until: now() + cooldownFor(failure), failure };
|
|
34
|
+
states.set(key, next);
|
|
35
|
+
return next;
|
|
36
|
+
}
|
|
37
|
+
case 'quota_exhausted':
|
|
38
|
+
case 'auth_failed':
|
|
39
|
+
case 'not_configured': {
|
|
40
|
+
const next = { state: 'disabled', failure, ...(detail ? { detail } : {}) };
|
|
41
|
+
states.set(key, next);
|
|
42
|
+
return next;
|
|
43
|
+
}
|
|
44
|
+
default:
|
|
45
|
+
return current;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return { get, record };
|
|
49
|
+
}
|
|
@@ -36,7 +36,20 @@ export declare function validateBackendUrl(value: string): {
|
|
|
36
36
|
ok: false;
|
|
37
37
|
message: string;
|
|
38
38
|
};
|
|
39
|
-
export declare function
|
|
39
|
+
export declare function validateAllowRanges(value: string): {
|
|
40
|
+
ok: true;
|
|
41
|
+
value: string;
|
|
42
|
+
} | {
|
|
43
|
+
ok: false;
|
|
44
|
+
message: string;
|
|
45
|
+
};
|
|
46
|
+
export declare function createBackendUrlEditor(theme: any, label: string, placeholderUrl: string, onOpenChange?: (open: boolean) => void, validate?: (value: string) => {
|
|
47
|
+
ok: true;
|
|
48
|
+
value: string;
|
|
49
|
+
} | {
|
|
50
|
+
ok: false;
|
|
51
|
+
message: string;
|
|
52
|
+
}): (currentValue: string, done: (selectedValue?: string) => void) => Component;
|
|
40
53
|
export declare function getInheritedConfigForScope(loaded: Awaited<LoadedPresentationConfig>, scope: PresentationScope): PresentationConfig;
|
|
41
54
|
export declare function getScopeDisplayConfig(loaded: Awaited<LoadedPresentationConfig>, scope: PresentationScope): PresentationConfig;
|
|
42
55
|
export declare function getInheritedBackendsForScope(loaded: Awaited<LoadedPresentationConfig>, scope: PresentationScope): BackendConfig;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DEFAULT_BACKEND_CONFIG, isValidProxyUrl, mergeBackendConfigLayers, stripProxyCredentials, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
|
|
2
2
|
import { checkBackendHealth } from '../backends/doctor.js';
|
|
3
|
+
import { parseCidr } from '../fetch/network-guard.js';
|
|
3
4
|
import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
|
|
4
5
|
import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
|
|
5
6
|
import { DEFAULT_PRESENTATION_CONFIG, mergePresentationConfigLayers, resolvePresentationMode } from '../presentation/config.js';
|
|
@@ -32,7 +33,10 @@ function cloneBackendConfig(config) {
|
|
|
32
33
|
options: config.fetch.options ? { ...config.fetch.options } : undefined
|
|
33
34
|
},
|
|
34
35
|
headless: { ...config.headless },
|
|
35
|
-
proxy: config.proxy ? { ...config.proxy } : undefined
|
|
36
|
+
proxy: config.proxy ? { ...config.proxy } : undefined,
|
|
37
|
+
network: config.network
|
|
38
|
+
? { ...config.network, ...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {}) }
|
|
39
|
+
: undefined
|
|
36
40
|
};
|
|
37
41
|
}
|
|
38
42
|
function sameJson(left, right) {
|
|
@@ -50,6 +54,22 @@ export function validateBackendUrl(value) {
|
|
|
50
54
|
return { ok: false, message: 'Invalid URL. Include http:// or https://.' };
|
|
51
55
|
}
|
|
52
56
|
}
|
|
57
|
+
function splitRanges(value) {
|
|
58
|
+
return value
|
|
59
|
+
.split(',')
|
|
60
|
+
.map((range) => range.trim())
|
|
61
|
+
.filter(Boolean);
|
|
62
|
+
}
|
|
63
|
+
export function validateAllowRanges(value) {
|
|
64
|
+
for (const range of splitRanges(value)) {
|
|
65
|
+
const cidr = parseCidr(range);
|
|
66
|
+
if (!cidr)
|
|
67
|
+
return { ok: false, message: `Not a valid CIDR range: ${range}` };
|
|
68
|
+
if (cidr.prefix === 0)
|
|
69
|
+
return { ok: false, message: `${range} allows every address. List specific ranges instead.` };
|
|
70
|
+
}
|
|
71
|
+
return { ok: true, value: splitRanges(value).join(', ') };
|
|
72
|
+
}
|
|
53
73
|
/**
|
|
54
74
|
* The extension re-applies this patch on load, so `pending` here means the
|
|
55
75
|
* patch could not be written (read-only install, permissions, or a dependency
|
|
@@ -140,7 +160,7 @@ function buildPresentationSettingsItems(scope, config) {
|
|
|
140
160
|
}))
|
|
141
161
|
];
|
|
142
162
|
}
|
|
143
|
-
export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChange) {
|
|
163
|
+
export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChange, validate = validateBackendUrl) {
|
|
144
164
|
return (currentValue, done) => {
|
|
145
165
|
onOpenChange?.(true);
|
|
146
166
|
const initialValue = currentValue && currentValue !== 'not set' ? currentValue : placeholderUrl;
|
|
@@ -164,7 +184,7 @@ export function createBackendUrlEditor(theme, label, placeholderUrl, onOpenChang
|
|
|
164
184
|
finish('');
|
|
165
185
|
return;
|
|
166
186
|
}
|
|
167
|
-
const validated =
|
|
187
|
+
const validated = validate(value);
|
|
168
188
|
if (!validated.ok) {
|
|
169
189
|
showError(validated.message);
|
|
170
190
|
return;
|
|
@@ -276,6 +296,20 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
|
|
|
276
296
|
label: 'Proxy URL',
|
|
277
297
|
currentValue: backends.proxy ? stripProxyCredentials(backends.proxy.url) : 'not set',
|
|
278
298
|
submenu: createBackendUrlEditor(theme, 'HTTP proxy URL', 'http://127.0.0.1:7890', onUrlEditorOpenChange)
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
id: 'backend:network:allowRanges',
|
|
302
|
+
label: 'Network allow list',
|
|
303
|
+
currentValue: backends.network?.allowRanges?.length ? backends.network.allowRanges.join(', ') : 'not set',
|
|
304
|
+
submenu: createBackendUrlEditor(theme, 'Private ranges to allow, comma separated CIDRs, e.g. 198.18.0.0/15',
|
|
305
|
+
// No prefilled example: pressing enter on an empty list must not quietly add an exception.
|
|
306
|
+
'', onUrlEditorOpenChange, validateAllowRanges)
|
|
307
|
+
},
|
|
308
|
+
{
|
|
309
|
+
id: 'backend:network:trustProxyDns',
|
|
310
|
+
label: 'Trust the upstream proxy to enforce private-address restrictions',
|
|
311
|
+
currentValue: backends.network?.trustProxyDns ? 'on' : 'off',
|
|
312
|
+
values: ['off', 'on']
|
|
279
313
|
}
|
|
280
314
|
];
|
|
281
315
|
}
|
|
@@ -456,6 +490,25 @@ export function applySettingsValue(state, id, newValue) {
|
|
|
456
490
|
delete currentBackends.proxy;
|
|
457
491
|
}
|
|
458
492
|
}
|
|
493
|
+
if (id === 'backend:network:allowRanges') {
|
|
494
|
+
const ranges = splitRanges(newValue);
|
|
495
|
+
const next = { ...currentBackends.network };
|
|
496
|
+
if (ranges.length > 0) {
|
|
497
|
+
next.allowRanges = ranges;
|
|
498
|
+
}
|
|
499
|
+
else {
|
|
500
|
+
delete next.allowRanges;
|
|
501
|
+
}
|
|
502
|
+
if (Object.keys(next).length > 0) {
|
|
503
|
+
currentBackends.network = next;
|
|
504
|
+
}
|
|
505
|
+
else {
|
|
506
|
+
delete currentBackends.network;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
if (id === 'backend:network:trustProxyDns') {
|
|
510
|
+
currentBackends.network = { ...currentBackends.network, trustProxyDns: newValue === 'on' };
|
|
511
|
+
}
|
|
459
512
|
nextDrafts[nextScope] = currentDraft;
|
|
460
513
|
nextBackendDrafts[nextScope] = currentBackends;
|
|
461
514
|
return {
|
|
@@ -531,6 +584,23 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
|
|
|
531
584
|
override.proxy = { url: '' };
|
|
532
585
|
}
|
|
533
586
|
}
|
|
587
|
+
if (!sameJson(config.network, inheritedConfig.network)) {
|
|
588
|
+
if (config.network) {
|
|
589
|
+
override.network = {
|
|
590
|
+
...config.network,
|
|
591
|
+
...(config.network.allowRanges ? { allowRanges: [...config.network.allowRanges] } : {})
|
|
592
|
+
};
|
|
593
|
+
// Cleared at this scope: an explicit empty list overrides the parent's.
|
|
594
|
+
if (!config.network.allowRanges && inheritedConfig.network?.allowRanges)
|
|
595
|
+
override.network.allowRanges = [];
|
|
596
|
+
}
|
|
597
|
+
else if (inheritedConfig.network) {
|
|
598
|
+
override.network = {
|
|
599
|
+
...(inheritedConfig.network.allowRanges ? { allowRanges: [] } : {}),
|
|
600
|
+
...(inheritedConfig.network.trustProxyDns ? { trustProxyDns: false } : {})
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
}
|
|
534
604
|
return override;
|
|
535
605
|
}
|
|
536
606
|
export function handleSettingsShortcut(data) {
|
|
@@ -753,6 +823,8 @@ export function registerWebAgentConfigCommands(pi, deps = {}) {
|
|
|
753
823
|
`typebox: ${typeboxOk ? 'ok' : 'missing'}`,
|
|
754
824
|
formatJitiCompatLine(checkCompat()),
|
|
755
825
|
formatBackendSummary(backendConfig),
|
|
826
|
+
`network allow list: ${backendConfig.network?.allowRanges?.length ? backendConfig.network.allowRanges.join(', ') : 'none'}`,
|
|
827
|
+
`trust upstream proxy for private addresses: ${backendConfig.network?.trustProxyDns ? 'on' : 'off'}`,
|
|
756
828
|
backendIssues.length > 0 ? `backend config: warning\n${backendIssues.join('\n')}` : 'backend config: ok',
|
|
757
829
|
...backendHealth
|
|
758
830
|
];
|
package/dist/extension.js
CHANGED
|
@@ -59,20 +59,64 @@ export default function extension(pi) {
|
|
|
59
59
|
registerWebAgentConfigCommands(pi);
|
|
60
60
|
const injectedWebExplore = pi.__webExploreTool;
|
|
61
61
|
let cachedBackendKey;
|
|
62
|
+
let cachedWorkflow;
|
|
62
63
|
let cachedWebExplore;
|
|
64
|
+
// Runs in flight per workflow, so a replaced workflow is never closed mid-run.
|
|
65
|
+
const activeRuns = new Map();
|
|
66
|
+
const retiring = new Set();
|
|
67
|
+
const closeWorkflow = (workflow) => {
|
|
68
|
+
retiring.delete(workflow);
|
|
69
|
+
void Promise.resolve(workflow.close?.()).catch(() => undefined);
|
|
70
|
+
};
|
|
71
|
+
const retire = (workflow) => {
|
|
72
|
+
if ((activeRuns.get(workflow) ?? 0) === 0) {
|
|
73
|
+
closeWorkflow(workflow);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
retiring.add(workflow);
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const leased = (workflow) => ({
|
|
80
|
+
run: async (input) => {
|
|
81
|
+
activeRuns.set(workflow, (activeRuns.get(workflow) ?? 0) + 1);
|
|
82
|
+
try {
|
|
83
|
+
return await workflow.run(input);
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
const remaining = (activeRuns.get(workflow) ?? 1) - 1;
|
|
87
|
+
if (remaining > 0) {
|
|
88
|
+
activeRuns.set(workflow, remaining);
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
activeRuns.delete(workflow);
|
|
92
|
+
if (retiring.has(workflow))
|
|
93
|
+
closeWorkflow(workflow);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
});
|
|
63
98
|
async function getConfiguredWebExplore() {
|
|
64
99
|
if (injectedWebExplore)
|
|
65
100
|
return injectedWebExplore;
|
|
66
101
|
const backendConfig = await getEffectiveBackendConfig(pi);
|
|
67
102
|
const backendKey = JSON.stringify(backendConfig);
|
|
68
103
|
if (!cachedWebExplore || cachedBackendKey !== backendKey) {
|
|
104
|
+
if (cachedWorkflow)
|
|
105
|
+
retire(cachedWorkflow);
|
|
69
106
|
cachedBackendKey = backendKey;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
});
|
|
107
|
+
cachedWorkflow = createResearchWorkflow({ backendConfig });
|
|
108
|
+
cachedWebExplore = createWebExploreTool({ explore: leased(cachedWorkflow) });
|
|
73
109
|
}
|
|
74
110
|
return cachedWebExplore;
|
|
75
111
|
}
|
|
112
|
+
pi.on('session_shutdown', async () => {
|
|
113
|
+
const workflow = cachedWorkflow;
|
|
114
|
+
cachedWorkflow = undefined;
|
|
115
|
+
cachedWebExplore = undefined;
|
|
116
|
+
cachedBackendKey = undefined;
|
|
117
|
+
if (workflow)
|
|
118
|
+
retire(workflow);
|
|
119
|
+
});
|
|
76
120
|
pi.on('session_start', async (_event, ctx) => {
|
|
77
121
|
try {
|
|
78
122
|
const notice = await getUpdateChangelogNotice();
|