@demigodmode/pi-web-agent 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +82 -79
- package/dist/backends/config.d.ts +27 -0
- package/dist/backends/config.js +97 -1
- package/dist/backends/factory.d.ts +2 -1
- package/dist/backends/factory.js +92 -23
- package/dist/commands/web-agent-config.d.ts +3 -0
- package/dist/commands/web-agent-config.js +57 -5
- package/dist/extension.d.ts +1 -0
- package/dist/extension.js +2 -0
- package/dist/fetch/headless-fetch.d.ts +8 -1
- package/dist/fetch/headless-fetch.js +3 -3
- package/dist/fetch/proxy-fetch.d.ts +22 -0
- package/dist/fetch/proxy-fetch.js +46 -0
- package/dist/jiti-compat-run.d.ts +1 -0
- package/dist/jiti-compat-run.js +9 -0
- package/dist/jiti-compat.d.ts +32 -0
- package/dist/jiti-compat.js +215 -0
- package/dist/presentation/config-store.js +4 -0
- package/dist/readers/youtube-reader.d.ts +3 -1
- package/dist/readers/youtube-reader.js +11 -3
- package/dist/search/duckduckgo.d.ts +10 -1
- package/dist/search/duckduckgo.js +23 -5
- package/dist/search/tavily.d.ts +2 -1
- package/dist/search/tavily.js +5 -3
- package/dist/tools/web-search.js +21 -9
- package/dist/types.d.ts +1 -1
- package/package.json +2 -1
- package/scripts/patch-jiti-compat.mjs +52 -8
package/dist/backends/factory.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { createFirecrawlFetcher } from '../fetch/firecrawl-fetch.js';
|
|
2
|
+
import { createHttpFetcher } from '../fetch/http-fetch.js';
|
|
3
|
+
import { createProxyFetch, resolveProxyCredentials } from '../fetch/proxy-fetch.js';
|
|
4
|
+
import { headlessFetch } from '../fetch/headless-fetch.js';
|
|
2
5
|
import { createBraveSearchTool } from '../search/brave.js';
|
|
3
6
|
import { createYouComSearchTool } from '../search/youcom.js';
|
|
7
|
+
import { fetchDuckDuckGoHtml } from '../search/duckduckgo.js';
|
|
4
8
|
import { createExaSearchTool } from '../search/exa.js';
|
|
5
9
|
import { createTavilySearchTool } from '../search/tavily.js';
|
|
6
10
|
import { createSearxngSearchTool } from '../search/searxng.js';
|
|
@@ -10,7 +14,7 @@ import { buildSearchPresentation } from '../presentation/search-presentation.js'
|
|
|
10
14
|
import { createWebFetchHeadlessTool } from '../tools/web-fetch-headless.js';
|
|
11
15
|
import { createWebFetchTool } from '../tools/web-fetch.js';
|
|
12
16
|
import { createWebSearchTool } from '../tools/web-search.js';
|
|
13
|
-
import { DEFAULT_BACKEND_CONFIG, usableSearchProviders } from './config.js';
|
|
17
|
+
import { DEFAULT_BACKEND_CONFIG, isValidProxyUrl, stripProxyCredentials, usableSearchProviders } from './config.js';
|
|
14
18
|
import { createSpecialContentResolver } from '../readers/resolver.js';
|
|
15
19
|
import { createGithubReader } from '../readers/github-reader.js';
|
|
16
20
|
import { createPdfReader } from '../readers/pdf-reader.js';
|
|
@@ -54,7 +58,10 @@ function withSearchFallback(primary, fallback, fallbackFrom) {
|
|
|
54
58
|
metadata: {
|
|
55
59
|
...second.metadata,
|
|
56
60
|
fallbackFrom,
|
|
57
|
-
fallbackReason: first.error?.message ?? `${fallbackFrom} search failed
|
|
61
|
+
fallbackReason: first.error?.message ?? `${fallbackFrom} search failed.`,
|
|
62
|
+
// Keep the primary's fanout provenance (which providers were tried/skipped) even though
|
|
63
|
+
// the answer came from the fallback backend.
|
|
64
|
+
...(first.metadata.fanout ? { fanout: first.metadata.fanout } : {})
|
|
58
65
|
}
|
|
59
66
|
};
|
|
60
67
|
return { ...result, presentation: buildSearchPresentation(result) };
|
|
@@ -87,52 +94,104 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
87
94
|
const createHttpFetch = deps.createHttpFetch ?? createWebFetchTool;
|
|
88
95
|
const createFirecrawlFetch = deps.createFirecrawlFetch ?? createFirecrawlFetcher;
|
|
89
96
|
const createHeadlessFetch = deps.createHeadlessFetch ?? createWebFetchHeadlessTool;
|
|
97
|
+
const makeProxyFetch = deps.createProxyFetch ?? createProxyFetch;
|
|
98
|
+
// A blank url is the "disable proxy" marker: treat it as no proxy at all.
|
|
99
|
+
const proxy = config.proxy && config.proxy.url.trim() !== '' ? config.proxy : undefined;
|
|
100
|
+
// A configured proxy whose url fails validation must not be silently ignored
|
|
101
|
+
// — that would send traffic direct to the websites. Every request errors out
|
|
102
|
+
// instead. No connectivity check is needed: the url itself is the problem.
|
|
103
|
+
if (proxy && !isValidProxyUrl(proxy.url)) {
|
|
104
|
+
const message = `backends.proxy.url (${proxy.url}) is not a valid http or https URL. ` +
|
|
105
|
+
'Web requests are blocked until it is fixed; set backends.proxy.url to "" to disable the proxy.';
|
|
106
|
+
return {
|
|
107
|
+
search: async () => {
|
|
108
|
+
const result = {
|
|
109
|
+
status: 'error',
|
|
110
|
+
results: [],
|
|
111
|
+
metadata: { backend: config.search.provider, cacheHit: false },
|
|
112
|
+
error: { code: 'BACKEND_CONFIG_INVALID', message }
|
|
113
|
+
};
|
|
114
|
+
return { ...result, presentation: buildSearchPresentation(result) };
|
|
115
|
+
},
|
|
116
|
+
fetchPage: async ({ url }) => {
|
|
117
|
+
const result = {
|
|
118
|
+
status: 'error',
|
|
119
|
+
url,
|
|
120
|
+
metadata: { method: 'http', cacheHit: false },
|
|
121
|
+
error: { code: 'BACKEND_CONFIG_INVALID', message }
|
|
122
|
+
};
|
|
123
|
+
return { ...result, presentation: buildFetchPresentation(result) };
|
|
124
|
+
},
|
|
125
|
+
headlessFetch: async ({ url }) => {
|
|
126
|
+
const result = {
|
|
127
|
+
status: 'error',
|
|
128
|
+
url,
|
|
129
|
+
metadata: { method: 'headless', cacheHit: false },
|
|
130
|
+
error: { code: 'BACKEND_CONFIG_INVALID', message }
|
|
131
|
+
};
|
|
132
|
+
return { ...result, presentation: buildFetchPresentation(result) };
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
// When a proxy is configured, every outbound HTTP request (search, fetch,
|
|
137
|
+
// readers, and doctor-style checks) goes through it; headless browser
|
|
138
|
+
// traffic gets the same proxy via Playwright launch options.
|
|
139
|
+
const fetchImpl = proxy ? makeProxyFetch(proxy) : fetch;
|
|
140
|
+
const proxyCredentials = proxy ? resolveProxyCredentials(proxy) : undefined;
|
|
141
|
+
const proxyBrowserOptions = proxy
|
|
142
|
+
? {
|
|
143
|
+
server: stripProxyCredentials(proxy.url),
|
|
144
|
+
...(proxyCredentials?.username !== undefined ? { username: proxyCredentials.username } : {}),
|
|
145
|
+
...(proxyCredentials?.password !== undefined ? { password: proxyCredentials.password } : {})
|
|
146
|
+
}
|
|
147
|
+
: undefined;
|
|
148
|
+
const createDuckDuckGo = () => createDuckDuckGoSearch({ searchHtml: (query) => fetchDuckDuckGoHtml(query, { fetchImpl }) });
|
|
90
149
|
function buildProviderSearch(name) {
|
|
91
150
|
switch (name) {
|
|
92
151
|
case 'searxng':
|
|
93
152
|
return config.search.baseUrl
|
|
94
|
-
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
153
|
+
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options, fetchImpl })
|
|
95
154
|
: invalidSearxngSearch();
|
|
96
155
|
case 'brave':
|
|
97
|
-
return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY });
|
|
156
|
+
return createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY, fetchImpl });
|
|
98
157
|
case 'youcom':
|
|
99
|
-
return createYouComSearch({ apiKey: process.env.YDC_API_KEY });
|
|
158
|
+
return createYouComSearch({ apiKey: process.env.YDC_API_KEY, fetchImpl });
|
|
100
159
|
case 'exa':
|
|
101
|
-
return createExaSearch({ apiKey: process.env.EXA_API_KEY });
|
|
160
|
+
return createExaSearch({ apiKey: process.env.EXA_API_KEY, fetchImpl });
|
|
102
161
|
case 'tavily':
|
|
103
|
-
return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY });
|
|
162
|
+
return createTavilySearch({ apiKey: process.env.TAVILY_API_KEY, fetchImpl });
|
|
104
163
|
case 'duckduckgo':
|
|
105
164
|
default:
|
|
106
|
-
return
|
|
165
|
+
return createDuckDuckGo();
|
|
107
166
|
}
|
|
108
167
|
}
|
|
109
168
|
let search = config.search.provider === 'searxng'
|
|
110
169
|
? config.search.baseUrl
|
|
111
|
-
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options })
|
|
170
|
+
? createSearxngSearch({ baseUrl: config.search.baseUrl, options: config.search.options, fetchImpl })
|
|
112
171
|
: invalidSearxngSearch()
|
|
113
172
|
: config.search.provider === 'brave'
|
|
114
|
-
? createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY })
|
|
173
|
+
? createBraveSearch({ apiKey: process.env.PI_WEB_AGENT_BRAVE_API_KEY, fetchImpl })
|
|
115
174
|
: config.search.provider === 'youcom'
|
|
116
|
-
? createYouComSearch({ apiKey: process.env.YDC_API_KEY })
|
|
175
|
+
? createYouComSearch({ apiKey: process.env.YDC_API_KEY, fetchImpl })
|
|
117
176
|
: config.search.provider === 'exa'
|
|
118
|
-
? createExaSearch({ apiKey: process.env.EXA_API_KEY })
|
|
177
|
+
? createExaSearch({ apiKey: process.env.EXA_API_KEY, fetchImpl })
|
|
119
178
|
: config.search.provider === 'tavily'
|
|
120
|
-
? createTavilySearch({ apiKey: process.env.TAVILY_API_KEY })
|
|
121
|
-
:
|
|
179
|
+
? createTavilySearch({ apiKey: process.env.TAVILY_API_KEY, fetchImpl })
|
|
180
|
+
: createDuckDuckGo();
|
|
122
181
|
if (config.search.provider === 'searxng' && config.search.fallback === 'duckduckgo') {
|
|
123
|
-
search = withSearchFallback(search,
|
|
182
|
+
search = withSearchFallback(search, createDuckDuckGo(), 'searxng');
|
|
124
183
|
}
|
|
125
184
|
if (config.search.provider === 'brave' && config.search.fallback === 'duckduckgo') {
|
|
126
|
-
search = withSearchFallback(search,
|
|
185
|
+
search = withSearchFallback(search, createDuckDuckGo(), 'brave');
|
|
127
186
|
}
|
|
128
187
|
if (config.search.provider === 'youcom' && config.search.fallback === 'duckduckgo') {
|
|
129
|
-
search = withSearchFallback(search,
|
|
188
|
+
search = withSearchFallback(search, createDuckDuckGo(), 'youcom');
|
|
130
189
|
}
|
|
131
190
|
if (config.search.provider === 'exa' && config.search.fallback === 'duckduckgo') {
|
|
132
|
-
search = withSearchFallback(search,
|
|
191
|
+
search = withSearchFallback(search, createDuckDuckGo(), 'exa');
|
|
133
192
|
}
|
|
134
193
|
if (config.search.provider === 'tavily' && config.search.fallback === 'duckduckgo') {
|
|
135
|
-
search = withSearchFallback(search,
|
|
194
|
+
search = withSearchFallback(search, createDuckDuckGo(), 'tavily');
|
|
136
195
|
}
|
|
137
196
|
const fanoutConfig = config.search.fanout;
|
|
138
197
|
if (fanoutConfig && fanoutConfig.mode !== 'off') {
|
|
@@ -149,14 +208,23 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
149
208
|
mode: fanoutConfig.mode
|
|
150
209
|
});
|
|
151
210
|
}
|
|
152
|
-
|
|
211
|
+
// Keep the keyless Tavily safety net for the no-key DuckDuckGo default, even under fanout —
|
|
212
|
+
// it wraps whatever search ended up being (plain DDG or the fanout set) so a total failure
|
|
213
|
+
// still has somewhere to go. Opt out with PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK=1.
|
|
214
|
+
const keylessFallbackDisabled = process.env.PI_WEB_AGENT_DISABLE_KEYLESS_FALLBACK === '1';
|
|
215
|
+
const usingDuckDuckGoDefault = config.search.provider === 'duckduckgo' || !config.search.provider;
|
|
216
|
+
if (usingDuckDuckGoDefault && !keylessFallbackDisabled) {
|
|
217
|
+
search = withSearchFallback(search, createTavilySearch({ keyless: true, fetchImpl }), 'duckduckgo');
|
|
218
|
+
}
|
|
219
|
+
const httpFetch = createHttpFetch({ fetchPage: createHttpFetcher({ fetchImpl }) });
|
|
153
220
|
let fetchPage = config.fetch.provider === 'firecrawl'
|
|
154
221
|
? config.fetch.baseUrl
|
|
155
222
|
? createHttpFetch({
|
|
156
223
|
fetchPage: createFirecrawlFetch({
|
|
157
224
|
baseUrl: config.fetch.baseUrl,
|
|
158
225
|
apiKey: config.fetch.apiKey ?? process.env.PI_WEB_AGENT_FIRECRAWL_API_KEY,
|
|
159
|
-
options: config.fetch.options
|
|
226
|
+
options: config.fetch.options,
|
|
227
|
+
fetchImpl
|
|
160
228
|
})
|
|
161
229
|
})
|
|
162
230
|
: createHttpFetch({ fetchPage: invalidFirecrawlFetch() })
|
|
@@ -165,12 +233,13 @@ export function createBackendSet(config = DEFAULT_BACKEND_CONFIG, deps = {}) {
|
|
|
165
233
|
fetchPage = withFetchFallback(fetchPage, httpFetch);
|
|
166
234
|
}
|
|
167
235
|
const fetchPageWithReaders = createSpecialContentResolver({
|
|
168
|
-
readers: [createGithubReader(), createPdfReader(), createYoutubeReader()],
|
|
236
|
+
readers: [createGithubReader({ fetchImpl }), createPdfReader({ fetchImpl }), createYoutubeReader({ fetchImpl })],
|
|
169
237
|
fallback: fetchPage
|
|
170
238
|
});
|
|
239
|
+
const headlessPage = (url) => proxyBrowserOptions ? headlessFetch(url, { proxy: proxyBrowserOptions }) : headlessFetch(url);
|
|
171
240
|
return {
|
|
172
241
|
search,
|
|
173
242
|
fetchPage: fetchPageWithReaders,
|
|
174
|
-
headlessFetch: createHeadlessFetch()
|
|
243
|
+
headlessFetch: createHeadlessFetch({ fetchPage: headlessPage })
|
|
175
244
|
};
|
|
176
245
|
}
|
|
@@ -16,6 +16,9 @@ type CommandDeps = {
|
|
|
16
16
|
arch: string;
|
|
17
17
|
};
|
|
18
18
|
checkTypebox?: () => Promise<boolean>;
|
|
19
|
+
checkJitiCompat?: () => {
|
|
20
|
+
pending: string[];
|
|
21
|
+
};
|
|
19
22
|
checkBackends?: (config: BackendConfig) => Promise<string[]>;
|
|
20
23
|
getChangelog?: () => Promise<string | undefined>;
|
|
21
24
|
};
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import { DEFAULT_BACKEND_CONFIG, mergeBackendConfigLayers, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
|
|
1
|
+
import { DEFAULT_BACKEND_CONFIG, isValidProxyUrl, mergeBackendConfigLayers, stripProxyCredentials, validateBackendConfig, usableSearchProviders } from '../backends/config.js';
|
|
2
2
|
import { checkBackendHealth } from '../backends/doctor.js';
|
|
3
3
|
import { DynamicBorder, getSettingsListTheme } from '@earendil-works/pi-coding-agent';
|
|
4
4
|
import { Container, Input, SelectList, SettingsList, Text } from '@earendil-works/pi-tui';
|
|
5
5
|
import { DEFAULT_PRESENTATION_CONFIG, mergePresentationConfigLayers, resolvePresentationMode } from '../presentation/config.js';
|
|
6
6
|
import { loadPresentationConfigLayers, resetPresentationConfigScope, saveBackendConfigScope, savePresentationConfigScope } from '../presentation/config-store.js';
|
|
7
7
|
import { resolveBrowserExecutable } from '../fetch/browser-resolution.js';
|
|
8
|
+
import { createProxyFetch } from '../fetch/proxy-fetch.js';
|
|
9
|
+
import { checkJitiCompat } from '../jiti-compat.js';
|
|
8
10
|
import { getLatestChangelogEntry } from '../changelog-notice.js';
|
|
9
11
|
const PRESENTATION_TOOL_NAMES = ['web_explore'];
|
|
10
12
|
function parseScopeToken(token) {
|
|
@@ -29,7 +31,8 @@ function cloneBackendConfig(config) {
|
|
|
29
31
|
...config.fetch,
|
|
30
32
|
options: config.fetch.options ? { ...config.fetch.options } : undefined
|
|
31
33
|
},
|
|
32
|
-
headless: { ...config.headless }
|
|
34
|
+
headless: { ...config.headless },
|
|
35
|
+
proxy: config.proxy ? { ...config.proxy } : undefined
|
|
33
36
|
};
|
|
34
37
|
}
|
|
35
38
|
function sameJson(left, right) {
|
|
@@ -47,6 +50,21 @@ export function validateBackendUrl(value) {
|
|
|
47
50
|
return { ok: false, message: 'Invalid URL. Include http:// or https://.' };
|
|
48
51
|
}
|
|
49
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* The extension re-applies this patch on load, so `pending` here means the
|
|
55
|
+
* patch could not be written (read-only install, permissions, or a dependency
|
|
56
|
+
* whose shape changed). Point at the manual script in that case: it is the
|
|
57
|
+
* same recovery step people have been finding by digging through issue #34.
|
|
58
|
+
*/
|
|
59
|
+
function formatJitiCompatLine(status) {
|
|
60
|
+
if (status.pending.length === 0)
|
|
61
|
+
return 'jsdom compat patch: ok';
|
|
62
|
+
return [
|
|
63
|
+
`jsdom compat patch: needed (${status.pending.join(', ')})`,
|
|
64
|
+
'Run: node ~/.pi/agent/npm/node_modules/@demigodmode/pi-web-agent/scripts/patch-jiti-compat.mjs',
|
|
65
|
+
'then restart Pi.'
|
|
66
|
+
].join('\n');
|
|
67
|
+
}
|
|
50
68
|
async function defaultCheckTypebox() {
|
|
51
69
|
try {
|
|
52
70
|
await import('typebox');
|
|
@@ -87,8 +105,11 @@ function formatBackendSummary(config = DEFAULT_BACKEND_CONFIG) {
|
|
|
87
105
|
return [
|
|
88
106
|
searchSuffix ? `${searchBase} ${searchSuffix}` : searchBase,
|
|
89
107
|
fetchSuffix ? `${fetchBase} ${fetchSuffix}` : fetchBase,
|
|
90
|
-
`headless: ${config.headless.provider}
|
|
91
|
-
|
|
108
|
+
`headless: ${config.headless.provider}`,
|
|
109
|
+
config.proxy ? `proxy: ${stripProxyCredentials(config.proxy.url)}` : undefined
|
|
110
|
+
]
|
|
111
|
+
.filter(Boolean)
|
|
112
|
+
.join('\n');
|
|
92
113
|
}
|
|
93
114
|
function formatConfigSummary(config) {
|
|
94
115
|
const lines = [`defaultMode: ${config.defaultMode}`];
|
|
@@ -249,6 +270,12 @@ function buildBackendSettingsItems(scope, backends, theme, onUrlEditorOpenChange
|
|
|
249
270
|
label: 'Firecrawl API key',
|
|
250
271
|
currentValue: 'env var',
|
|
251
272
|
values: ['env var']
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
id: 'backend:proxy:url',
|
|
276
|
+
label: 'Proxy URL',
|
|
277
|
+
currentValue: backends.proxy ? stripProxyCredentials(backends.proxy.url) : 'not set',
|
|
278
|
+
submenu: createBackendUrlEditor(theme, 'HTTP proxy URL', 'http://127.0.0.1:7890', onUrlEditorOpenChange)
|
|
252
279
|
}
|
|
253
280
|
];
|
|
254
281
|
}
|
|
@@ -421,6 +448,14 @@ export function applySettingsValue(state, id, newValue) {
|
|
|
421
448
|
delete currentBackends.fetch.baseUrl;
|
|
422
449
|
}
|
|
423
450
|
}
|
|
451
|
+
if (id === 'backend:proxy:url') {
|
|
452
|
+
if (newValue.trim()) {
|
|
453
|
+
currentBackends.proxy = { ...currentBackends.proxy, url: newValue.trim() };
|
|
454
|
+
}
|
|
455
|
+
else {
|
|
456
|
+
delete currentBackends.proxy;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
424
459
|
nextDrafts[nextScope] = currentDraft;
|
|
425
460
|
nextBackendDrafts[nextScope] = currentBackends;
|
|
426
461
|
return {
|
|
@@ -485,6 +520,17 @@ export function collapseBackendConfigToOverride(config, inheritedConfig) {
|
|
|
485
520
|
if (!sameJson(config.headless, inheritedConfig.headless)) {
|
|
486
521
|
override.headless = { ...config.headless };
|
|
487
522
|
}
|
|
523
|
+
if (!sameJson(config.proxy, inheritedConfig.proxy)) {
|
|
524
|
+
if (config.proxy) {
|
|
525
|
+
const { password: _password, ...proxy } = config.proxy;
|
|
526
|
+
override.proxy = { ...proxy };
|
|
527
|
+
}
|
|
528
|
+
else if (inheritedConfig.proxy) {
|
|
529
|
+
// The proxy was cleared at this scope; record an explicit disable so it
|
|
530
|
+
// overrides a proxy set in a lower (e.g. global) layer.
|
|
531
|
+
override.proxy = { url: '' };
|
|
532
|
+
}
|
|
533
|
+
}
|
|
488
534
|
return override;
|
|
489
535
|
}
|
|
490
536
|
export function handleSettingsShortcut(data) {
|
|
@@ -669,7 +715,12 @@ export function registerWebAgentConfigCommands(pi, deps = {}) {
|
|
|
669
715
|
arch: process.arch
|
|
670
716
|
};
|
|
671
717
|
const checkTypebox = deps.checkTypebox ?? defaultCheckTypebox;
|
|
672
|
-
const
|
|
718
|
+
const checkCompat = deps.checkJitiCompat ?? checkJitiCompat;
|
|
719
|
+
const checkBackends = deps.checkBackends ?? ((config) => checkBackendHealth(config, {
|
|
720
|
+
// An invalid proxy url is reported by validateBackendConfig below; never
|
|
721
|
+
// build a proxy agent from it (no connectivity check is attempted).
|
|
722
|
+
fetchImpl: config.proxy && isValidProxyUrl(config.proxy.url) ? createProxyFetch(config.proxy) : fetch
|
|
723
|
+
}));
|
|
673
724
|
const getChangelog = deps.getChangelog ?? (() => getLatestChangelogEntry());
|
|
674
725
|
pi.registerCommand('web-agent', {
|
|
675
726
|
description: 'Open settings or manage pi-web-agent presentation config',
|
|
@@ -700,6 +751,7 @@ export function registerWebAgentConfigCommands(pi, deps = {}) {
|
|
|
700
751
|
'pi-web-agent: loaded',
|
|
701
752
|
`runtime: node ${runtime.nodeVersion} ${runtime.platform} ${runtime.arch}`,
|
|
702
753
|
`typebox: ${typeboxOk ? 'ok' : 'missing'}`,
|
|
754
|
+
formatJitiCompatLine(checkCompat()),
|
|
703
755
|
formatBackendSummary(backendConfig),
|
|
704
756
|
backendIssues.length > 0 ? `backend config: warning\n${backendIssues.join('\n')}` : 'backend config: ok',
|
|
705
757
|
...backendHealth
|
package/dist/extension.d.ts
CHANGED
package/dist/extension.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
// Keep first: re-applies the jsdom/jiti compat patch before jsdom loads (#34).
|
|
2
|
+
import './jiti-compat-run.js';
|
|
1
3
|
import { DEFAULT_BACKEND_CONFIG } from './backends/config.js';
|
|
2
4
|
import { Type } from 'typebox';
|
|
3
5
|
import { registerWebAgentConfigCommands } from './commands/web-agent-config.js';
|
|
@@ -1,13 +1,20 @@
|
|
|
1
1
|
import { type BrowserResolutionResult } from './browser-resolution.js';
|
|
2
2
|
import type { WebFetchHeadlessResponse } from '../types.js';
|
|
3
|
-
export
|
|
3
|
+
export type BrowserProxyOptions = {
|
|
4
|
+
server: string;
|
|
5
|
+
username?: string;
|
|
6
|
+
password?: string;
|
|
7
|
+
};
|
|
8
|
+
export declare function headlessFetch(url: string, { configuredPath, proxy, resolveBrowser, launchBrowser, now }?: {
|
|
4
9
|
configuredPath?: string;
|
|
10
|
+
proxy?: BrowserProxyOptions;
|
|
5
11
|
resolveBrowser?: (options?: {
|
|
6
12
|
configuredPath?: string;
|
|
7
13
|
}) => Promise<BrowserResolutionResult>;
|
|
8
14
|
launchBrowser?: (options: {
|
|
9
15
|
executablePath?: string;
|
|
10
16
|
headless: true;
|
|
17
|
+
proxy?: BrowserProxyOptions;
|
|
11
18
|
}) => Promise<{
|
|
12
19
|
newContext: () => Promise<{
|
|
13
20
|
newPage: () => Promise<any>;
|
|
@@ -7,7 +7,7 @@ function cleanupRenderedText(text) {
|
|
|
7
7
|
cleaned = cleaned.replace(/\s+/g, ' ').trim();
|
|
8
8
|
return cleaned;
|
|
9
9
|
}
|
|
10
|
-
export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless }) => chromium.launch(executablePath ? { executablePath, headless } : { headless }), now = () => Date.now() } = {}) {
|
|
10
|
+
export async function headlessFetch(url, { configuredPath, proxy, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless, proxy }) => chromium.launch(executablePath ? { executablePath, headless, ...(proxy ? { proxy } : {}) } : { headless, ...(proxy ? { proxy } : {}) }), now = () => Date.now() } = {}) {
|
|
11
11
|
const resolved = await resolveBrowser({ configuredPath });
|
|
12
12
|
if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
|
|
13
13
|
return {
|
|
@@ -19,8 +19,8 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
|
|
|
19
19
|
}
|
|
20
20
|
const browserName = resolved.ok ? resolved.browser : 'chromium';
|
|
21
21
|
const launchOptions = resolved.ok
|
|
22
|
-
? { executablePath: resolved.executablePath, headless: true }
|
|
23
|
-
: { headless: true };
|
|
22
|
+
? { executablePath: resolved.executablePath, headless: true, ...(proxy ? { proxy } : {}) }
|
|
23
|
+
: { headless: true, ...(proxy ? { proxy } : {}) };
|
|
24
24
|
let browser;
|
|
25
25
|
let context;
|
|
26
26
|
let page;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type ProxyConfig } from '../backends/config.js';
|
|
2
|
+
export type ProxyCredentials = {
|
|
3
|
+
username?: string;
|
|
4
|
+
password?: string;
|
|
5
|
+
};
|
|
6
|
+
export type ProxyFetchOptions = {
|
|
7
|
+
/** TLS options for the tunneled connection to the origin (https targets). */
|
|
8
|
+
tls?: {
|
|
9
|
+
rejectUnauthorized?: boolean;
|
|
10
|
+
};
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Resolve proxy credentials from the config, falling back to environment
|
|
14
|
+
* variables so secrets can stay out of config files (same policy as API keys).
|
|
15
|
+
*/
|
|
16
|
+
export declare function resolveProxyCredentials(proxy: ProxyConfig, env?: NodeJS.ProcessEnv): ProxyCredentials;
|
|
17
|
+
/**
|
|
18
|
+
* Build a fetch implementation that routes all traffic through the configured
|
|
19
|
+
* HTTP/HTTPS proxy. Uses undici (the engine behind Node's global fetch) with a
|
|
20
|
+
* ProxyAgent dispatcher while keeping the standard fetch API surface.
|
|
21
|
+
*/
|
|
22
|
+
export declare function createProxyFetch(proxy: ProxyConfig, options?: ProxyFetchOptions): typeof fetch;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { fetch as undiciFetch, ProxyAgent } from 'undici';
|
|
2
|
+
import { stripProxyCredentials } from '../backends/config.js';
|
|
3
|
+
/**
|
|
4
|
+
* Resolve proxy credentials from the config, falling back to environment
|
|
5
|
+
* variables so secrets can stay out of config files (same policy as API keys).
|
|
6
|
+
*/
|
|
7
|
+
export function resolveProxyCredentials(proxy, env = process.env) {
|
|
8
|
+
const username = proxy.username ?? env.PI_WEB_AGENT_PROXY_USERNAME;
|
|
9
|
+
const password = proxy.password ?? env.PI_WEB_AGENT_PROXY_PASSWORD;
|
|
10
|
+
return {
|
|
11
|
+
username: username?.trim() ? username : undefined,
|
|
12
|
+
password: password ? password : undefined
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Build a fetch implementation that routes all traffic through the configured
|
|
17
|
+
* HTTP/HTTPS proxy. Uses undici (the engine behind Node's global fetch) with a
|
|
18
|
+
* ProxyAgent dispatcher while keeping the standard fetch API surface.
|
|
19
|
+
*/
|
|
20
|
+
export function createProxyFetch(proxy, options = {}) {
|
|
21
|
+
const credentials = resolveProxyCredentials(proxy);
|
|
22
|
+
const agent = new ProxyAgent({
|
|
23
|
+
// Never trust credentials embedded in the URL; they come from config fields
|
|
24
|
+
// or the PI_WEB_AGENT_PROXY_* env vars via resolveProxyCredentials.
|
|
25
|
+
uri: stripProxyCredentials(proxy.url),
|
|
26
|
+
// undici's `token` option is used verbatim as the Proxy-Authorization header
|
|
27
|
+
// value for both CONNECT tunnels and forwarded HTTP requests.
|
|
28
|
+
...(credentials.username !== undefined
|
|
29
|
+
? {
|
|
30
|
+
token: `Basic ${Buffer.from(credentials.password !== undefined
|
|
31
|
+
? `${credentials.username}:${credentials.password}`
|
|
32
|
+
: `${credentials.username}:`).toString('base64')}`
|
|
33
|
+
}
|
|
34
|
+
: {}),
|
|
35
|
+
...(options.tls ? { requestTls: { rejectUnauthorized: options.tls.rejectUnauthorized } } : {})
|
|
36
|
+
});
|
|
37
|
+
// undici's Request/Response types are structurally near-identical to Node's
|
|
38
|
+
// global fetch types but not nominatively identical, so the wrapper is cast
|
|
39
|
+
// to the standard fetch signature (runtime behavior is identical: Node's
|
|
40
|
+
// global fetch is built on this same undici engine).
|
|
41
|
+
const proxyFetch = (input, init) => undiciFetch(input, {
|
|
42
|
+
...init,
|
|
43
|
+
dispatcher: agent
|
|
44
|
+
});
|
|
45
|
+
return proxyFetch;
|
|
46
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { ensureJitiCompat } from './jiti-compat.js';
|
|
2
|
+
// Side-effect-only module. It exists so `extension.ts` can run the compat
|
|
3
|
+
// patch as its *first* import: ESM evaluates a module's dependency subtree,
|
|
4
|
+
// including that module's body, before moving on to the next import. A plain
|
|
5
|
+
// `ensureJitiCompat()` call in the extension body would run too late, after
|
|
6
|
+
// jsdom (and therefore tr46/cssstyle) had already been evaluated.
|
|
7
|
+
//
|
|
8
|
+
// Keep this import first in extension.ts.
|
|
9
|
+
ensureJitiCompat();
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filesystem/resolver seam so tests can point this at a throwaway directory
|
|
3
|
+
* instead of the real `node_modules/tr46` and `node_modules/cssstyle`. All
|
|
4
|
+
* production call sites use the defaults (real `fs` + real module
|
|
5
|
+
* resolution) and never pass this in.
|
|
6
|
+
*/
|
|
7
|
+
export type JitiCompatDeps = {
|
|
8
|
+
/**
|
|
9
|
+
* Resolve `specifier` as `fromFile` would. Omitting `fromFile` resolves from
|
|
10
|
+
* this module, which is only correct when nothing else claims the package.
|
|
11
|
+
*/
|
|
12
|
+
resolve: (specifier: string, fromFile?: string) => string;
|
|
13
|
+
existsSync: (path: string) => boolean;
|
|
14
|
+
readFileSync: (path: string) => string;
|
|
15
|
+
writeFileSync: (path: string, contents: string) => void;
|
|
16
|
+
};
|
|
17
|
+
export type JitiCompatStatus = {
|
|
18
|
+
/** Files that still need patching. Empty means the tree is healthy. */
|
|
19
|
+
pending: string[];
|
|
20
|
+
/** Files patched during this call. */
|
|
21
|
+
patched: string[];
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Apply both patches if they are missing. Safe to call repeatedly: an already
|
|
25
|
+
* patched tree is a few `readFileSync` calls and no writes. Never throws.
|
|
26
|
+
*/
|
|
27
|
+
export declare function ensureJitiCompat(deps?: JitiCompatDeps): JitiCompatStatus;
|
|
28
|
+
/**
|
|
29
|
+
* Read-only view of the same checks, for `/web-agent doctor`. Never throws and
|
|
30
|
+
* never writes.
|
|
31
|
+
*/
|
|
32
|
+
export declare function checkJitiCompat(deps?: JitiCompatDeps): JitiCompatStatus;
|