@openlines/opl-pi-sht 0.2.12 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -85
- package/configs/opl-browser.json.sample +3 -2
- package/configs/opl-footer.json.sample +3 -2
- package/configs/opl-guardian.json.sample +8 -2
- package/configs/opl-modes.json.sample +5 -0
- package/configs/opl-webaccess.json.sample +7 -1
- package/extensions/opl-browser/README.md +81 -55
- package/extensions/opl-browser/browser.ts +244 -50
- package/extensions/opl-browser/config.ts +15 -7
- package/extensions/opl-browser/index.ts +5 -4
- package/extensions/opl-browser/package-lock.json +3 -3
- package/extensions/opl-browser/package.json +4 -4
- package/extensions/opl-browser/validate.ts +313 -60
- package/extensions/opl-configurator/README.md +40 -0
- package/extensions/opl-configurator/configurator.ts +151 -0
- package/extensions/opl-configurator/index.ts +12 -0
- package/extensions/opl-ctxtrim/README.md +45 -14
- package/extensions/opl-ctxtrim/index.ts +33 -13
- package/extensions/opl-footer/README.md +31 -6
- package/extensions/opl-footer/config.ts +11 -10
- package/extensions/opl-footer/configure.ts +159 -115
- package/extensions/opl-footer/git-status.ts +58 -1
- package/extensions/opl-footer/icons.ts +3 -0
- package/extensions/opl-footer/index.ts +209 -105
- package/extensions/opl-footer/segments/caveman.ts +9 -1
- package/extensions/opl-footer/segments/chat-mode.ts +8 -1
- package/extensions/opl-footer/segments/compactions.ts +13 -0
- package/extensions/opl-footer/segments/context.ts +6 -5
- package/extensions/opl-footer/segments/index.ts +2 -0
- package/extensions/opl-footer/segments/mode-switcher.ts +14 -1
- package/extensions/opl-footer/segments/path.ts +1 -1
- package/extensions/opl-footer/segments/plan-mode.ts +8 -1
- package/extensions/opl-footer/theme.ts +5 -2
- package/extensions/opl-footer/types.ts +4 -0
- package/extensions/opl-guardian/README.md +17 -5
- package/extensions/opl-guardian/config.ts +72 -6
- package/extensions/opl-guardian/guardian.ts +42 -10
- package/extensions/opl-guardian/index.ts +52 -17
- package/extensions/opl-guardian/policies.ts +52 -16
- package/extensions/opl-init/README.md +12 -4
- package/extensions/opl-init/index.ts +228 -27
- package/extensions/opl-input/README.md +17 -3
- package/extensions/opl-input/config.ts +59 -5
- package/extensions/opl-input/configure.ts +146 -0
- package/extensions/opl-input/index.ts +38 -30
- package/extensions/opl-input/mode-style.ts +6 -2
- package/extensions/opl-modes/README.md +22 -8
- package/extensions/opl-modes/config.ts +62 -8
- package/extensions/opl-modes/index.ts +87 -28
- package/extensions/opl-modes/state.ts +50 -2
- package/extensions/opl-modes/types.ts +6 -0
- package/extensions/opl-modes/utils.ts +18 -8
- package/extensions/opl-questionnaire/index.ts +10 -4
- package/extensions/opl-questionnaire/types.ts +4 -0
- package/extensions/opl-simplebench/README.md +21 -15
- package/extensions/opl-simplebench/artifact.ts +7 -4
- package/extensions/opl-simplebench/benchmark.ts +96 -35
- package/extensions/opl-simplebench/coding.ts +9 -4
- package/extensions/opl-simplebench/index.ts +19 -19
- package/extensions/opl-simplebench/llama-server.ts +3 -2
- package/extensions/opl-simplebench/report.ts +0 -6
- package/extensions/opl-simplebench/research.ts +5 -4
- package/extensions/opl-simplebench/scoring.ts +4 -2
- package/extensions/opl-simplebench/util/config.ts +15 -8
- package/extensions/opl-simplebench/util/exec-env.ts +36 -0
- package/extensions/opl-simplebench/util/format.ts +2 -2
- package/extensions/opl-simplebench/util/providers.ts +23 -194
- package/extensions/opl-todo/README.md +5 -5
- package/extensions/opl-todo/config.ts +7 -2
- package/extensions/opl-todo/index.ts +50 -9
- package/extensions/opl-webaccess/README.md +31 -5
- package/extensions/opl-webaccess/config.ts +36 -7
- package/extensions/opl-webaccess/extract.ts +37 -47
- package/extensions/opl-webaccess/http.ts +259 -0
- package/extensions/opl-webaccess/index.ts +12 -6
- package/extensions/opl-webaccess/package-lock.json +3 -3
- package/extensions/opl-webaccess/package.json +4 -9
- package/extensions/opl-webaccess/pdf.ts +4 -4
- package/extensions/opl-webaccess/search.ts +19 -8
- package/extensions/opl-webaccess/storage.ts +134 -2
- package/extensions/opl-webaccess/types.ts +4 -0
- package/extensions/opl-webaccess/utils.ts +221 -33
- package/package.json +19 -12
|
@@ -2,11 +2,16 @@ import { Readability } from "@mozilla/readability";
|
|
|
2
2
|
import { parseHTML } from "linkedom";
|
|
3
3
|
import TurndownService from "turndown";
|
|
4
4
|
import { extractPdfBuffer } from "./pdf.js";
|
|
5
|
-
import { errorMessage, isAbortError, isPdfUrl, isPdfContentType,
|
|
5
|
+
import { errorMessage, isAbortError, isPdfUrl, isPdfContentType, resolveSafeHostUrl, type HttpUrlOptions } from "./utils.js";
|
|
6
|
+
import { pinnedFetch, isRedirectStatus, type PinnedResponse } from "./http.js";
|
|
6
7
|
import type { ExtractedContent } from "./types.js";
|
|
7
8
|
|
|
8
9
|
const CONCURRENT_LIMIT = 3;
|
|
9
|
-
const MAX_FETCH_BYTES = 10 * 1024 * 1024; //
|
|
10
|
+
const MAX_FETCH_BYTES = 10 * 1024 * 1024; // default response cap, overridable by maxResponseBytes
|
|
11
|
+
const FETCH_HEADERS = {
|
|
12
|
+
"User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
|
|
13
|
+
Accept: "text/html,application/xhtml+xml,application/pdf,*/*",
|
|
14
|
+
};
|
|
10
15
|
const FETCH_TIMEOUT_MS = 30_000;
|
|
11
16
|
const MAX_REDIRECTS = 5;
|
|
12
17
|
/** Ceiling on the number of URLs a single fetch_content call may include. */
|
|
@@ -17,33 +22,6 @@ const td = new TurndownService({
|
|
|
17
22
|
codeBlockStyle: "fenced",
|
|
18
23
|
});
|
|
19
24
|
|
|
20
|
-
/** Read a response body into bytes, aborting past maxBytes (guards memory against huge payloads). */
|
|
21
|
-
async function readBufferCapped(response: Response, maxBytes: number): Promise<ArrayBuffer> {
|
|
22
|
-
const declared = Number(response.headers.get("content-length") ?? "0");
|
|
23
|
-
if (declared > maxBytes) throw new Error(`Response too large (${declared} bytes)`);
|
|
24
|
-
const reader = response.body?.getReader();
|
|
25
|
-
if (!reader) return await response.arrayBuffer();
|
|
26
|
-
const chunks: Uint8Array[] = [];
|
|
27
|
-
let total = 0;
|
|
28
|
-
for (;;) {
|
|
29
|
-
const { done, value } = await reader.read();
|
|
30
|
-
if (done) break;
|
|
31
|
-
total += value.byteLength;
|
|
32
|
-
if (total > maxBytes) {
|
|
33
|
-
await reader.cancel().catch(() => {});
|
|
34
|
-
throw new Error(`Response exceeds ${maxBytes} byte limit`);
|
|
35
|
-
}
|
|
36
|
-
chunks.push(value);
|
|
37
|
-
}
|
|
38
|
-
const merged = new Uint8Array(total);
|
|
39
|
-
let offset = 0;
|
|
40
|
-
for (const chunk of chunks) {
|
|
41
|
-
merged.set(chunk, offset);
|
|
42
|
-
offset += chunk.byteLength;
|
|
43
|
-
}
|
|
44
|
-
return merged.buffer;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
25
|
export async function fetchAllContent(
|
|
48
26
|
urls: string[],
|
|
49
27
|
signal?: AbortSignal,
|
|
@@ -62,36 +40,47 @@ export async function fetchAllContent(
|
|
|
62
40
|
}
|
|
63
41
|
|
|
64
42
|
/**
|
|
65
|
-
* Fetch with redirects followed manually so each hop is re-validated
|
|
66
|
-
* `redirect: "follow"` would let a public URL bounce to a private
|
|
67
|
-
* past the
|
|
43
|
+
* Fetch with redirects followed manually so each hop is re-validated, re-resolved **and
|
|
44
|
+
* re-pinned**. The native `redirect: "follow"` would let a public URL bounce to a private or
|
|
45
|
+
* link-local host past the guard; a hostname that merely reads public can answer with an
|
|
46
|
+
* internal address; and a record with a short TTL can answer one address at check time and
|
|
47
|
+
* another at connect time. So every hop goes through resolveSafeHostUrl and the socket is
|
|
48
|
+
* pinned to the addresses that hop approved.
|
|
68
49
|
*/
|
|
69
50
|
async function fetchWithRedirectValidation(
|
|
70
51
|
url: string,
|
|
71
52
|
fetchSignal: AbortSignal,
|
|
72
53
|
opts: HttpUrlOptions
|
|
73
|
-
): Promise<
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
"User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
|
|
77
|
-
Accept: "text/html,application/xhtml+xml,application/pdf,*/*",
|
|
78
|
-
};
|
|
54
|
+
): Promise<PinnedResponse> {
|
|
55
|
+
const maxResponseBytes = opts.maxResponseBytes ?? MAX_FETCH_BYTES;
|
|
56
|
+
let current = url;
|
|
79
57
|
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
80
|
-
const
|
|
81
|
-
|
|
58
|
+
const target = await resolveSafeHostUrl(current, { ...opts, signal: fetchSignal });
|
|
59
|
+
const request = {
|
|
60
|
+
signal: fetchSignal,
|
|
61
|
+
headers: FETCH_HEADERS,
|
|
62
|
+
maxResponseBytes,
|
|
63
|
+
timeoutMs: opts.timeoutMs,
|
|
64
|
+
};
|
|
65
|
+
const response = await (opts.transport ?? pinnedFetch)(target, request);
|
|
66
|
+
if (!isRedirectStatus(response.status)) return response;
|
|
82
67
|
const location = response.headers.get("location");
|
|
83
68
|
if (!location) return response; // 3xx without a Location: treat as final
|
|
84
|
-
|
|
85
|
-
current = assertHttpUrl(new URL(location, current).href, opts);
|
|
69
|
+
current = new URL(location, target.url).href;
|
|
86
70
|
}
|
|
87
71
|
throw new Error(`Too many redirects (max ${MAX_REDIRECTS})`);
|
|
88
72
|
}
|
|
89
73
|
|
|
90
74
|
async function fetchOne(url: string, signal?: AbortSignal, opts: HttpUrlOptions = {}): Promise<ExtractedContent> {
|
|
75
|
+
const deadline = new AbortController();
|
|
76
|
+
const budget = opts.timeoutMs ?? FETCH_TIMEOUT_MS;
|
|
77
|
+
// Own this timer across all hops. Bun cancels AbortSignal.timeout when its last listener
|
|
78
|
+
// is removed, so transport cleanup between literal-IP hops must not cancel the deadline.
|
|
79
|
+
const timer = setTimeout(() => deadline.abort(), budget);
|
|
91
80
|
try {
|
|
92
81
|
const fetchSignal = signal
|
|
93
|
-
? AbortSignal.any([signal,
|
|
94
|
-
:
|
|
82
|
+
? AbortSignal.any([signal, deadline.signal])
|
|
83
|
+
: deadline.signal;
|
|
95
84
|
const response = await fetchWithRedirectValidation(url, fetchSignal, opts);
|
|
96
85
|
|
|
97
86
|
if (!response.ok) {
|
|
@@ -101,15 +90,14 @@ async function fetchOne(url: string, signal?: AbortSignal, opts: HttpUrlOptions
|
|
|
101
90
|
const contentType = response.headers.get("content-type") ?? "";
|
|
102
91
|
|
|
103
92
|
if (isPdfUrl(url) || isPdfContentType(contentType)) {
|
|
104
|
-
const
|
|
105
|
-
const content = await extractPdfBuffer(buffer);
|
|
93
|
+
const content = await extractPdfBuffer(response.body);
|
|
106
94
|
return { url, title: url, content, error: null };
|
|
107
95
|
}
|
|
108
96
|
|
|
109
97
|
const isHtml =
|
|
110
98
|
contentType.includes("text/html") || contentType.includes("application/xhtml");
|
|
111
99
|
|
|
112
|
-
const body =
|
|
100
|
+
const body = response.text();
|
|
113
101
|
|
|
114
102
|
if (!isHtml) {
|
|
115
103
|
// Plain text, markdown, JSON, etc. — return as-is
|
|
@@ -137,5 +125,7 @@ async function fetchOne(url: string, signal?: AbortSignal, opts: HttpUrlOptions
|
|
|
137
125
|
} catch (err) {
|
|
138
126
|
if (isAbortError(err)) return { url, title: "", content: "", error: "Aborted" };
|
|
139
127
|
return { url, title: "", content: "", error: errorMessage(err) };
|
|
128
|
+
} finally {
|
|
129
|
+
clearTimeout(timer);
|
|
140
130
|
}
|
|
141
131
|
}
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pinned HTTP transport for model-supplied URLs.
|
|
3
|
+
*
|
|
4
|
+
* The guard in utils.ts classifies every DNS answer for a hostname. Handing only the *name* to a
|
|
5
|
+
* generic client throws that work away: the client resolves again, and a record with a one-second
|
|
6
|
+
* TTL can answer a public address at check time and 127.0.0.1 or 169.254.169.254 at connect time.
|
|
7
|
+
* This module takes the validated set and dials it, so the socket cannot be re-steered.
|
|
8
|
+
*
|
|
9
|
+
* Node's global fetch has no resolver hook, so the transport is node:http/node:https with a
|
|
10
|
+
* `lookup` that never reaches DNS. The hostname still drives TLS (`servername`) and the `Host`
|
|
11
|
+
* header, so certificate validation and virtual hosting behave as they would without pinning.
|
|
12
|
+
*/
|
|
13
|
+
import http from "node:http";
|
|
14
|
+
import https from "node:https";
|
|
15
|
+
import type { LookupAddress, LookupOptions } from "node:dns";
|
|
16
|
+
import { createGunzip, createInflate, createBrotliDecompress } from "node:zlib";
|
|
17
|
+
import type { Transform } from "node:stream";
|
|
18
|
+
import type { SafeHttpTarget } from "./utils.ts";
|
|
19
|
+
|
|
20
|
+
/** Keeps site responses identical to the fetch path this replaces. */
|
|
21
|
+
const USER_AGENT = "Mozilla/5.0 (compatible; opl-pi-sht/webaccess)";
|
|
22
|
+
|
|
23
|
+
export interface PinnedFetchOptions {
|
|
24
|
+
method?: string;
|
|
25
|
+
headers?: Record<string, string>;
|
|
26
|
+
/** Caller's abort signal; aborting destroys the socket and rejects with an AbortError. */
|
|
27
|
+
signal?: AbortSignal;
|
|
28
|
+
/** Socket inactivity deadline (ms). 0 or unset means no transport-side deadline. */
|
|
29
|
+
timeoutMs?: number;
|
|
30
|
+
/** Hard ceiling on both received and decoded body bytes. 0 or unset means unlimited. */
|
|
31
|
+
maxResponseBytes?: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface PinnedResponse {
|
|
35
|
+
status: number;
|
|
36
|
+
ok: boolean;
|
|
37
|
+
headers: { get(name: string): string | null };
|
|
38
|
+
body: Buffer;
|
|
39
|
+
text(): string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function isIpv6(address: string): boolean {
|
|
43
|
+
return address.includes(":");
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function familyOf(address: string): 4 | 6 {
|
|
47
|
+
return isIpv6(address) ? 6 : 4;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function abortError(): Error & { name: string } {
|
|
51
|
+
return Object.assign(new Error("The operation was aborted"), { name: "AbortError" });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* A resolver that answers only from the addresses the guard approved. It ignores the name it is
|
|
56
|
+
* asked about - the point is that nothing downstream gets to choose a different answer.
|
|
57
|
+
*/
|
|
58
|
+
export function pinnedLookup(addresses: string[]) {
|
|
59
|
+
return (
|
|
60
|
+
hostname: string,
|
|
61
|
+
options: LookupOptions,
|
|
62
|
+
callback: (err: Error | null, address?: string | LookupAddress[], family?: number) => void,
|
|
63
|
+
): void => {
|
|
64
|
+
if (options && options.all) {
|
|
65
|
+
callback(null, addresses.map((address) => ({ address, family: familyOf(address) })));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const asked = options && typeof options.family === "string" ? (options.family === "IPv6" ? 6 : 4) : options?.family;
|
|
69
|
+
const want = asked === 6 ? 6 : asked === 4 ? 4 : 0;
|
|
70
|
+
const hit = want ? addresses.find((address) => familyOf(address) === want) ?? addresses[0] : addresses[0];
|
|
71
|
+
if (!hit) {
|
|
72
|
+
callback(new Error(`No pinned address available for "${hostname}"`));
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
callback(null, hit, familyOf(hit));
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The request options for one pinned hop. Exported so the contract is testable without a socket. */
|
|
80
|
+
export function buildRequestOptions(
|
|
81
|
+
target: SafeHttpTarget,
|
|
82
|
+
opts: PinnedFetchOptions = {},
|
|
83
|
+
): http.RequestOptions & { servername?: string } {
|
|
84
|
+
const parsed = new URL(target.url);
|
|
85
|
+
const tls = parsed.protocol === "https:";
|
|
86
|
+
const port = parsed.port ? Number(parsed.port) : tls ? 443 : 80;
|
|
87
|
+
const headers: Record<string, string> = {
|
|
88
|
+
"user-agent": USER_AGENT,
|
|
89
|
+
accept: "*/*",
|
|
90
|
+
// Prefer identity, but decode supported encodings if the server compresses anyway.
|
|
91
|
+
"accept-encoding": "identity",
|
|
92
|
+
...opts.headers,
|
|
93
|
+
};
|
|
94
|
+
headers.host = parsed.host; // keeps the port and IPv6 brackets exactly as the URL spells them
|
|
95
|
+
const options: http.RequestOptions & { servername?: string } = {
|
|
96
|
+
protocol: parsed.protocol,
|
|
97
|
+
method: (opts.method ?? "GET").toUpperCase(),
|
|
98
|
+
host: target.host,
|
|
99
|
+
hostname: target.host,
|
|
100
|
+
port,
|
|
101
|
+
path: `${parsed.pathname}${parsed.search}`,
|
|
102
|
+
headers,
|
|
103
|
+
agent: pinnedAgent(target.url),
|
|
104
|
+
// Node declares this hook as two overloads (`all` or not); the pin answers both, so the widening
|
|
105
|
+
// happens here instead of pretending it is only one of them.
|
|
106
|
+
lookup: pinnedLookup(target.addresses) as http.RequestOptions["lookup"],
|
|
107
|
+
};
|
|
108
|
+
if (tls) options.servername = target.host; // certificate validation stays bound to the name
|
|
109
|
+
return options;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* A fresh agent per request. Pooling is off (`keepAlive: false`) and, because nothing is shared,
|
|
114
|
+
* a socket established under one hop's approved address can never be reused by a later request
|
|
115
|
+
* whose guard approved a different one.
|
|
116
|
+
*/
|
|
117
|
+
export function pinnedAgent(url: string): http.Agent | https.Agent {
|
|
118
|
+
return url.startsWith("https:") ? new https.Agent({ keepAlive: false }) : new http.Agent({ keepAlive: false });
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Only these status codes redirect a GET; other 3xx responses are final. */
|
|
122
|
+
export function isRedirectStatus(status: number): boolean {
|
|
123
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Fetch one already-validated target. Redirects are returned, never followed: the caller re-runs the
|
|
128
|
+
* guard and re-pins for each hop, which is the only place a new host can appear.
|
|
129
|
+
*/
|
|
130
|
+
export function pinnedFetch(target: SafeHttpTarget, opts: PinnedFetchOptions = {}): Promise<PinnedResponse> {
|
|
131
|
+
const options = buildRequestOptions(target, opts);
|
|
132
|
+
const transport = new URL(target.url).protocol === "https:" ? https : http;
|
|
133
|
+
const limit = opts.maxResponseBytes && opts.maxResponseBytes > 0 ? opts.maxResponseBytes : 0;
|
|
134
|
+
|
|
135
|
+
return new Promise<PinnedResponse>((resolve, reject) => {
|
|
136
|
+
let settled = false;
|
|
137
|
+
let incoming: http.IncomingMessage | undefined;
|
|
138
|
+
let decoder: Transform | undefined;
|
|
139
|
+
let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
|
|
140
|
+
const abort = () => fail(abortError());
|
|
141
|
+
const resetTimeout = () => {
|
|
142
|
+
if (!opts.timeoutMs || opts.timeoutMs <= 0) return;
|
|
143
|
+
clearTimeout(inactivityTimer);
|
|
144
|
+
inactivityTimer = setTimeout(() => fail(new Error(`Request to ${target.host} timed out after ${opts.timeoutMs} ms`)), opts.timeoutMs);
|
|
145
|
+
};
|
|
146
|
+
const cleanup = () => {
|
|
147
|
+
opts.signal?.removeEventListener("abort", abort);
|
|
148
|
+
clearTimeout(inactivityTimer);
|
|
149
|
+
decoder?.destroy();
|
|
150
|
+
incoming?.destroy();
|
|
151
|
+
request.destroy();
|
|
152
|
+
if (options.agent && typeof options.agent === "object") options.agent.destroy();
|
|
153
|
+
};
|
|
154
|
+
const fail = (err: Error) => {
|
|
155
|
+
if (settled) return;
|
|
156
|
+
settled = true;
|
|
157
|
+
cleanup();
|
|
158
|
+
reject(err);
|
|
159
|
+
};
|
|
160
|
+
const request = transport.request(options, (response) => {
|
|
161
|
+
incoming = response;
|
|
162
|
+
response.on("error", fail);
|
|
163
|
+
response.on("aborted", () => fail(new Error(`Response from ${target.host} ended before its body completed`)));
|
|
164
|
+
const status = response.statusCode ?? 0;
|
|
165
|
+
const headers = {
|
|
166
|
+
get(name: string): string | null {
|
|
167
|
+
const value = response.headers[name.toLowerCase()];
|
|
168
|
+
if (Array.isArray(value)) return value.join(", ");
|
|
169
|
+
return value === undefined ? null : String(value);
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
const finish = (body: Buffer) => {
|
|
173
|
+
if (settled) return;
|
|
174
|
+
settled = true;
|
|
175
|
+
cleanup();
|
|
176
|
+
resolve({ status, ok: status >= 200 && status < 300, headers, body, text: () => body.toString("utf8") });
|
|
177
|
+
};
|
|
178
|
+
// A redirect body is irrelevant. Close this socket and let the caller validate the next hop.
|
|
179
|
+
if (isRedirectStatus(status) && headers.get("location")) {
|
|
180
|
+
finish(Buffer.alloc(0));
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
// These responses cannot carry a body, even if they advertise its size or encoding.
|
|
184
|
+
if (options.method === "HEAD" || status === 204 || status === 304) {
|
|
185
|
+
finish(Buffer.alloc(0));
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
const declared = Number(response.headers["content-length"]);
|
|
189
|
+
if (limit > 0 && Number.isFinite(declared) && declared > limit) {
|
|
190
|
+
fail(new Error(`Response from ${target.host} is ${declared} bytes, over the ${limit} byte limit (set maxResponseBytes to raise it)`));
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
const encoding = (headers.get("content-encoding") ?? "identity").trim().toLowerCase();
|
|
194
|
+
switch (encoding) {
|
|
195
|
+
case "identity": break;
|
|
196
|
+
case "gzip": decoder = createGunzip(); break;
|
|
197
|
+
case "deflate": decoder = createInflate(); break;
|
|
198
|
+
case "br": decoder = createBrotliDecompress(); break;
|
|
199
|
+
default:
|
|
200
|
+
fail(new Error(`Unsupported content encoding "${encoding}" from ${target.host}`));
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
const chunks: Buffer[] = [];
|
|
204
|
+
let received = 0;
|
|
205
|
+
let decoded = 0;
|
|
206
|
+
const collect = (chunk: Buffer) => {
|
|
207
|
+
if (settled) return;
|
|
208
|
+
decoded += chunk.length;
|
|
209
|
+
if (limit > 0 && decoded > limit) {
|
|
210
|
+
fail(new Error(`Decoded response from ${target.host} exceeds the ${limit} byte limit (set maxResponseBytes to raise it)`));
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
chunks.push(chunk);
|
|
214
|
+
};
|
|
215
|
+
response.on("data", (chunk: Buffer) => {
|
|
216
|
+
if (settled) return;
|
|
217
|
+
resetTimeout();
|
|
218
|
+
received += chunk.length;
|
|
219
|
+
if (limit > 0 && received > limit) {
|
|
220
|
+
fail(new Error(`Response from ${target.host} exceeds the ${limit} byte limit (set maxResponseBytes to raise it)`));
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
const bodyStream = decoder ?? response;
|
|
224
|
+
bodyStream.on("data", collect);
|
|
225
|
+
bodyStream.on("end", () => finish(Buffer.concat(chunks)));
|
|
226
|
+
if (decoder) {
|
|
227
|
+
decoder.on("error", (err: Error) => fail(new Error(`Cannot decompress ${encoding} response from ${target.host}: ${err.message}`, { cause: err })));
|
|
228
|
+
response.pipe(decoder);
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
request.on("error", (err) => {
|
|
232
|
+
const failure = err as Error & { code?: string };
|
|
233
|
+
if (failure.name === "AbortError" || /byte limit|timed out/.test(failure.message)) {
|
|
234
|
+
fail(failure);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
// Name the address the guard approved: "connect ECONNREFUSED api.internal:80" on its own hides
|
|
238
|
+
// which validated answer the socket was pinned to, which is the first thing triage needs.
|
|
239
|
+
fail(
|
|
240
|
+
Object.assign(new Error(`${target.host} -> ${target.addresses.join(", ")}: ${failure.message}`), {
|
|
241
|
+
code: failure.code,
|
|
242
|
+
cause: failure,
|
|
243
|
+
}),
|
|
244
|
+
);
|
|
245
|
+
});
|
|
246
|
+
// Both the deadline and the caller's abort own their error objects. Handing destroy() an error
|
|
247
|
+
// reaches the rejection reliably on Node; on Bun the socket comes back as ECONNRESET or a bare
|
|
248
|
+
// "Connection closed", which would hide the reason from the model and from these tests.
|
|
249
|
+
resetTimeout();
|
|
250
|
+
if (opts.signal) {
|
|
251
|
+
if (opts.signal.aborted) {
|
|
252
|
+
fail(abortError());
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
opts.signal.addEventListener("abort", abort, { once: true });
|
|
256
|
+
}
|
|
257
|
+
request.end();
|
|
258
|
+
});
|
|
259
|
+
}
|
|
@@ -10,13 +10,14 @@ import {
|
|
|
10
10
|
restoreFromSession,
|
|
11
11
|
} from "./storage.js";
|
|
12
12
|
import { truncate, errorMessage, paginateContent, continuationNotice } from "./utils.js";
|
|
13
|
-
import { loadConfig, resolveCaps } from "./config.js";
|
|
13
|
+
import { loadConfig, resolveCaps, resolveTimeoutMs, resolveMaxResponseBytes } from "./config.js";
|
|
14
14
|
import type { StoredData } from "./types.js";
|
|
15
15
|
|
|
16
16
|
export default function (pi: ExtensionAPI) {
|
|
17
17
|
const config = loadConfig();
|
|
18
18
|
const caps = resolveCaps(config);
|
|
19
19
|
const allowPrivateNetwork = config.allowPrivateNetwork === true;
|
|
20
|
+
const allowLoopback = config.allowLoopback === true;
|
|
20
21
|
|
|
21
22
|
// Note shown to the model when the per-call URL/query cap drops some inputs.
|
|
22
23
|
const skippedNote = (dropped: number, limit: number, singular: string, plural: string) =>
|
|
@@ -57,11 +58,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
57
58
|
try {
|
|
58
59
|
// allSettled so one provider's throw (non-JSON body, missing key) cannot
|
|
59
60
|
// discard the sibling queries' results.
|
|
60
|
-
const settled = await Promise.allSettled(queryList.map((q) => searchWeb(q, signal)));
|
|
61
|
+
const settled = await Promise.allSettled(queryList.map((q) => searchWeb(q, signal, config)));
|
|
61
62
|
results = settled.map((s, i) =>
|
|
62
63
|
s.status === "fulfilled"
|
|
63
64
|
? s.value
|
|
64
|
-
: { query: queryList[i]
|
|
65
|
+
: { query: queryList[i] ?? `query #${i}`, answer: "", results: [], error: errorMessage(s.reason) } // index-aligned by allSettled; the fallback only names a list that cannot happen
|
|
65
66
|
);
|
|
66
67
|
} catch (err) {
|
|
67
68
|
return {
|
|
@@ -121,7 +122,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
121
122
|
let results;
|
|
122
123
|
const dropped = urlList.length > caps.maxFetchUrls ? urlList.length - caps.maxFetchUrls : 0;
|
|
123
124
|
try {
|
|
124
|
-
results = await fetchAllContent(urlList, signal, { allowPrivateNetwork }, caps.maxFetchUrls);
|
|
125
|
+
results = await fetchAllContent(urlList, signal, { allowPrivateNetwork, allowLoopback, timeoutMs: resolveTimeoutMs(config), maxResponseBytes: resolveMaxResponseBytes(config) }, caps.maxFetchUrls);
|
|
125
126
|
} catch (err) {
|
|
126
127
|
return {
|
|
127
128
|
content: [{ type: "text", text: `Error: ${errorMessage(err)}` }],
|
|
@@ -176,6 +177,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
176
177
|
}),
|
|
177
178
|
async execute(_toolCallId, params) {
|
|
178
179
|
const data = getResult(params.responseId);
|
|
180
|
+
// A restored entry is a capped preview until its spilled body is found; say so
|
|
181
|
+
// instead of handing the model a short answer that looks complete.
|
|
182
|
+
const staleNote = data?.truncated
|
|
183
|
+
? "\n\n*Session copy: the cached full body is no longer on disk, so this excerpt is capped. Re-fetch to retrieve the rest.*"
|
|
184
|
+
: "";
|
|
179
185
|
if (!data) {
|
|
180
186
|
return {
|
|
181
187
|
content: [
|
|
@@ -215,7 +221,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
215
221
|
caps.maxRetrievalChars,
|
|
216
222
|
);
|
|
217
223
|
return {
|
|
218
|
-
content: [{ type: "text", text: page.text + continuationNotice(page) }],
|
|
224
|
+
content: [{ type: "text", text: page.text + continuationNotice(page) + staleNote }],
|
|
219
225
|
details: { responseId: params.responseId },
|
|
220
226
|
};
|
|
221
227
|
}
|
|
@@ -252,7 +258,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
252
258
|
caps.maxRetrievalChars,
|
|
253
259
|
);
|
|
254
260
|
return {
|
|
255
|
-
content: [{ type: "text", text: page.text + continuationNotice(page) }],
|
|
261
|
+
content: [{ type: "text", text: page.text + continuationNotice(page) + staleNote }],
|
|
256
262
|
details: { responseId: params.responseId },
|
|
257
263
|
};
|
|
258
264
|
}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opl-webaccess",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "opl-webaccess",
|
|
9
|
-
"version": "
|
|
10
|
-
"license": "
|
|
9
|
+
"version": "0.2.0",
|
|
10
|
+
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@mozilla/readability": "^0.6.0",
|
|
13
13
|
"linkedom": "^0.18.13",
|
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opl-webaccess",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"test": "echo \"Error: no test specified\" && exit 1"
|
|
8
|
-
},
|
|
9
|
-
"keywords": [],
|
|
10
|
-
"author": "",
|
|
11
|
-
"license": "ISC",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"private": true,
|
|
5
|
+
"description": "Web search and content extraction for pi. Search the web via Gemini AI, fetch and read web pages, and extract text from PDFs \u2014 all from within the agent.",
|
|
6
|
+
"license": "MIT",
|
|
12
7
|
"dependencies": {
|
|
13
8
|
"@mozilla/readability": "^0.6.0",
|
|
14
9
|
"linkedom": "^0.18.13",
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
export async function extractPdfBuffer(buffer: ArrayBuffer): Promise<string> {
|
|
1
|
+
export async function extractPdfBuffer(buffer: ArrayBuffer | Uint8Array): Promise<string> {
|
|
4
2
|
const { getDocumentProxy, extractText } = await import("unpdf");
|
|
5
|
-
|
|
3
|
+
// Buffer extends Uint8Array, but PDF.js rejects it; copy into the accepted type.
|
|
4
|
+
const bytes = new Uint8Array(buffer);
|
|
5
|
+
const pdf = await getDocumentProxy(bytes);
|
|
6
6
|
const { text } = await extractText(pdf, { mergePages: true });
|
|
7
7
|
return text ?? "";
|
|
8
8
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadConfig, getApiKey } from "./config.js";
|
|
1
|
+
import { loadConfig, getApiKey, resolveTimeoutMs, type WebAccessConfig } from "./config.js";
|
|
2
2
|
import { searchGemini } from "./providers/gemini.js";
|
|
3
3
|
import { searchTavily } from "./providers/tavily.js";
|
|
4
4
|
import { searchDdgs } from "./providers/ddgs.js";
|
|
@@ -6,25 +6,36 @@ import { searchSearxng } from "./providers/searxng.js";
|
|
|
6
6
|
import { searchExa } from "./providers/exa.js";
|
|
7
7
|
import type { SearchQueryResult } from "./types.js";
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Run one search with the configured provider.
|
|
11
|
+
*
|
|
12
|
+
* `config` is passed in by callers that fan out over queries, so a multi-query
|
|
13
|
+
* call reads and parses the config file once instead of once per query. Every
|
|
14
|
+
* provider request gets a deadline: the agent signal alone can stay pending
|
|
15
|
+
* forever if a provider accepts the connection and never answers.
|
|
16
|
+
*/
|
|
9
17
|
export async function searchWeb(
|
|
10
18
|
query: string,
|
|
11
|
-
signal?: AbortSignal
|
|
19
|
+
signal?: AbortSignal,
|
|
20
|
+
config: WebAccessConfig = loadConfig(),
|
|
12
21
|
): Promise<SearchQueryResult> {
|
|
13
|
-
const config = loadConfig();
|
|
14
22
|
const providerName = config.provider;
|
|
15
23
|
const providerCfg = config.providers?.[providerName] ?? {};
|
|
24
|
+
const timeoutMs = resolveTimeoutMs(config);
|
|
25
|
+
const deadline = AbortSignal.timeout(timeoutMs);
|
|
26
|
+
const tied = signal ? AbortSignal.any([signal, deadline]) : deadline;
|
|
16
27
|
|
|
17
28
|
switch (providerName) {
|
|
18
29
|
case "gemini":
|
|
19
|
-
return searchGemini(query, providerCfg, getApiKey(providerCfg),
|
|
30
|
+
return searchGemini(query, providerCfg, getApiKey(providerCfg), tied);
|
|
20
31
|
case "tavily":
|
|
21
|
-
return searchTavily(query, providerCfg, getApiKey(providerCfg),
|
|
32
|
+
return searchTavily(query, providerCfg, getApiKey(providerCfg), tied);
|
|
22
33
|
case "ddgs":
|
|
23
|
-
return searchDdgs(query, providerCfg,
|
|
34
|
+
return searchDdgs(query, providerCfg, tied);
|
|
24
35
|
case "searxng":
|
|
25
|
-
return searchSearxng(query, providerCfg,
|
|
36
|
+
return searchSearxng(query, providerCfg, tied);
|
|
26
37
|
case "exa":
|
|
27
|
-
return searchExa(query, providerCfg, getApiKey(providerCfg),
|
|
38
|
+
return searchExa(query, providerCfg, getApiKey(providerCfg), tied);
|
|
28
39
|
default:
|
|
29
40
|
return {
|
|
30
41
|
query,
|