@openlines/opl-pi-sht 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -8
- package/configs/opl-browser.json.sample +3 -2
- package/configs/opl-guardian.json.sample +28 -0
- package/configs/opl-modes.json.sample +3 -3
- package/configs/opl-webaccess.json.sample +5 -1
- package/extensions/opl-browser/README.md +20 -6
- package/extensions/opl-browser/browser.ts +76 -19
- package/extensions/opl-browser/config.ts +5 -0
- package/extensions/opl-browser/index.ts +13 -5
- package/extensions/opl-browser/validate.ts +123 -5
- package/extensions/opl-guardian/README.md +25 -27
- package/extensions/opl-guardian/config.ts +199 -0
- package/extensions/opl-guardian/guardian.ts +5 -1
- package/extensions/opl-guardian/index.ts +129 -7
- package/extensions/opl-guardian/policies.ts +123 -0
- package/extensions/opl-modes/README.md +2 -2
- package/extensions/opl-questionnaire/README.md +1 -1
- package/extensions/opl-questionnaire/index.ts +36 -21
- package/extensions/opl-webaccess/README.md +8 -5
- package/extensions/opl-webaccess/config.ts +56 -5
- package/extensions/opl-webaccess/extract.ts +39 -14
- package/extensions/opl-webaccess/index.ts +27 -7
- package/extensions/opl-webaccess/providers/gemini.ts +6 -1
- package/extensions/opl-webaccess/providers/tavily.ts +6 -1
- package/extensions/opl-webaccess/storage.ts +8 -2
- package/extensions/opl-webaccess/utils.ts +117 -2
- package/package.json +3 -3
|
@@ -27,16 +27,41 @@ import {
|
|
|
27
27
|
type RenderOption,
|
|
28
28
|
} from "./types.js";
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
/** Build an error tool result. Marked as an error (not a cancellation) so the
|
|
31
|
+
* result card renders the message instead of mislabeling it "Cancelled". */
|
|
32
|
+
export function errorResult(
|
|
31
33
|
message: string,
|
|
32
|
-
|
|
33
|
-
): { content: { type: "text"; text: string }[]; details: QuestionnaireResult } {
|
|
34
|
+
): { content: { type: "text"; text: string }[]; isError: true; details: QuestionnaireResult } {
|
|
34
35
|
return {
|
|
35
36
|
content: [{ type: "text", text: message }],
|
|
36
|
-
|
|
37
|
+
isError: true,
|
|
38
|
+
details: { questions: [], answers: [], cancelled: false },
|
|
37
39
|
};
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
/** Validate the question list. Returns the first error message, or null when valid.
|
|
43
|
+
* Blank ids are rejected (answers are keyed by id and "" would produce an empty key),
|
|
44
|
+
* as are duplicate ids and questions with nothing selectable. */
|
|
45
|
+
export function validateQuestions(questions: Question[]): string | null {
|
|
46
|
+
const seenIds = new Set<string>();
|
|
47
|
+
for (const q of questions) {
|
|
48
|
+
if (q.id.trim() === "") {
|
|
49
|
+
return "Error: Question id must be a non-empty string";
|
|
50
|
+
}
|
|
51
|
+
if (seenIds.has(q.id)) {
|
|
52
|
+
return `Error: Duplicate question id "${q.id}" — question ids must be unique`;
|
|
53
|
+
}
|
|
54
|
+
seenIds.add(q.id);
|
|
55
|
+
}
|
|
56
|
+
for (const q of questions) {
|
|
57
|
+
const allowOther = q.allowOther !== false;
|
|
58
|
+
if (q.options.length === 0 && !allowOther) {
|
|
59
|
+
return `Error: Question "${q.id}" has no options and allowOther is false — nothing is selectable`;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
40
65
|
export default function questionnaire(pi: ExtensionAPI) {
|
|
41
66
|
pi.registerTool({
|
|
42
67
|
name: "questionnaire",
|
|
@@ -58,23 +83,9 @@ export default function questionnaire(pi: ExtensionAPI) {
|
|
|
58
83
|
return errorResult("Error: No questions provided");
|
|
59
84
|
}
|
|
60
85
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (seenIds.has(q.id)) {
|
|
65
|
-
return errorResult(`Error: Duplicate question id "${q.id}" — question ids must be unique`);
|
|
66
|
-
}
|
|
67
|
-
seenIds.add(q.id);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// Reject unselectable questions (no options and no free-text fallback).
|
|
71
|
-
for (const q of params.questions) {
|
|
72
|
-
const allowOther = q.allowOther !== false;
|
|
73
|
-
if (q.options.length === 0 && !allowOther) {
|
|
74
|
-
return errorResult(
|
|
75
|
-
`Error: Question "${q.id}" has no options and allowOther is false — nothing is selectable`,
|
|
76
|
-
);
|
|
77
|
-
}
|
|
86
|
+
const validationError = validateQuestions(params.questions);
|
|
87
|
+
if (validationError) {
|
|
88
|
+
return errorResult(validationError);
|
|
78
89
|
}
|
|
79
90
|
|
|
80
91
|
// Normalize questions with defaults
|
|
@@ -424,6 +435,10 @@ export default function questionnaire(pi: ExtensionAPI) {
|
|
|
424
435
|
},
|
|
425
436
|
|
|
426
437
|
renderResult(result, _options, theme, _context) {
|
|
438
|
+
if (result.isError) {
|
|
439
|
+
const text = result.content[0];
|
|
440
|
+
return new Text(theme.fg("error", text?.type === "text" ? text.text : ""), 0, 0);
|
|
441
|
+
}
|
|
427
442
|
const details = result.details as QuestionnaireResult | undefined;
|
|
428
443
|
if (!details) {
|
|
429
444
|
const text = result.content[0];
|
|
@@ -5,8 +5,8 @@ Provides configurable web search and readable URL/PDF retrieval with session-bac
|
|
|
5
5
|
## Commands, flags, and shortcuts
|
|
6
6
|
|
|
7
7
|
- Tools: `web_search`, `fetch_content`, and `get_search_content`.
|
|
8
|
-
- `web_search` accepts `query` or parallel `queries
|
|
9
|
-
- `fetch_content` accepts `url` or `urls` with at most three requests in flight.
|
|
8
|
+
- `web_search` accepts `query` or parallel `queries` (at most 10 per call).
|
|
9
|
+
- `fetch_content` accepts `url` or `urls` (at most 20 per call) with at most three requests in flight.
|
|
10
10
|
- `get_search_content` uses a prior `responseId`, with `queryIndex`, `urlIndex`, or exact `url` selection. It returns one bounded page; pass `offset` to continue a truncated retrieval.
|
|
11
11
|
- No slash commands or shortcuts.
|
|
12
12
|
|
|
@@ -14,8 +14,8 @@ Provides configurable web search and readable URL/PDF retrieval with session-bac
|
|
|
14
14
|
|
|
15
15
|
- Searches through `gemini`, `tavily`, `ddgs`, `searxng`, or `exa`; multiple queries run concurrently and results include citations where available.
|
|
16
16
|
- Extracts HTML with Readability and Markdown conversion, falls back to full-document Turndown conversion, passes PDF responses to the PDF extractor, and returns plain text, Markdown, and JSON directly.
|
|
17
|
-
- Restricts `fetch_content` to http/https,
|
|
18
|
-
- Caps initial tool output at a configurable length (default 30,000 characters), then keeps it for retrieval for one hour or until the session ends.
|
|
17
|
+
- Restricts `fetch_content` to http/https and blocks private-range, link-local, and reserved hosts by default, re-checks every redirect hop, and always blocks cloud-metadata endpoints (SSRF guard). Loopback (`localhost`/`127.0.0.0/8`/`::1`) is allowed by default for local development; set `allowPrivateNetwork: true` to reach private ranges. Caps each response at 10 MB and applies a 30s timeout; Gemini keys are sent via the `x-goog-api-key` header rather than the query string.
|
|
18
|
+
- Caps initial tool output at a configurable length (default 30,000 characters), then keeps it for retrieval for one hour (expiry is enforced on read, not just on write) or until the session ends.
|
|
19
19
|
- Pages `get_search_content` results (default 30,000 characters per page) with `offset` continuation, so a model never loads the whole stored body in one call.
|
|
20
20
|
- Honors abort signals and returns provider, HTTP, and per-result failures through the tool boundary rather than throwing.
|
|
21
21
|
|
|
@@ -33,7 +33,7 @@ Copy [`configs/opl-webaccess.json.sample`](../../configs/opl-webaccess.json.samp
|
|
|
33
33
|
}
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
-
Provider fields include `apiKeyEnv`, `apiUrl`, `baseUrl`, `model`, `maxResults`, `instanceUrl`, `categories`, `safeSearch`, `searchType`, and `includeSummary`. API keys are read only from named environment variables.
|
|
36
|
+
Provider fields include `apiKeyEnv`, `apiUrl`, `baseUrl`, `model`, `maxResults`, `instanceUrl`, `categories`, `safeSearch`, `searchType`, and `includeSummary`. API keys are read only from named environment variables. Malformed config falls back to Gemini defaults - a non-object config, a non-string `provider`, a non-object `providers` map, or non-positive cap values (`maxContentChars`/`maxRetrievalChars`) are all treated as unset; an unknown provider returns an error.
|
|
37
37
|
|
|
38
38
|
Optional top-level caps control how much content reaches the model:
|
|
39
39
|
|
|
@@ -41,6 +41,9 @@ Optional top-level caps control how much content reaches the model:
|
|
|
41
41
|
|---|---|---|
|
|
42
42
|
| `maxContentChars` | `30000` | Cap on the initial `web_search`/`fetch_content` body. |
|
|
43
43
|
| `maxRetrievalChars` | `30000` | Cap on one `get_search_content` page. Pass `offset` to continue. |
|
|
44
|
+
| `maxSearchQueries` | `10` | Cap on how many queries one `web_search` call runs. Excess queries are skipped and reported. |
|
|
45
|
+
| `maxFetchUrls` | `20` | Cap on how many URLs one `fetch_content` call fetches. Excess URLs are skipped and reported. |
|
|
46
|
+
| `allowPrivateNetwork` | `false` | Allow `fetch_content` to reach private/link-local ranges (loopback is always allowed; cloud metadata is always blocked). Provider API endpoints (e.g. `ddgs.apiUrl`, `searxng.instanceUrl`) are excluded from this guard. |
|
|
44
47
|
|
|
45
48
|
Install extraction dependencies before use:
|
|
46
49
|
|
|
@@ -22,16 +22,36 @@ export interface WebAccessConfig {
|
|
|
22
22
|
maxContentChars?: number;
|
|
23
23
|
/** Cap on one get_search_content retrieval page (chars). */
|
|
24
24
|
maxRetrievalChars?: number;
|
|
25
|
+
/** Cap on the number of queries a single web_search call may run. */
|
|
26
|
+
maxSearchQueries?: number;
|
|
27
|
+
/** Cap on the number of URLs a single fetch_content call may fetch. */
|
|
28
|
+
maxFetchUrls?: number;
|
|
29
|
+
/** Allow fetch_content to reach private/link-local ranges (loopback is always
|
|
30
|
+
* allowed; cloud metadata is always blocked). Default false. */
|
|
31
|
+
allowPrivateNetwork?: boolean;
|
|
25
32
|
}
|
|
26
33
|
|
|
27
34
|
export const DEFAULT_MAX_CONTENT_CHARS = 30_000;
|
|
28
35
|
export const DEFAULT_MAX_RETRIEVAL_CHARS = 30_000;
|
|
36
|
+
export const DEFAULT_MAX_SEARCH_QUERIES = 10;
|
|
37
|
+
export const DEFAULT_MAX_FETCH_URLS = 20;
|
|
29
38
|
|
|
30
|
-
/**
|
|
31
|
-
|
|
39
|
+
/** Coerce an unknown value into a positive integer, else the fallback. */
|
|
40
|
+
function positiveInt(value: unknown, fallback: number): number {
|
|
41
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Resolve configurable caps, falling back to defaults when a cap is missing or not
|
|
45
|
+
* a positive finite number (a string/negative/zero cap would otherwise make
|
|
46
|
+
* truncate/slice produce empty, inverted, or unbounded output). */
|
|
47
|
+
export function resolveCaps(
|
|
48
|
+
cfg: Pick<WebAccessConfig, "maxContentChars" | "maxRetrievalChars" | "maxSearchQueries" | "maxFetchUrls">,
|
|
49
|
+
) {
|
|
32
50
|
return {
|
|
33
|
-
maxContentChars: cfg.maxContentChars
|
|
34
|
-
maxRetrievalChars: cfg.maxRetrievalChars
|
|
51
|
+
maxContentChars: positiveInt(cfg.maxContentChars, DEFAULT_MAX_CONTENT_CHARS),
|
|
52
|
+
maxRetrievalChars: positiveInt(cfg.maxRetrievalChars, DEFAULT_MAX_RETRIEVAL_CHARS),
|
|
53
|
+
maxSearchQueries: positiveInt(cfg.maxSearchQueries, DEFAULT_MAX_SEARCH_QUERIES),
|
|
54
|
+
maxFetchUrls: positiveInt(cfg.maxFetchUrls, DEFAULT_MAX_FETCH_URLS),
|
|
35
55
|
};
|
|
36
56
|
}
|
|
37
57
|
|
|
@@ -50,12 +70,43 @@ const DEFAULT_CONFIG: WebAccessConfig = {
|
|
|
50
70
|
|
|
51
71
|
export function loadConfig(): WebAccessConfig {
|
|
52
72
|
if (!existsSync(CONFIG_PATH)) return DEFAULT_CONFIG;
|
|
73
|
+
let parsed: unknown;
|
|
53
74
|
try {
|
|
54
|
-
|
|
75
|
+
parsed = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
|
|
55
76
|
} catch {
|
|
56
77
|
console.error(`[opl-webaccess] failed to parse ${CONFIG_PATH}, using defaults`);
|
|
57
78
|
return DEFAULT_CONFIG;
|
|
58
79
|
}
|
|
80
|
+
return normalizeConfig(parsed);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Validate a parsed config so malformed fields fall back to defaults instead of
|
|
84
|
+
* silently misbehaving downstream (a non-object config, a non-string provider, or
|
|
85
|
+
* a non-object providers map are all treated as unset). Cap values are validated
|
|
86
|
+
* in resolveCaps. */
|
|
87
|
+
function normalizeConfig(parsed: unknown): WebAccessConfig {
|
|
88
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
89
|
+
console.error("[opl-webaccess] config is not a JSON object, using defaults");
|
|
90
|
+
return DEFAULT_CONFIG;
|
|
91
|
+
}
|
|
92
|
+
const record = parsed as Record<string, unknown>;
|
|
93
|
+
const provider =
|
|
94
|
+
typeof record.provider === "string" && record.provider.trim() !== ""
|
|
95
|
+
? record.provider
|
|
96
|
+
: DEFAULT_CONFIG.provider;
|
|
97
|
+
const providers =
|
|
98
|
+
typeof record.providers === "object" && record.providers !== null && !Array.isArray(record.providers)
|
|
99
|
+
? (record.providers as Record<string, ProviderConfig>)
|
|
100
|
+
: DEFAULT_CONFIG.providers;
|
|
101
|
+
return {
|
|
102
|
+
provider,
|
|
103
|
+
providers,
|
|
104
|
+
maxContentChars: record.maxContentChars as number | undefined,
|
|
105
|
+
maxRetrievalChars: record.maxRetrievalChars as number | undefined,
|
|
106
|
+
maxSearchQueries: record.maxSearchQueries as number | undefined,
|
|
107
|
+
maxFetchUrls: record.maxFetchUrls as number | undefined,
|
|
108
|
+
allowPrivateNetwork: record.allowPrivateNetwork === true,
|
|
109
|
+
};
|
|
59
110
|
}
|
|
60
111
|
|
|
61
112
|
export function getApiKey(cfg: ProviderConfig): string {
|
|
@@ -2,12 +2,15 @@ import { Readability } from "@mozilla/readability";
|
|
|
2
2
|
import { parseHTML } from "linkedom";
|
|
3
3
|
import TurndownService from "turndown";
|
|
4
4
|
import { extractPdfBuffer } from "./pdf.js";
|
|
5
|
-
import { errorMessage, isAbortError, isPdfUrl, isPdfContentType, assertHttpUrl } from "./utils.js";
|
|
5
|
+
import { errorMessage, isAbortError, isPdfUrl, isPdfContentType, assertHttpUrl, type HttpUrlOptions } from "./utils.js";
|
|
6
6
|
import type { ExtractedContent } from "./types.js";
|
|
7
7
|
|
|
8
8
|
const CONCURRENT_LIMIT = 3;
|
|
9
9
|
const MAX_FETCH_BYTES = 10 * 1024 * 1024; // 10 MB response cap
|
|
10
10
|
const FETCH_TIMEOUT_MS = 30_000;
|
|
11
|
+
const MAX_REDIRECTS = 5;
|
|
12
|
+
/** Ceiling on the number of URLs a single fetch_content call may include. */
|
|
13
|
+
export const MAX_FETCH_URLS = 20;
|
|
11
14
|
|
|
12
15
|
const td = new TurndownService({
|
|
13
16
|
headingStyle: "atx",
|
|
@@ -43,31 +46,53 @@ async function readBufferCapped(response: Response, maxBytes: number): Promise<A
|
|
|
43
46
|
|
|
44
47
|
export async function fetchAllContent(
|
|
45
48
|
urls: string[],
|
|
46
|
-
signal?: AbortSignal
|
|
49
|
+
signal?: AbortSignal,
|
|
50
|
+
opts: HttpUrlOptions = {},
|
|
51
|
+
maxUrls: number = MAX_FETCH_URLS,
|
|
47
52
|
): Promise<ExtractedContent[]> {
|
|
53
|
+
const bounded = urls.slice(0, maxUrls);
|
|
48
54
|
const results: ExtractedContent[] = [];
|
|
49
|
-
for (let i = 0; i <
|
|
55
|
+
for (let i = 0; i < bounded.length; i += CONCURRENT_LIMIT) {
|
|
50
56
|
if (signal?.aborted) break;
|
|
51
|
-
const batch =
|
|
52
|
-
const batchResults = await Promise.all(batch.map((url) => fetchOne(url, signal)));
|
|
57
|
+
const batch = bounded.slice(i, i + CONCURRENT_LIMIT);
|
|
58
|
+
const batchResults = await Promise.all(batch.map((url) => fetchOne(url, signal, opts)));
|
|
53
59
|
results.push(...batchResults);
|
|
54
60
|
}
|
|
55
61
|
return results;
|
|
56
62
|
}
|
|
57
63
|
|
|
58
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Fetch with redirects followed manually so each hop is re-validated. The native
|
|
66
|
+
* `redirect: "follow"` would let a public URL bounce to a private/link-local host
|
|
67
|
+
* past the `assertHttpUrl` check.
|
|
68
|
+
*/
|
|
69
|
+
async function fetchWithRedirectValidation(
|
|
70
|
+
url: string,
|
|
71
|
+
fetchSignal: AbortSignal,
|
|
72
|
+
opts: HttpUrlOptions
|
|
73
|
+
): Promise<Response> {
|
|
74
|
+
let current = assertHttpUrl(url, opts);
|
|
75
|
+
const headers = {
|
|
76
|
+
"User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
|
|
77
|
+
Accept: "text/html,application/xhtml+xml,application/pdf,*/*",
|
|
78
|
+
};
|
|
79
|
+
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
|
80
|
+
const response = await fetch(current, { signal: fetchSignal, headers, redirect: "manual" });
|
|
81
|
+
if (response.status < 300 || response.status >= 400) return response;
|
|
82
|
+
const location = response.headers.get("location");
|
|
83
|
+
if (!location) return response; // 3xx without a Location: treat as final
|
|
84
|
+
await response.body?.cancel().catch(() => {});
|
|
85
|
+
current = assertHttpUrl(new URL(location, current).href, opts);
|
|
86
|
+
}
|
|
87
|
+
throw new Error(`Too many redirects (max ${MAX_REDIRECTS})`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function fetchOne(url: string, signal?: AbortSignal, opts: HttpUrlOptions = {}): Promise<ExtractedContent> {
|
|
59
91
|
try {
|
|
60
|
-
const target = assertHttpUrl(url);
|
|
61
92
|
const fetchSignal = signal
|
|
62
93
|
? AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)])
|
|
63
94
|
: AbortSignal.timeout(FETCH_TIMEOUT_MS);
|
|
64
|
-
const response = await
|
|
65
|
-
signal: fetchSignal,
|
|
66
|
-
headers: {
|
|
67
|
-
"User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
|
|
68
|
-
Accept: "text/html,application/xhtml+xml,application/pdf,*/*",
|
|
69
|
-
},
|
|
70
|
-
});
|
|
95
|
+
const response = await fetchWithRedirectValidation(url, fetchSignal, opts);
|
|
71
96
|
|
|
72
97
|
if (!response.ok) {
|
|
73
98
|
return { url, title: "", content: "", error: `HTTP ${response.status}: ${url}` };
|
|
@@ -14,7 +14,15 @@ import { loadConfig, resolveCaps } from "./config.js";
|
|
|
14
14
|
import type { StoredData } from "./types.js";
|
|
15
15
|
|
|
16
16
|
export default function (pi: ExtensionAPI) {
|
|
17
|
-
const
|
|
17
|
+
const config = loadConfig();
|
|
18
|
+
const caps = resolveCaps(config);
|
|
19
|
+
const allowPrivateNetwork = config.allowPrivateNetwork === true;
|
|
20
|
+
|
|
21
|
+
// Note shown to the model when the per-call URL/query cap drops some inputs.
|
|
22
|
+
const skippedNote = (dropped: number, limit: number, singular: string, plural: string) =>
|
|
23
|
+
dropped > 0
|
|
24
|
+
? `\n\n*Note: ${dropped} additional ${dropped === 1 ? singular : plural} beyond the per-call limit of ${limit} ${dropped === 1 ? "was" : "were"} skipped.*`
|
|
25
|
+
: "";
|
|
18
26
|
|
|
19
27
|
pi.on("session_start", async (_event, ctx) => {
|
|
20
28
|
restoreFromSession(ctx);
|
|
@@ -34,18 +42,27 @@ export default function (pi: ExtensionAPI) {
|
|
|
34
42
|
),
|
|
35
43
|
}),
|
|
36
44
|
async execute(_toolCallId, params, signal) {
|
|
37
|
-
const
|
|
38
|
-
if (!
|
|
45
|
+
const rawQueries = params.queries ?? (params.query ? [params.query] : null);
|
|
46
|
+
if (!rawQueries?.length) {
|
|
39
47
|
return {
|
|
40
48
|
content: [{ type: "text", text: "Error: provide either query or queries." }],
|
|
41
49
|
isError: true,
|
|
42
50
|
details: {} as Record<string, unknown>,
|
|
43
51
|
};
|
|
44
52
|
}
|
|
53
|
+
const queryList = rawQueries.slice(0, caps.maxSearchQueries);
|
|
54
|
+
const dropped = rawQueries.length - queryList.length;
|
|
45
55
|
|
|
46
56
|
let results;
|
|
47
57
|
try {
|
|
48
|
-
|
|
58
|
+
// allSettled so one provider's throw (non-JSON body, missing key) cannot
|
|
59
|
+
// discard the sibling queries' results.
|
|
60
|
+
const settled = await Promise.allSettled(queryList.map((q) => searchWeb(q, signal)));
|
|
61
|
+
results = settled.map((s, i) =>
|
|
62
|
+
s.status === "fulfilled"
|
|
63
|
+
? s.value
|
|
64
|
+
: { query: queryList[i], answer: "", results: [], error: errorMessage(s.reason) }
|
|
65
|
+
);
|
|
49
66
|
} catch (err) {
|
|
50
67
|
return {
|
|
51
68
|
content: [{ type: "text", text: `Error: ${errorMessage(err)}` }],
|
|
@@ -70,8 +87,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
70
87
|
});
|
|
71
88
|
|
|
72
89
|
const body = truncate(sections.join("\n\n---\n\n"), caps.maxContentChars);
|
|
90
|
+
const note = skippedNote(dropped, caps.maxSearchQueries, "query", "queries");
|
|
73
91
|
return {
|
|
74
|
-
content: [{ type: "text", text: `${body}\n\n*responseId: ${id}*` }],
|
|
92
|
+
content: [{ type: "text", text: `${body}${note}\n\n*responseId: ${id}*` }],
|
|
75
93
|
details: { responseId: id, queryCount: results.length },
|
|
76
94
|
};
|
|
77
95
|
},
|
|
@@ -101,8 +119,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
101
119
|
}
|
|
102
120
|
|
|
103
121
|
let results;
|
|
122
|
+
const dropped = urlList.length > caps.maxFetchUrls ? urlList.length - caps.maxFetchUrls : 0;
|
|
104
123
|
try {
|
|
105
|
-
results = await fetchAllContent(urlList, signal);
|
|
124
|
+
results = await fetchAllContent(urlList, signal, { allowPrivateNetwork }, caps.maxFetchUrls);
|
|
106
125
|
} catch (err) {
|
|
107
126
|
return {
|
|
108
127
|
content: [{ type: "text", text: `Error: ${errorMessage(err)}` }],
|
|
@@ -123,8 +142,9 @@ export default function (pi: ExtensionAPI) {
|
|
|
123
142
|
});
|
|
124
143
|
|
|
125
144
|
const body = truncate(sections.join("\n\n---\n\n"), caps.maxContentChars);
|
|
145
|
+
const note = skippedNote(dropped, caps.maxFetchUrls, "URL", "URLs");
|
|
126
146
|
return {
|
|
127
|
-
content: [{ type: "text", text: `${body}\n\n*responseId: ${id}*` }],
|
|
147
|
+
content: [{ type: "text", text: `${body}${note}\n\n*responseId: ${id}*` }],
|
|
128
148
|
details: { responseId: id, urlCount: results.length },
|
|
129
149
|
};
|
|
130
150
|
},
|
|
@@ -41,7 +41,12 @@ export async function searchGemini(
|
|
|
41
41
|
return { query, answer: "", results: [], error: errorMessage(err) };
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
let data: GeminiResponse;
|
|
45
|
+
try {
|
|
46
|
+
data = (await response.json()) as GeminiResponse;
|
|
47
|
+
} catch {
|
|
48
|
+
return { query, answer: "", results: [], error: `Gemini API error: HTTP ${response.status} (non-JSON response)` };
|
|
49
|
+
}
|
|
45
50
|
|
|
46
51
|
if (!response.ok) {
|
|
47
52
|
const msg = data.error?.message ?? `HTTP ${response.status}`;
|
|
@@ -39,7 +39,12 @@ export async function searchTavily(
|
|
|
39
39
|
return { query, answer: "", results: [], error: errorMessage(err) };
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
let data: TavilyResponse;
|
|
43
|
+
try {
|
|
44
|
+
data = (await response.json()) as TavilyResponse;
|
|
45
|
+
} catch {
|
|
46
|
+
return { query, answer: "", results: [], error: `Tavily API error: HTTP ${response.status} (non-JSON response)` };
|
|
47
|
+
}
|
|
43
48
|
|
|
44
49
|
if (!response.ok) {
|
|
45
50
|
return { query, answer: "", results: [], error: `Tavily API error: HTTP ${response.status}` };
|
|
@@ -23,8 +23,14 @@ export function storeResult(id: string, data: StoredData): void {
|
|
|
23
23
|
evictExpired();
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
export function getResult(id: string): StoredData | null {
|
|
27
|
-
|
|
26
|
+
export function getResult(id: string, now: number = Date.now()): StoredData | null {
|
|
27
|
+
const data = store.get(id);
|
|
28
|
+
if (!data) return null;
|
|
29
|
+
if (now - data.timestamp >= CACHE_TTL_MS) {
|
|
30
|
+
store.delete(id);
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
return data;
|
|
28
34
|
}
|
|
29
35
|
|
|
30
36
|
export function clearStore(): void {
|
|
@@ -54,8 +54,120 @@ export function isPdfContentType(contentType: string): boolean {
|
|
|
54
54
|
return contentType.toLowerCase().includes("application/pdf");
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
export interface HttpUrlOptions {
|
|
58
|
+
/** Allow private, link-local, and reserved network hosts (default false). Loopback is always allowed; cloud metadata is always blocked. */
|
|
59
|
+
allowPrivateNetwork?: boolean;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Well-known cloud metadata hostnames that resolve to instance-credential endpoints. */
|
|
63
|
+
const BLOCKED_METADATA_HOSTS = new Set([
|
|
64
|
+
"metadata",
|
|
65
|
+
"metadata.google.internal",
|
|
66
|
+
"instance-data",
|
|
67
|
+
"instance-data.ec2.internal",
|
|
68
|
+
]);
|
|
69
|
+
|
|
70
|
+
/** Cloud metadata IPv6 literals (e.g. AWS IMDS over IPv6). Always blocked. */
|
|
71
|
+
const BLOCKED_METADATA_V6 = new Set(["fd00:ec2::254"]);
|
|
72
|
+
|
|
73
|
+
type HostClass = "loopback" | "metadata" | "private" | "public";
|
|
74
|
+
|
|
75
|
+
/** Decode a bare unsigned 32-bit integer hostname (e.g. 2130706433) into octets. */
|
|
76
|
+
function ipv4FromInteger(n: number): number[] | null {
|
|
77
|
+
if (!Number.isInteger(n) || n < 0 || n > 0xffffffff) return null;
|
|
78
|
+
return [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Parse a host into IPv4 octets (dotted-quad or integer form), or null when not an IPv4 literal. */
|
|
82
|
+
function ipv4Octets(host: string): number[] | null {
|
|
83
|
+
if (/^\d+$/.test(host)) return ipv4FromInteger(Number(host));
|
|
84
|
+
const octets = host.split(".").map((part) => (/^\d+$/.test(part) ? Number(part) : NaN));
|
|
85
|
+
if (octets.length !== 4 || octets.some((o) => !Number.isInteger(o) || o < 0 || o > 255)) return null;
|
|
86
|
+
return octets;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function isLoopbackIpv4(o: number[]): boolean {
|
|
90
|
+
return o[0] === 127;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function isMetadataIpv4(o: number[]): boolean {
|
|
94
|
+
return (
|
|
95
|
+
(o[0] === 169 && o[1] === 254 && o[2] === 169 && o[3] === 254) || // AWS/Azure/GCP/Oracle
|
|
96
|
+
(o[0] === 100 && o[1] === 100 && o[2] === 100 && o[3] === 200) // Alibaba
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Non-globally-routable (blocked by default, opt-in via allowPrivateNetwork). */
|
|
101
|
+
function isPrivateIpv4(o: number[]): boolean {
|
|
102
|
+
const [a, b, c] = o;
|
|
103
|
+
return (
|
|
104
|
+
a === 0 || a === 10 || // 0.0.0.0/8, 10/8
|
|
105
|
+
(a === 100 && b >= 64 && b <= 127) || // 100.64/10 carrier-grade NAT
|
|
106
|
+
(a === 169 && b === 254) || // 169.254/16 link-local
|
|
107
|
+
(a === 172 && b >= 16 && b <= 31) || // 172.16/12
|
|
108
|
+
(a === 192 && b === 168) || // 192.168/16
|
|
109
|
+
(a === 192 && b === 0 && (c === 0 || c === 2)) || // 192.0.0/24, 192.0.2/24
|
|
110
|
+
(a === 198 && (b === 18 || b === 19)) || // 198.18/15 benchmark
|
|
111
|
+
(a === 198 && b === 51 && c === 100) || // 198.51.100/24
|
|
112
|
+
(a === 203 && b === 0 && c === 113) || // 203.0.113/24
|
|
113
|
+
a >= 224 // 224/4 multicast and 240/4 reserved
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Decode IPv4 from an IPv4-mapped IPv6 literal (::ffff:a.b.c.d or ::ffff:hex:hex). */
|
|
118
|
+
function ipv4MappedOctets(ip: string): number[] | null {
|
|
119
|
+
const dotted = ip.match(/^::ffff:(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
|
|
120
|
+
if (dotted) return dotted.slice(1).map(Number);
|
|
121
|
+
const hex = ip.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/);
|
|
122
|
+
if (hex) {
|
|
123
|
+
const n = (parseInt(hex[1], 16) << 16) | parseInt(hex[2], 16);
|
|
124
|
+
return [(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff];
|
|
125
|
+
}
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function classifyIpv4Octets(o: number[]): HostClass {
|
|
130
|
+
if (isLoopbackIpv4(o)) return "loopback";
|
|
131
|
+
if (isMetadataIpv4(o)) return "metadata";
|
|
132
|
+
if (isPrivateIpv4(o)) return "private";
|
|
133
|
+
return "public";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function classifyIpv6Host(host: string): HostClass {
|
|
137
|
+
const ip = host.replace(/^\[/, "").replace(/\]$/, "").replace(/%.*$/, "").toLowerCase();
|
|
138
|
+
if (BLOCKED_METADATA_V6.has(ip)) return "metadata";
|
|
139
|
+
if (ip === "::1") return "loopback";
|
|
140
|
+
const mapped = ipv4MappedOctets(ip);
|
|
141
|
+
if (mapped) return classifyIpv4Octets(mapped);
|
|
142
|
+
if (ip === "::") return "private"; // unspecified
|
|
143
|
+
if (/^f[cd]/.test(ip)) return "private"; // fc00::/7 unique-local
|
|
144
|
+
if (/^fe[89ab]/.test(ip)) return "private"; // fe80::/10 link-local
|
|
145
|
+
if (/^ff/.test(ip)) return "private"; // ff00::/8 multicast
|
|
146
|
+
return "public";
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function classifyHost(host: string): HostClass {
|
|
150
|
+
const lowered = host.toLowerCase();
|
|
151
|
+
if (BLOCKED_METADATA_HOSTS.has(lowered)) return "metadata";
|
|
152
|
+
if (lowered === "localhost" || lowered.endsWith(".localhost")) return "loopback";
|
|
153
|
+
if (lowered.includes(":")) return classifyIpv6Host(lowered);
|
|
154
|
+
const octets = ipv4Octets(lowered);
|
|
155
|
+
return octets ? classifyIpv4Octets(octets) : "public";
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function isBlockedNetworkHost(host: string, allowPrivateNetwork: boolean): boolean {
|
|
159
|
+
const cls = classifyHost(host);
|
|
160
|
+
if (cls === "metadata") return true; // never toggleable
|
|
161
|
+
if (cls === "loopback" || cls === "public") return false;
|
|
162
|
+
return !allowPrivateNetwork; // private / link-local / reserved
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Normalize a URL, allowing only http/https. Loopback (localhost/127.0.0.0/8/::1) is
|
|
167
|
+
* allowed by default for local development; private/link-local/reserved ranges are blocked
|
|
168
|
+
* unless allowPrivateNetwork is set; cloud metadata is always blocked.
|
|
169
|
+
*/
|
|
170
|
+
export function assertHttpUrl(url: string, opts: HttpUrlOptions = {}): string {
|
|
59
171
|
let parsed: URL;
|
|
60
172
|
try {
|
|
61
173
|
parsed = new URL(url);
|
|
@@ -65,5 +177,8 @@ export function assertHttpUrl(url: string): string {
|
|
|
65
177
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
66
178
|
throw new Error(`Unsupported protocol "${parsed.protocol}//" — only http/https`);
|
|
67
179
|
}
|
|
180
|
+
if (isBlockedNetworkHost(parsed.hostname, opts.allowPrivateNetwork === true)) {
|
|
181
|
+
throw new Error(`Blocked network host "${parsed.hostname}" — private, link-local, reserved, and cloud-metadata URLs are not allowed (set allowPrivateNetwork to opt in to private ranges; metadata is always blocked)`);
|
|
182
|
+
}
|
|
68
183
|
return parsed.href;
|
|
69
184
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openlines/opl-pi-sht",
|
|
3
|
-
"version": "0.2.
|
|
4
|
-
"description": "opl-pi-sht (shouldHaveThat) Pi coding-agent extension kit: token-efficient context,
|
|
3
|
+
"version": "0.2.8",
|
|
4
|
+
"description": "opl-pi-sht (shouldHaveThat) Pi coding-agent extension kit: token-efficient context, tool and session safety, fingerprinted /init, model benchmarking, web access, and browser automation.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "https://github.com/linellazatin/opl-pi-sht.git"
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"test:opl-init": "bun test tests/opl-init-crawl.test.mjs tests/opl-init-fingerprint.test.mjs tests/opl-init-packet.test.mjs && OPL_EXTENSION=opl-init bun test tests/extension-smoke.test.mjs",
|
|
48
48
|
"test:opl-input": "bun test tests/opl-input-style.test.mjs && OPL_EXTENSION=opl-input bun test tests/extension-smoke.test.mjs",
|
|
49
49
|
"test:opl-modes": "bun test tests/opl-modes-helpers.test.mjs tests/opl-modes-lifecycle.test.mjs && OPL_EXTENSION=opl-modes bun test tests/extension-smoke.test.mjs",
|
|
50
|
-
"test:opl-questionnaire": "OPL_EXTENSION=opl-questionnaire bun test tests/extension-smoke.test.mjs",
|
|
50
|
+
"test:opl-questionnaire": "bun test tests/opl-questionnaire-helpers.test.mjs && OPL_EXTENSION=opl-questionnaire bun test tests/extension-smoke.test.mjs",
|
|
51
51
|
"test:opl-todo": "bun test tests/opl-todo-config.test.mjs tests/opl-todo-render.test.mjs && OPL_EXTENSION=opl-todo bun test tests/extension-smoke.test.mjs",
|
|
52
52
|
"test:opl-webaccess": "bun test tests/opl-webaccess-helpers.test.mjs && OPL_EXTENSION=opl-webaccess bun test tests/extension-smoke.test.mjs",
|
|
53
53
|
"test:opl-simplebench": "bun test tests/opl-simplebench.test.ts && OPL_EXTENSION=opl-simplebench bun test tests/extension-smoke.test.mjs",
|