@oh-my-pi/pi-coding-agent 16.4.2 → 16.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/cli.js +17514 -17799
- package/dist/types/advisor/config.d.ts +4 -3
- package/dist/types/commit/agentic/agent.d.ts +1 -0
- package/dist/types/config/settings-schema.d.ts +28 -0
- package/dist/types/extensibility/plugins/legacy-pi-compat.d.ts +10 -22
- package/dist/types/lsp/deferred-diagnostics.d.ts +11 -0
- package/dist/types/modes/components/move-overlay.d.ts +1 -1
- package/dist/types/modes/components/status-line/component.d.ts +3 -0
- package/dist/types/modes/components/status-line/types.d.ts +3 -0
- package/dist/types/modes/interactive-mode.d.ts +8 -0
- package/dist/types/modes/types.d.ts +2 -0
- package/dist/types/session/agent-session.d.ts +16 -0
- package/dist/types/task/executor.d.ts +31 -0
- package/dist/types/tools/__tests__/vibe-render.test.d.ts +1 -0
- package/dist/types/tools/browser/cmux/cmux-tab.d.ts +3 -1
- package/dist/types/tools/browser/cmux/rpc.d.ts +20 -0
- package/dist/types/tools/browser/run-output.d.ts +25 -0
- package/dist/types/tools/browser/tab-worker.d.ts +16 -0
- package/dist/types/tools/index.d.ts +1 -0
- package/dist/types/tools/vibe.d.ts +161 -0
- package/dist/types/utils/changelog.d.ts +35 -1
- package/dist/types/vibe/runtime.d.ts +124 -0
- package/dist/types/vibe/state.d.ts +4 -0
- package/dist/types/web/search/provider.d.ts +2 -0
- package/dist/types/web/search/providers/bing.d.ts +14 -0
- package/dist/types/web/search/providers/browser-headers.d.ts +9 -0
- package/dist/types/web/search/providers/browser-page.d.ts +32 -0
- package/dist/types/web/search/providers/ecosia.d.ts +14 -0
- package/dist/types/web/search/providers/google.d.ts +13 -0
- package/dist/types/web/search/providers/mojeek.d.ts +14 -0
- package/dist/types/web/search/providers/public.d.ts +37 -0
- package/dist/types/web/search/providers/startpage.d.ts +14 -0
- package/dist/types/web/search/providers/yahoo.d.ts +14 -0
- package/dist/types/web/search/types.d.ts +28 -0
- package/package.json +15 -18
- package/scripts/build-binary.ts +56 -73
- package/scripts/bundle-dist.ts +36 -40
- package/scripts/compile-binary.ts +68 -0
- package/scripts/generate-docs-index.ts +3 -93
- package/scripts/legacy-pi-virtual-module.ts +192 -0
- package/src/advisor/__tests__/advisor.test.ts +13 -0
- package/src/advisor/__tests__/config.test.ts +36 -0
- package/src/advisor/config.ts +11 -8
- package/src/commit/agentic/agent.ts +4 -0
- package/src/commit/agentic/index.ts +46 -21
- package/src/edit/index.ts +10 -76
- package/src/exec/non-interactive-env.ts +0 -1
- package/src/extensibility/plugins/legacy-pi-compat.ts +328 -162
- package/src/extensibility/plugins/legacy-pi-virtual-modules.d.ts +4 -0
- package/src/internal-urls/docs-index.ts +2 -1
- package/src/internal-urls/skill-protocol.ts +1 -1
- package/src/lsp/deferred-diagnostics.ts +66 -0
- package/src/main.ts +13 -13
- package/src/mcp/transports/stdio.test.ts +45 -1
- package/src/mcp/transports/stdio.ts +6 -3
- package/src/modes/acp/acp-agent.ts +65 -1
- package/src/modes/acp/acp-event-mapper.ts +5 -0
- package/src/modes/acp/acp-mode.ts +11 -0
- package/src/modes/components/__tests__/move-overlay.test.ts +16 -1
- package/src/modes/components/advisor-config.ts +15 -7
- package/src/modes/components/move-overlay.ts +2 -3
- package/src/modes/components/status-line/component.ts +6 -0
- package/src/modes/components/status-line/segments.ts +6 -0
- package/src/modes/components/status-line/types.ts +3 -0
- package/src/modes/controllers/command-controller.ts +8 -10
- package/src/modes/interactive-mode.ts +117 -1
- package/src/modes/types.ts +2 -0
- package/src/prompts/system/eager-task.md +2 -2
- package/src/prompts/system/system-prompt.md +9 -2
- package/src/prompts/system/vibe-mode-active.md +23 -0
- package/src/prompts/tools/browser.md +3 -3
- package/src/prompts/tools/grep.md +1 -1
- package/src/prompts/tools/vibe-kill.md +3 -0
- package/src/prompts/tools/vibe-list.md +3 -0
- package/src/prompts/tools/vibe-send.md +9 -0
- package/src/prompts/tools/vibe-spawn.md +10 -0
- package/src/prompts/tools/vibe-turn-result.md +19 -0
- package/src/prompts/tools/vibe-wait.md +8 -0
- package/src/sdk.ts +5 -0
- package/src/session/agent-session.ts +103 -13
- package/src/session/snapcompact-inline.ts +3 -19
- package/src/slash-commands/builtin-registry.ts +24 -8
- package/src/task/agents.ts +0 -2
- package/src/task/executor.ts +105 -0
- package/src/tools/__tests__/vibe-render.test.ts +210 -0
- package/src/tools/bash-skill-urls.ts +1 -1
- package/src/tools/browser/cmux/cmux-tab.ts +46 -48
- package/src/tools/browser/cmux/rpc.ts +50 -0
- package/src/tools/browser/run-output.ts +76 -0
- package/src/tools/browser/tab-worker.ts +264 -129
- package/src/tools/glob.ts +20 -6
- package/src/tools/index.ts +1 -0
- package/src/tools/read.ts +17 -9
- package/src/tools/renderers.ts +6 -0
- package/src/tools/vibe.ts +608 -0
- package/src/tools/write.ts +15 -2
- package/src/utils/changelog.ts +106 -9
- package/src/utils/git.ts +0 -1
- package/src/utils/title-generator.ts +70 -7
- package/src/vibe/runtime.ts +710 -0
- package/src/vibe/state.ts +4 -0
- package/src/web/search/index.ts +14 -6
- package/src/web/search/provider.ts +37 -1
- package/src/web/search/providers/bing.ts +197 -0
- package/src/web/search/providers/browser-headers.ts +92 -0
- package/src/web/search/providers/browser-page.ts +123 -0
- package/src/web/search/providers/duckduckgo.ts +13 -34
- package/src/web/search/providers/ecosia.ts +178 -0
- package/src/web/search/providers/google.ts +193 -0
- package/src/web/search/providers/mojeek.ts +206 -0
- package/src/web/search/providers/public.ts +201 -0
- package/src/web/search/providers/startpage.ts +213 -0
- package/src/web/search/providers/yahoo.ts +179 -0
- package/src/web/search/types.ts +35 -0
- package/dist/types/extensibility/plugins/legacy-pi-bundled-keys.d.ts +0 -10
- package/dist/types/extensibility/plugins/legacy-pi-bundled-registry.d.ts +0 -10
- package/scripts/generate-legacy-pi-bundled-registry.ts +0 -420
- package/src/extensibility/plugins/legacy-pi-bundled-keys.ts +0 -1011
- package/src/extensibility/plugins/legacy-pi-bundled-registry.ts +0 -3430
- package/src/internal-urls/docs-index.generated.txt +0 -2
- package/src/prompts/agents/plan.md +0 -47
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
|
2
|
+
import { formatSearchProviderFailures, getSearchProvider, isSearchProviderExcluded } from "../provider";
|
|
3
|
+
import type { SearchProviderId, SearchResponse, SearchSource } from "../types";
|
|
4
|
+
import { SearchProviderError } from "../types";
|
|
5
|
+
import { clampNumResults } from "../utils";
|
|
6
|
+
import type { SearchParams } from "./base";
|
|
7
|
+
import { SearchProvider } from "./base";
|
|
8
|
+
import { withHardTimeout } from "./utils";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Credential-free engines the Public Web aggregate fans out to. Order is the
|
|
12
|
+
* tiebreak for merged ranking (earlier engines win equal consensus/rank), so
|
|
13
|
+
* engines with the best ranking quality when they answer come first:
|
|
14
|
+
* Google-index engines (startpage, google) lead, Bing-backed scrapers follow,
|
|
15
|
+
* and Mojeek's independent index breaks remaining ties (measured 2026-07).
|
|
16
|
+
*/
|
|
17
|
+
const PUBLIC_ENGINE_IDS = [
|
|
18
|
+
"startpage",
|
|
19
|
+
"google",
|
|
20
|
+
"duckduckgo",
|
|
21
|
+
"bing",
|
|
22
|
+
"yahoo",
|
|
23
|
+
"ecosia",
|
|
24
|
+
"mojeek",
|
|
25
|
+
] as const satisfies readonly SearchProviderId[];
|
|
26
|
+
|
|
27
|
+
/** Aggregates get a wider default window than single engines: consensus needs breadth. */
|
|
28
|
+
const DEFAULT_NUM_RESULTS = 15;
|
|
29
|
+
const MAX_NUM_RESULTS = 30;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Soft deadline for the fan-out: past this point the aggregate returns as
|
|
33
|
+
* soon as it has at least one engine's results. Fast HTML engines answer
|
|
34
|
+
* well under this; browser-backed engines (google, ecosia, mojeek) routinely
|
|
35
|
+
* exceed it and are treated as bonus coverage rather than latency floor.
|
|
36
|
+
*/
|
|
37
|
+
const SOFT_DEADLINE_MS = 5_000;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Hard deadline for the fan-out: the aggregate returns whatever it has, even
|
|
41
|
+
* nothing, so one pathologically slow engine can never pin the tool call to
|
|
42
|
+
* the per-request 60s ceiling.
|
|
43
|
+
*/
|
|
44
|
+
const HARD_DEADLINE_MS = 30_000;
|
|
45
|
+
|
|
46
|
+
/** Deadline overrides — test seam; production callers use the defaults. */
|
|
47
|
+
export interface PublicWebDeadlines {
|
|
48
|
+
softMs?: number;
|
|
49
|
+
hardMs?: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Accumulator for one deduplicated URL across engines. */
|
|
53
|
+
interface MergedSource {
|
|
54
|
+
source: SearchSource;
|
|
55
|
+
/** Number of engines that returned this URL — the primary ranking signal. */
|
|
56
|
+
engines: number;
|
|
57
|
+
/** Best (lowest) per-engine rank observed. */
|
|
58
|
+
bestRank: number;
|
|
59
|
+
/** First-seen insertion index; final tiebreak keeps ordering deterministic. */
|
|
60
|
+
order: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Canonical dedup key for a result URL: case-normalized host without a
|
|
65
|
+
* leading `www.`, path without a trailing slash, query preserved, fragment
|
|
66
|
+
* dropped. Engines disagree on exactly these variations for the same page.
|
|
67
|
+
*/
|
|
68
|
+
function dedupKey(rawUrl: string): string {
|
|
69
|
+
try {
|
|
70
|
+
const url = new URL(rawUrl);
|
|
71
|
+
const host = url.hostname.toLowerCase().replace(/^www\./, "");
|
|
72
|
+
let path = url.pathname;
|
|
73
|
+
if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
|
|
74
|
+
return `${host}${path}${url.search}`;
|
|
75
|
+
} catch {
|
|
76
|
+
return rawUrl;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Merge one engine's ranked sources into the accumulator map. */
|
|
81
|
+
function mergeSources(merged: Map<string, MergedSource>, sources: readonly SearchSource[]): void {
|
|
82
|
+
for (const [rank, source] of sources.entries()) {
|
|
83
|
+
const key = dedupKey(source.url);
|
|
84
|
+
const existing = merged.get(key);
|
|
85
|
+
if (!existing) {
|
|
86
|
+
merged.set(key, { source: { ...source }, engines: 1, bestRank: rank, order: merged.size });
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
existing.engines += 1;
|
|
90
|
+
if (rank < existing.bestRank) {
|
|
91
|
+
existing.bestRank = rank;
|
|
92
|
+
existing.source.title = source.title;
|
|
93
|
+
existing.source.url = source.url;
|
|
94
|
+
}
|
|
95
|
+
// Keep the most informative snippet regardless of which engine ranked it best.
|
|
96
|
+
if (source.snippet && source.snippet.length > (existing.source.snippet?.length ?? 0)) {
|
|
97
|
+
existing.source.snippet = source.snippet;
|
|
98
|
+
}
|
|
99
|
+
existing.source.publishedDate ??= source.publishedDate;
|
|
100
|
+
existing.source.ageSeconds ??= source.ageSeconds;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Execute a web search against every credential-free engine in parallel and
|
|
106
|
+
* consolidate the results: URLs are deduplicated across engines, ranked by
|
|
107
|
+
* cross-engine consensus (how many engines returned them), then by best
|
|
108
|
+
* per-engine rank.
|
|
109
|
+
*
|
|
110
|
+
* The fan-out races three exits and returns at the earliest: every engine
|
|
111
|
+
* settled; the soft deadline elapsed with at least one success in hand; the
|
|
112
|
+
* hard deadline elapsed regardless. If the soft deadline fires before any
|
|
113
|
+
* engine has delivered, the aggregate keeps waiting (up to the hard cap) for
|
|
114
|
+
* the first success, so a slow field degrades to fewer engines rather than
|
|
115
|
+
* an empty answer. Stragglers are aborted once the race resolves. Individual
|
|
116
|
+
* engine failures (bot challenges, timeouts) are tolerated; the call fails
|
|
117
|
+
* only when every engine fails.
|
|
118
|
+
*/
|
|
119
|
+
export async function searchPublicWeb(
|
|
120
|
+
params: SearchParams,
|
|
121
|
+
deadlines: PublicWebDeadlines = {},
|
|
122
|
+
): Promise<SearchResponse> {
|
|
123
|
+
const softMs = deadlines.softMs ?? SOFT_DEADLINE_MS;
|
|
124
|
+
const hardMs = deadlines.hardMs ?? HARD_DEADLINE_MS;
|
|
125
|
+
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
|
126
|
+
const engineIds = PUBLIC_ENGINE_IDS.filter(id => !isSearchProviderExcluded(id));
|
|
127
|
+
if (engineIds.length === 0) {
|
|
128
|
+
throw new SearchProviderError("public", "Every credential-free engine is excluded by settings.", 400);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Each engine composes its own per-request ceiling on top of the shared
|
|
132
|
+
// hard deadline; the straggler controller lets the aggregate cancel
|
|
133
|
+
// still-running engines once it decides to return.
|
|
134
|
+
const straggler = new AbortController();
|
|
135
|
+
const signal = AbortSignal.any([withHardTimeout(params.signal), straggler.signal]);
|
|
136
|
+
|
|
137
|
+
const responses: (SearchResponse | undefined)[] = new Array(engineIds.length);
|
|
138
|
+
const failures: { provider: { id: SearchProviderId; label: string }; error: unknown }[] = [];
|
|
139
|
+
const firstSuccess = Promise.withResolvers<void>();
|
|
140
|
+
const all = Promise.all(
|
|
141
|
+
engineIds.map(async (id, index) => {
|
|
142
|
+
try {
|
|
143
|
+
const provider = await getSearchProvider(id);
|
|
144
|
+
responses[index] = await provider.search({ ...params, signal });
|
|
145
|
+
firstSuccess.resolve();
|
|
146
|
+
} catch (error) {
|
|
147
|
+
failures.push({ provider: { id, label: id }, error });
|
|
148
|
+
}
|
|
149
|
+
}),
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
await Promise.race([all, Bun.sleep(softMs)]);
|
|
153
|
+
if (!responses.some(response => response !== undefined) && failures.length < engineIds.length) {
|
|
154
|
+
await Promise.race([all, firstSuccess.promise, Bun.sleep(Math.max(0, hardMs - softMs))]);
|
|
155
|
+
}
|
|
156
|
+
straggler.abort();
|
|
157
|
+
|
|
158
|
+
// Merge in engine-priority order (not settlement order) so ranking
|
|
159
|
+
// tiebreaks stay deterministic.
|
|
160
|
+
const merged = new Map<string, MergedSource>();
|
|
161
|
+
for (const response of responses) {
|
|
162
|
+
if (response) mergeSources(merged, response.sources);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
if (merged.size === 0 && failures.length === engineIds.length) {
|
|
166
|
+
throw new SearchProviderError(
|
|
167
|
+
"public",
|
|
168
|
+
`All public engines failed: ${formatSearchProviderFailures(failures)}`,
|
|
169
|
+
503,
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const sources = [...merged.values()]
|
|
174
|
+
.sort((a, b) => b.engines - a.engines || a.bestRank - b.bestRank || a.order - b.order)
|
|
175
|
+
.slice(0, numResults)
|
|
176
|
+
.map(entry => entry.source);
|
|
177
|
+
|
|
178
|
+
return { provider: "public", sources };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Aggregate meta-provider over every credential-free engine. Explicit-only:
|
|
183
|
+
* the auto chain already walks the individual engines sequentially, so
|
|
184
|
+
* fanning out to all of them is a deliberate user choice, not a fallback.
|
|
185
|
+
*/
|
|
186
|
+
export class PublicWebProvider extends SearchProvider {
|
|
187
|
+
readonly id = "public";
|
|
188
|
+
readonly label = "Public Web";
|
|
189
|
+
|
|
190
|
+
isAvailable(_authStorage: AuthStorage): boolean {
|
|
191
|
+
return false;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
|
|
195
|
+
return true;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
search(params: SearchParams): Promise<SearchResponse> {
|
|
199
|
+
return searchPublicWeb(params);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
4
|
+
import { SearchProviderError } from "../../../web/search/types";
|
|
5
|
+
import { clampNumResults } from "../utils";
|
|
6
|
+
import type { SearchParams } from "./base";
|
|
7
|
+
import { SearchProvider } from "./base";
|
|
8
|
+
import type { LoadedHtmlPage } from "./browser-page";
|
|
9
|
+
import { browserFetch } from "./browser-page";
|
|
10
|
+
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Startpage proxies Google's index behind a privacy frontend and serves fully
|
|
14
|
+
* server-rendered result pages — no JS challenge on the happy path. Its bot
|
|
15
|
+
* defense keys on requests that skip the homepage handshake: the search form
|
|
16
|
+
* carries a session token (`sc`) plus sibling hidden inputs, and posting the
|
|
17
|
+
* form with a stale/absent token 302s to the `/en/errors/` CAPTCHA shell.
|
|
18
|
+
* The robust flow is therefore the same dance a real browser performs: GET
|
|
19
|
+
* the homepage, lift the form's hidden inputs, POST them back with the query.
|
|
20
|
+
*/
|
|
21
|
+
const STARTPAGE_HOME_URL = "https://www.startpage.com/";
|
|
22
|
+
const STARTPAGE_SEARCH_URL = "https://www.startpage.com/sp/search";
|
|
23
|
+
const DEFAULT_NUM_RESULTS = 10;
|
|
24
|
+
const MAX_NUM_RESULTS = 20;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Recency → Startpage `with_date` param. Accepts single letters; an absent
|
|
28
|
+
* value returns the unfiltered default.
|
|
29
|
+
*/
|
|
30
|
+
const RECENCY_TO_STARTPAGE_WITH_DATE: Record<NonNullable<SearchParams["recency"]>, string> = {
|
|
31
|
+
day: "d",
|
|
32
|
+
week: "w",
|
|
33
|
+
month: "m",
|
|
34
|
+
year: "y",
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** One organic result lifted from the Startpage results page. */
|
|
38
|
+
interface ParsedResult {
|
|
39
|
+
title: string;
|
|
40
|
+
url: string;
|
|
41
|
+
snippet?: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function normalizeText(value: string | null | undefined): string {
|
|
45
|
+
return (value ?? "").replace(/\s+/g, " ").trim();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* `true` when Startpage answered with its CAPTCHA/error shell instead of
|
|
50
|
+
* results. Rejected requests 302 to `/en/errors/` (legacy: `/sp/captcha`), a
|
|
51
|
+
* Gatsby SPA whose chunk map names the captcha page components; the body
|
|
52
|
+
* marker matters because mocked fetch responses carry no final URL. A bare
|
|
53
|
+
* "captcha" substring is deliberately not used — result snippets for
|
|
54
|
+
* captcha-related queries would false-positive.
|
|
55
|
+
*/
|
|
56
|
+
function isChallengeResponse(page: LoadedHtmlPage): boolean {
|
|
57
|
+
if (/\/(?:errors|captcha)\//.test(page.url) || page.url.includes("/sp/captcha")) return true;
|
|
58
|
+
return page.html.includes("component---src-pages-captcha") || page.html.includes("/sp/captcha");
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Lift the hidden inputs from the homepage's `/sp/search` form. Returns
|
|
63
|
+
* `undefined` when the form or its `sc` anti-bot token cannot be found so the
|
|
64
|
+
* caller can degrade to a tokenless GET instead of posting a doomed form.
|
|
65
|
+
*/
|
|
66
|
+
function parseSearchFormInputs(html: string): Record<string, string> | undefined {
|
|
67
|
+
const { document } = parseHTML(html);
|
|
68
|
+
const form = document.querySelector('form[action="/sp/search"]');
|
|
69
|
+
if (!form) return undefined;
|
|
70
|
+
const inputs: Record<string, string> = {};
|
|
71
|
+
for (const input of form.querySelectorAll('input[type="hidden"]')) {
|
|
72
|
+
const name = input.getAttribute("name");
|
|
73
|
+
if (name) inputs[name] = input.getAttribute("value") ?? "";
|
|
74
|
+
}
|
|
75
|
+
return inputs.sc ? inputs : undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Accept only http(s) result targets that point away from Startpage itself. */
|
|
79
|
+
function sanitizeResultUrl(href: string | null | undefined): string | undefined {
|
|
80
|
+
if (!href) return undefined;
|
|
81
|
+
let url: URL;
|
|
82
|
+
try {
|
|
83
|
+
url = new URL(href, STARTPAGE_HOME_URL);
|
|
84
|
+
} catch {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
|
|
88
|
+
if (url.hostname === "startpage.com" || url.hostname.endsWith(".startpage.com")) return undefined;
|
|
89
|
+
return url.href;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Walk the server-rendered results page in document order.
|
|
94
|
+
*
|
|
95
|
+
* Each organic hit lives in a `div.result` container holding the title
|
|
96
|
+
* anchor `a.result-link` (with an `h2.wgl-title` heading) and an optional
|
|
97
|
+
* `p.description` snippet. Hrefs are direct target URLs — Startpage does not
|
|
98
|
+
* wrap outbound clicks. The offscreen adblock-honeypot div uses the class
|
|
99
|
+
* token `a-bg-result`, which a CSS class selector correctly ignores, and
|
|
100
|
+
* sponsored placements render outside `div.result` containers.
|
|
101
|
+
*/
|
|
102
|
+
function parseHtmlResults(html: string): ParsedResult[] {
|
|
103
|
+
const { document } = parseHTML(html);
|
|
104
|
+
const results: ParsedResult[] = [];
|
|
105
|
+
for (const block of document.querySelectorAll("div.result")) {
|
|
106
|
+
const anchor = block.querySelector("a.result-link");
|
|
107
|
+
if (!anchor) continue;
|
|
108
|
+
const url = sanitizeResultUrl(anchor.getAttribute("href"));
|
|
109
|
+
if (!url) continue;
|
|
110
|
+
const title = normalizeText(anchor.querySelector("h2, h3")?.textContent ?? anchor.textContent);
|
|
111
|
+
if (!title) continue;
|
|
112
|
+
const snippet = normalizeText(block.querySelector("p.description")?.textContent);
|
|
113
|
+
results.push({ title, url, snippet: snippet || undefined });
|
|
114
|
+
}
|
|
115
|
+
return results;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Fetch the homepage and lift the search form's hidden inputs. Best effort:
|
|
120
|
+
* any failure (network, non-OK status, challenge shell, markup drift) yields
|
|
121
|
+
* `undefined` and the caller falls back to a direct GET.
|
|
122
|
+
*/
|
|
123
|
+
async function fetchFormInputs(fetchImpl: FetchImpl, signal: AbortSignal): Promise<Record<string, string> | undefined> {
|
|
124
|
+
let page: LoadedHtmlPage;
|
|
125
|
+
try {
|
|
126
|
+
page = await browserFetch(STARTPAGE_HOME_URL, { fetch: fetchImpl, signal });
|
|
127
|
+
} catch (error) {
|
|
128
|
+
if (signal.aborted) throw error;
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
if (page.status < 200 || page.status >= 300 || isChallengeResponse(page)) return undefined;
|
|
132
|
+
return parseSearchFormInputs(page.html);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function callStartpageHtml(params: SearchParams): Promise<string> {
|
|
136
|
+
const fetchImpl = params.fetch ?? fetch;
|
|
137
|
+
const signal = withHardTimeout(params.signal);
|
|
138
|
+
const withDate = params.recency ? RECENCY_TO_STARTPAGE_WITH_DATE[params.recency] : undefined;
|
|
139
|
+
|
|
140
|
+
const formInputs = await fetchFormInputs(fetchImpl, signal);
|
|
141
|
+
let page: LoadedHtmlPage;
|
|
142
|
+
if (formInputs) {
|
|
143
|
+
const form = new URLSearchParams(formInputs);
|
|
144
|
+
form.set("query", params.query);
|
|
145
|
+
if (withDate) form.set("with_date", withDate);
|
|
146
|
+
page = await browserFetch(STARTPAGE_SEARCH_URL, {
|
|
147
|
+
fetch: fetchImpl,
|
|
148
|
+
signal,
|
|
149
|
+
referer: STARTPAGE_HOME_URL,
|
|
150
|
+
init: { method: "POST", body: form.toString() },
|
|
151
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
152
|
+
});
|
|
153
|
+
} else {
|
|
154
|
+
const url = new URL(STARTPAGE_SEARCH_URL);
|
|
155
|
+
url.searchParams.set("query", params.query);
|
|
156
|
+
if (withDate) url.searchParams.set("with_date", withDate);
|
|
157
|
+
page = await browserFetch(url.href, {
|
|
158
|
+
fetch: fetchImpl,
|
|
159
|
+
signal,
|
|
160
|
+
referer: STARTPAGE_HOME_URL,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
if (isChallengeResponse(page)) {
|
|
165
|
+
throw new SearchProviderError(
|
|
166
|
+
"startpage",
|
|
167
|
+
"Startpage blocked the request with a CAPTCHA challenge. Startpage rate-limits automated searches from datacenter/shared-egress IPs; try another provider such as DuckDuckGo or Mojeek, or retry later.",
|
|
168
|
+
429,
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
if (page.status < 200 || page.status >= 300) {
|
|
172
|
+
const classified = classifyProviderHttpError("startpage", page.status, page.html);
|
|
173
|
+
if (classified) throw classified;
|
|
174
|
+
throw new SearchProviderError("startpage", `Startpage HTML error (${page.status})`, page.status);
|
|
175
|
+
}
|
|
176
|
+
return page.html;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Execute a Startpage web search via the homepage-token form flow. */
|
|
180
|
+
export async function searchStartpage(params: SearchParams): Promise<SearchResponse> {
|
|
181
|
+
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
|
182
|
+
const html = await callStartpageHtml(params);
|
|
183
|
+
const parsed = parseHtmlResults(html);
|
|
184
|
+
|
|
185
|
+
const sources: SearchSource[] = [];
|
|
186
|
+
const seen = new Set<string>();
|
|
187
|
+
for (const result of parsed) {
|
|
188
|
+
if (seen.has(result.url)) continue;
|
|
189
|
+
seen.add(result.url);
|
|
190
|
+
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
|
191
|
+
if (sources.length >= numResults) break;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return { provider: "startpage", sources };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Search provider for Startpage (no API key required). */
|
|
198
|
+
export class StartpageProvider extends SearchProvider {
|
|
199
|
+
readonly id = "startpage";
|
|
200
|
+
readonly label = "Startpage";
|
|
201
|
+
|
|
202
|
+
isAvailable(_authStorage: AuthStorage): boolean {
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
search(params: SearchParams): Promise<SearchResponse> {
|
|
211
|
+
return searchStartpage(params);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import type { AuthStorage } from "@oh-my-pi/pi-ai";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
4
|
+
import { SearchProviderError } from "../../../web/search/types";
|
|
5
|
+
import { clampNumResults } from "../utils";
|
|
6
|
+
import type { SearchParams } from "./base";
|
|
7
|
+
import { SearchProvider } from "./base";
|
|
8
|
+
import { browserFetch } from "./browser-page";
|
|
9
|
+
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Yahoo Search's server-rendered results page. A plain GET with browser
|
|
13
|
+
* navigation headers returns the full SERP without any JavaScript challenge,
|
|
14
|
+
* so no headless-browser fallback is needed (verified live 2026-07).
|
|
15
|
+
*/
|
|
16
|
+
const YAHOO_HOME_URL = "https://search.yahoo.com/";
|
|
17
|
+
const YAHOO_SEARCH_URL = "https://search.yahoo.com/search";
|
|
18
|
+
const DEFAULT_NUM_RESULTS = 10;
|
|
19
|
+
const MAX_NUM_RESULTS = 20;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Recency → Yahoo `btf` query param. Yahoo's time filter only offers
|
|
23
|
+
* day/week/month; `year` has no equivalent and is silently dropped per the
|
|
24
|
+
* {@link SearchParams.recency} contract.
|
|
25
|
+
*/
|
|
26
|
+
const RECENCY_TO_YAHOO_BTF: Partial<Record<NonNullable<SearchParams["recency"]>, string>> = {
|
|
27
|
+
day: "d",
|
|
28
|
+
week: "w",
|
|
29
|
+
month: "m",
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
interface ParsedResult {
|
|
33
|
+
title: string;
|
|
34
|
+
url: string;
|
|
35
|
+
snippet?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Resolve a Yahoo result href back to the underlying target URL.
|
|
40
|
+
*
|
|
41
|
+
* Organic hrefs are wrapped through the click tracker
|
|
42
|
+
* `https://r.search.yahoo.com/_ylt=…/RU=<percent-encoded-target>/RK=…/RS=…`;
|
|
43
|
+
* the `/RU=` path segment carries the destination. Older layouts emit plain
|
|
44
|
+
* absolute hrefs, so both shapes are handled. Tracker links without a
|
|
45
|
+
* recoverable target and Yahoo-internal navigation are rejected.
|
|
46
|
+
*/
|
|
47
|
+
function unwrapResultUrl(href: string): string | undefined {
|
|
48
|
+
let url: URL;
|
|
49
|
+
try {
|
|
50
|
+
url = new URL(href, YAHOO_HOME_URL);
|
|
51
|
+
} catch {
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
|
|
55
|
+
|
|
56
|
+
const wrapped = /\/RU=([^/]+)/.exec(url.pathname);
|
|
57
|
+
if (wrapped) {
|
|
58
|
+
let target: string;
|
|
59
|
+
try {
|
|
60
|
+
target = decodeURIComponent(wrapped[1]);
|
|
61
|
+
} catch {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
return target.startsWith("http://") || target.startsWith("https://") ? target : undefined;
|
|
65
|
+
}
|
|
66
|
+
// A tracker link without an RU segment has no recoverable destination.
|
|
67
|
+
if (url.hostname === "r.search.yahoo.com") return undefined;
|
|
68
|
+
// Relative hrefs resolve against the search host: internal navigation.
|
|
69
|
+
if (url.hostname === "search.yahoo.com") return undefined;
|
|
70
|
+
return url.href;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Walk the SERP and pull organic result blocks in document order.
|
|
75
|
+
*
|
|
76
|
+
* Organics render as `<div class="… algo …">` blocks (inside `#web`'s
|
|
77
|
+
* `<ol>`): the title `<h3>` sits inside the tracker `<a>` in the current
|
|
78
|
+
* layout, while legacy layouts nested the `<a>` inside `<h3 class="title">`
|
|
79
|
+
* — both are handled. The preview text lives in a sibling
|
|
80
|
+
* `<div class="compText">`. Module headers ("Videos", "People also ask")
|
|
81
|
+
* carry `<h3>`s outside `.algo` blocks and are excluded by construction.
|
|
82
|
+
*/
|
|
83
|
+
function parseHtmlResults(html: string): ParsedResult[] {
|
|
84
|
+
const { document } = parseHTML(html);
|
|
85
|
+
const results: ParsedResult[] = [];
|
|
86
|
+
for (const block of document.querySelectorAll("div.algo")) {
|
|
87
|
+
const heading = block.querySelector("h3");
|
|
88
|
+
if (!heading) continue;
|
|
89
|
+
const anchor = heading.querySelector("a") ?? heading.closest("a");
|
|
90
|
+
const href = anchor?.getAttribute("href");
|
|
91
|
+
if (!href) continue;
|
|
92
|
+
const url = unwrapResultUrl(href);
|
|
93
|
+
if (!url) continue;
|
|
94
|
+
const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim();
|
|
95
|
+
if (!title) continue;
|
|
96
|
+
const snippet = (block.querySelector(".compText")?.textContent ?? "").replace(/\s+/g, " ").trim() || undefined;
|
|
97
|
+
results.push({ title, url, snippet });
|
|
98
|
+
}
|
|
99
|
+
return results;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* `true` when Yahoo answered with its EU consent interstitial instead of
|
|
104
|
+
* results: either the request was redirected to consent.yahoo.com /
|
|
105
|
+
* guce.yahoo.com, or the body carries the consent form. The normal SERP
|
|
106
|
+
* mentions guce.yahoo.com only in a meta tag, so detection keys on the
|
|
107
|
+
* consent-host redirect and the `collectConsent` form action.
|
|
108
|
+
*/
|
|
109
|
+
function isConsentInterstitial(finalUrl: string, html: string): boolean {
|
|
110
|
+
if (/^https?:\/\/(?:[^/]*\.)?(?:consent|guce)\.yahoo\.com\//i.test(finalUrl)) return true;
|
|
111
|
+
return html.includes("consent.yahoo.com") || html.includes("collectConsent");
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function callYahooHtml(params: SearchParams, numResults: number): Promise<string> {
|
|
115
|
+
const url = new URL(YAHOO_SEARCH_URL);
|
|
116
|
+
url.searchParams.set("p", params.query);
|
|
117
|
+
url.searchParams.set("n", String(numResults));
|
|
118
|
+
const btf = params.recency ? RECENCY_TO_YAHOO_BTF[params.recency] : undefined;
|
|
119
|
+
if (btf) url.searchParams.set("btf", btf);
|
|
120
|
+
|
|
121
|
+
const page = await browserFetch(url.href, {
|
|
122
|
+
fetch: params.fetch ?? fetch,
|
|
123
|
+
signal: withHardTimeout(params.signal),
|
|
124
|
+
referer: YAHOO_HOME_URL,
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
const body = page.html;
|
|
128
|
+
if (page.status < 200 || page.status >= 300) {
|
|
129
|
+
const classified = classifyProviderHttpError("yahoo", page.status, body);
|
|
130
|
+
if (classified) throw classified;
|
|
131
|
+
throw new SearchProviderError("yahoo", `Yahoo HTML error (${page.status})`, page.status);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if (isConsentInterstitial(page.url, body)) {
|
|
135
|
+
throw new SearchProviderError(
|
|
136
|
+
"yahoo",
|
|
137
|
+
"Yahoo served its GDPR consent interstitial instead of search results. This typically affects EU egress IPs; use another web search provider such as DuckDuckGo, Brave, or Mojeek.",
|
|
138
|
+
429,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
return body;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Execute a Yahoo web search via the server-rendered HTML results page. */
|
|
146
|
+
export async function searchYahoo(params: SearchParams): Promise<SearchResponse> {
|
|
147
|
+
const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
|
148
|
+
const html = await callYahooHtml(params, numResults);
|
|
149
|
+
const parsed = parseHtmlResults(html);
|
|
150
|
+
|
|
151
|
+
const sources: SearchSource[] = [];
|
|
152
|
+
const seen = new Set<string>();
|
|
153
|
+
for (const result of parsed) {
|
|
154
|
+
if (seen.has(result.url)) continue;
|
|
155
|
+
seen.add(result.url);
|
|
156
|
+
sources.push({ title: result.title, url: result.url, snippet: result.snippet });
|
|
157
|
+
if (sources.length >= numResults) break;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return { provider: "yahoo", sources };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Search provider for Yahoo (no API key required). */
|
|
164
|
+
export class YahooProvider extends SearchProvider {
|
|
165
|
+
readonly id = "yahoo";
|
|
166
|
+
readonly label = "Yahoo";
|
|
167
|
+
|
|
168
|
+
isAvailable(_authStorage: AuthStorage): boolean {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
search(params: SearchParams): Promise<SearchResponse> {
|
|
177
|
+
return searchYahoo(params);
|
|
178
|
+
}
|
|
179
|
+
}
|
package/src/web/search/types.ts
CHANGED
|
@@ -43,11 +43,46 @@ export const SEARCH_PROVIDER_OPTIONS = [
|
|
|
43
43
|
{ value: "parallel", label: "Parallel", description: "Requires PARALLEL_API_KEY" },
|
|
44
44
|
{ value: "synthetic", label: "Synthetic", description: "Requires SYNTHETIC_API_KEY" },
|
|
45
45
|
{ value: "searxng", label: "SearXNG", description: "Requires SEARXNG_ENDPOINT or searxng.endpoint" },
|
|
46
|
+
{
|
|
47
|
+
value: "startpage",
|
|
48
|
+
label: "Startpage",
|
|
49
|
+
description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged",
|
|
50
|
+
},
|
|
46
51
|
{
|
|
47
52
|
value: "duckduckgo",
|
|
48
53
|
label: "DuckDuckGo",
|
|
49
54
|
description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs",
|
|
50
55
|
},
|
|
56
|
+
{
|
|
57
|
+
value: "bing",
|
|
58
|
+
label: "Bing",
|
|
59
|
+
description: "Credential-free HTML scrape of Bing results; may be bot-challenged",
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
value: "yahoo",
|
|
63
|
+
label: "Yahoo",
|
|
64
|
+
description: "Credential-free HTML scrape of Yahoo (Bing-backed) results",
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
value: "ecosia",
|
|
68
|
+
label: "Ecosia",
|
|
69
|
+
description: "Credential-free browser-backed scrape of Ecosia (Google-backed) results",
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
value: "google",
|
|
73
|
+
label: "Google",
|
|
74
|
+
description: "Credential-free browser-backed fallback; slower and may be bot-challenged",
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
value: "mojeek",
|
|
78
|
+
label: "Mojeek",
|
|
79
|
+
description: "Credential-free browser-backed scrape of Mojeek's independent index",
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
value: "public",
|
|
83
|
+
label: "Public Web",
|
|
84
|
+
description: "Queries every credential-free engine in parallel and consolidates deduplicated results",
|
|
85
|
+
},
|
|
51
86
|
] as const;
|
|
52
87
|
|
|
53
88
|
/** Supported web search providers (every option except `auto`). */
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Canonical keys served by the `omp-legacy-pi-bundled:` virtual namespace.
|
|
3
|
-
* Mirrors `Object.keys(BUNDLED_PI_REGISTRY)` from
|
|
4
|
-
* `legacy-pi-bundled-keys.ts`'s sibling registry file. `legacy-pi-compat.ts`
|
|
5
|
-
* statically imports this set to seed `LEGACY_PI_PACKAGE_ROOT_OVERRIDES` in
|
|
6
|
-
* compiled-binary mode without dragging the heavy registry's transitive
|
|
7
|
-
* graph into dev/test runs (the registry itself stays behind a dynamic
|
|
8
|
-
* import — see `ensureBundledRegistryLoaded` in `legacy-pi-compat.ts`).
|
|
9
|
-
*/
|
|
10
|
-
export declare const BUNDLED_PI_REGISTRY_KEYS: ReadonlySet<string>;
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Canonical specifier → live module namespace. Keys MUST match the right-hand
|
|
3
|
-
* side of `bundledRegistryVirtualSpecifier(...)` calls in
|
|
4
|
-
* `legacy-pi-compat.ts`; the synthesizer enumerates each namespace's own
|
|
5
|
-
* enumerable exports at extension load time. The companion
|
|
6
|
-
* `legacy-pi-bundled-keys.ts` mirrors `Object.keys(BUNDLED_PI_REGISTRY)` and
|
|
7
|
-
* is statically imported by `legacy-pi-compat.ts` to seed the override map
|
|
8
|
-
* without paying the cascade above.
|
|
9
|
-
*/
|
|
10
|
-
export declare const BUNDLED_PI_REGISTRY: Readonly<Record<string, Readonly<Record<string, unknown>>>>;
|