@oh-my-pi/pi-coding-agent 16.4.2 → 16.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/dist/cli.js +17514 -17799
  3. package/dist/types/advisor/config.d.ts +4 -3
  4. package/dist/types/commit/agentic/agent.d.ts +1 -0
  5. package/dist/types/config/settings-schema.d.ts +28 -0
  6. package/dist/types/extensibility/plugins/legacy-pi-compat.d.ts +10 -22
  7. package/dist/types/lsp/deferred-diagnostics.d.ts +11 -0
  8. package/dist/types/modes/components/move-overlay.d.ts +1 -1
  9. package/dist/types/modes/components/status-line/component.d.ts +3 -0
  10. package/dist/types/modes/components/status-line/types.d.ts +3 -0
  11. package/dist/types/modes/interactive-mode.d.ts +8 -0
  12. package/dist/types/modes/types.d.ts +2 -0
  13. package/dist/types/session/agent-session.d.ts +16 -0
  14. package/dist/types/task/executor.d.ts +31 -0
  15. package/dist/types/tools/__tests__/vibe-render.test.d.ts +1 -0
  16. package/dist/types/tools/browser/cmux/cmux-tab.d.ts +3 -1
  17. package/dist/types/tools/browser/cmux/rpc.d.ts +20 -0
  18. package/dist/types/tools/browser/run-output.d.ts +25 -0
  19. package/dist/types/tools/browser/tab-worker.d.ts +16 -0
  20. package/dist/types/tools/index.d.ts +1 -0
  21. package/dist/types/tools/vibe.d.ts +161 -0
  22. package/dist/types/utils/changelog.d.ts +35 -1
  23. package/dist/types/vibe/runtime.d.ts +124 -0
  24. package/dist/types/vibe/state.d.ts +4 -0
  25. package/dist/types/web/search/provider.d.ts +2 -0
  26. package/dist/types/web/search/providers/bing.d.ts +14 -0
  27. package/dist/types/web/search/providers/browser-headers.d.ts +9 -0
  28. package/dist/types/web/search/providers/browser-page.d.ts +32 -0
  29. package/dist/types/web/search/providers/ecosia.d.ts +14 -0
  30. package/dist/types/web/search/providers/google.d.ts +13 -0
  31. package/dist/types/web/search/providers/mojeek.d.ts +14 -0
  32. package/dist/types/web/search/providers/public.d.ts +37 -0
  33. package/dist/types/web/search/providers/startpage.d.ts +14 -0
  34. package/dist/types/web/search/providers/yahoo.d.ts +14 -0
  35. package/dist/types/web/search/types.d.ts +28 -0
  36. package/package.json +15 -18
  37. package/scripts/build-binary.ts +56 -73
  38. package/scripts/bundle-dist.ts +36 -40
  39. package/scripts/compile-binary.ts +68 -0
  40. package/scripts/generate-docs-index.ts +3 -93
  41. package/scripts/legacy-pi-virtual-module.ts +192 -0
  42. package/src/advisor/__tests__/advisor.test.ts +13 -0
  43. package/src/advisor/__tests__/config.test.ts +36 -0
  44. package/src/advisor/config.ts +11 -8
  45. package/src/commit/agentic/agent.ts +4 -0
  46. package/src/commit/agentic/index.ts +46 -21
  47. package/src/edit/index.ts +10 -76
  48. package/src/exec/non-interactive-env.ts +0 -1
  49. package/src/extensibility/plugins/legacy-pi-compat.ts +328 -162
  50. package/src/extensibility/plugins/legacy-pi-virtual-modules.d.ts +4 -0
  51. package/src/internal-urls/docs-index.ts +2 -1
  52. package/src/internal-urls/skill-protocol.ts +1 -1
  53. package/src/lsp/deferred-diagnostics.ts +66 -0
  54. package/src/main.ts +13 -13
  55. package/src/mcp/transports/stdio.test.ts +45 -1
  56. package/src/mcp/transports/stdio.ts +6 -3
  57. package/src/modes/acp/acp-agent.ts +65 -1
  58. package/src/modes/acp/acp-event-mapper.ts +5 -0
  59. package/src/modes/acp/acp-mode.ts +11 -0
  60. package/src/modes/components/__tests__/move-overlay.test.ts +16 -1
  61. package/src/modes/components/advisor-config.ts +15 -7
  62. package/src/modes/components/move-overlay.ts +2 -3
  63. package/src/modes/components/status-line/component.ts +6 -0
  64. package/src/modes/components/status-line/segments.ts +6 -0
  65. package/src/modes/components/status-line/types.ts +3 -0
  66. package/src/modes/controllers/command-controller.ts +8 -10
  67. package/src/modes/interactive-mode.ts +117 -1
  68. package/src/modes/types.ts +2 -0
  69. package/src/prompts/system/eager-task.md +2 -2
  70. package/src/prompts/system/system-prompt.md +9 -2
  71. package/src/prompts/system/vibe-mode-active.md +23 -0
  72. package/src/prompts/tools/browser.md +3 -3
  73. package/src/prompts/tools/grep.md +1 -1
  74. package/src/prompts/tools/vibe-kill.md +3 -0
  75. package/src/prompts/tools/vibe-list.md +3 -0
  76. package/src/prompts/tools/vibe-send.md +9 -0
  77. package/src/prompts/tools/vibe-spawn.md +10 -0
  78. package/src/prompts/tools/vibe-turn-result.md +19 -0
  79. package/src/prompts/tools/vibe-wait.md +8 -0
  80. package/src/sdk.ts +5 -0
  81. package/src/session/agent-session.ts +103 -13
  82. package/src/session/snapcompact-inline.ts +3 -19
  83. package/src/slash-commands/builtin-registry.ts +24 -8
  84. package/src/task/agents.ts +0 -2
  85. package/src/task/executor.ts +105 -0
  86. package/src/tools/__tests__/vibe-render.test.ts +210 -0
  87. package/src/tools/bash-skill-urls.ts +1 -1
  88. package/src/tools/browser/cmux/cmux-tab.ts +46 -48
  89. package/src/tools/browser/cmux/rpc.ts +50 -0
  90. package/src/tools/browser/run-output.ts +76 -0
  91. package/src/tools/browser/tab-worker.ts +264 -129
  92. package/src/tools/glob.ts +20 -6
  93. package/src/tools/index.ts +1 -0
  94. package/src/tools/read.ts +17 -9
  95. package/src/tools/renderers.ts +6 -0
  96. package/src/tools/vibe.ts +608 -0
  97. package/src/tools/write.ts +15 -2
  98. package/src/utils/changelog.ts +106 -9
  99. package/src/utils/git.ts +0 -1
  100. package/src/utils/title-generator.ts +70 -7
  101. package/src/vibe/runtime.ts +710 -0
  102. package/src/vibe/state.ts +4 -0
  103. package/src/web/search/index.ts +14 -6
  104. package/src/web/search/provider.ts +37 -1
  105. package/src/web/search/providers/bing.ts +197 -0
  106. package/src/web/search/providers/browser-headers.ts +92 -0
  107. package/src/web/search/providers/browser-page.ts +123 -0
  108. package/src/web/search/providers/duckduckgo.ts +13 -34
  109. package/src/web/search/providers/ecosia.ts +178 -0
  110. package/src/web/search/providers/google.ts +193 -0
  111. package/src/web/search/providers/mojeek.ts +206 -0
  112. package/src/web/search/providers/public.ts +201 -0
  113. package/src/web/search/providers/startpage.ts +213 -0
  114. package/src/web/search/providers/yahoo.ts +179 -0
  115. package/src/web/search/types.ts +35 -0
  116. package/dist/types/extensibility/plugins/legacy-pi-bundled-keys.d.ts +0 -10
  117. package/dist/types/extensibility/plugins/legacy-pi-bundled-registry.d.ts +0 -10
  118. package/scripts/generate-legacy-pi-bundled-registry.ts +0 -420
  119. package/src/extensibility/plugins/legacy-pi-bundled-keys.ts +0 -1011
  120. package/src/extensibility/plugins/legacy-pi-bundled-registry.ts +0 -3430
  121. package/src/internal-urls/docs-index.generated.txt +0 -2
  122. package/src/prompts/agents/plan.md +0 -47
@@ -0,0 +1,201 @@
1
+ import type { AuthStorage } from "@oh-my-pi/pi-ai";
2
+ import { formatSearchProviderFailures, getSearchProvider, isSearchProviderExcluded } from "../provider";
3
+ import type { SearchProviderId, SearchResponse, SearchSource } from "../types";
4
+ import { SearchProviderError } from "../types";
5
+ import { clampNumResults } from "../utils";
6
+ import type { SearchParams } from "./base";
7
+ import { SearchProvider } from "./base";
8
+ import { withHardTimeout } from "./utils";
9
+
10
+ /**
11
+ * Credential-free engines the Public Web aggregate fans out to. Order is the
12
+ * tiebreak for merged ranking (earlier engines win equal consensus/rank), so
13
+ * engines with the best ranking quality when they answer come first:
14
+ * Google-index engines (startpage, google) lead, Bing-backed scrapers follow,
15
+ * and Mojeek's independent index breaks remaining ties (measured 2026-07).
16
+ */
17
+ const PUBLIC_ENGINE_IDS = [
18
+ "startpage",
19
+ "google",
20
+ "duckduckgo",
21
+ "bing",
22
+ "yahoo",
23
+ "ecosia",
24
+ "mojeek",
25
+ ] as const satisfies readonly SearchProviderId[];
26
+
27
+ /** Aggregates get a wider default window than single engines: consensus needs breadth. */
28
+ const DEFAULT_NUM_RESULTS = 15;
29
+ const MAX_NUM_RESULTS = 30;
30
+
31
+ /**
32
+ * Soft deadline for the fan-out: past this point the aggregate returns as
33
+ * soon as it has at least one engine's results. Fast HTML engines answer
34
+ * well under this; browser-backed engines (google, ecosia, mojeek) routinely
35
+ * exceed it and are treated as bonus coverage rather than latency floor.
36
+ */
37
+ const SOFT_DEADLINE_MS = 5_000;
38
+
39
+ /**
40
+ * Hard deadline for the fan-out: the aggregate returns whatever it has, even
41
+ * nothing, so one pathologically slow engine can never pin the tool call to
42
+ * the per-request 60s ceiling.
43
+ */
44
+ const HARD_DEADLINE_MS = 30_000;
45
+
46
+ /** Deadline overrides — test seam; production callers use the defaults. */
47
+ export interface PublicWebDeadlines {
48
+ softMs?: number;
49
+ hardMs?: number;
50
+ }
51
+
52
+ /** Accumulator for one deduplicated URL across engines. */
53
+ interface MergedSource {
54
+ source: SearchSource;
55
+ /** Number of engines that returned this URL — the primary ranking signal. */
56
+ engines: number;
57
+ /** Best (lowest) per-engine rank observed. */
58
+ bestRank: number;
59
+ /** First-seen insertion index; final tiebreak keeps ordering deterministic. */
60
+ order: number;
61
+ }
62
+
63
+ /**
64
+ * Canonical dedup key for a result URL: case-normalized host without a
65
+ * leading `www.`, path without a trailing slash, query preserved, fragment
66
+ * dropped. Engines disagree on exactly these variations for the same page.
67
+ */
68
+ function dedupKey(rawUrl: string): string {
69
+ try {
70
+ const url = new URL(rawUrl);
71
+ const host = url.hostname.toLowerCase().replace(/^www\./, "");
72
+ let path = url.pathname;
73
+ if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
74
+ return `${host}${path}${url.search}`;
75
+ } catch {
76
+ return rawUrl;
77
+ }
78
+ }
79
+
80
+ /** Merge one engine's ranked sources into the accumulator map. */
81
+ function mergeSources(merged: Map<string, MergedSource>, sources: readonly SearchSource[]): void {
82
+ for (const [rank, source] of sources.entries()) {
83
+ const key = dedupKey(source.url);
84
+ const existing = merged.get(key);
85
+ if (!existing) {
86
+ merged.set(key, { source: { ...source }, engines: 1, bestRank: rank, order: merged.size });
87
+ continue;
88
+ }
89
+ existing.engines += 1;
90
+ if (rank < existing.bestRank) {
91
+ existing.bestRank = rank;
92
+ existing.source.title = source.title;
93
+ existing.source.url = source.url;
94
+ }
95
+ // Keep the most informative snippet regardless of which engine ranked it best.
96
+ if (source.snippet && source.snippet.length > (existing.source.snippet?.length ?? 0)) {
97
+ existing.source.snippet = source.snippet;
98
+ }
99
+ existing.source.publishedDate ??= source.publishedDate;
100
+ existing.source.ageSeconds ??= source.ageSeconds;
101
+ }
102
+ }
103
+
104
+ /**
105
+ * Execute a web search against every credential-free engine in parallel and
106
+ * consolidate the results: URLs are deduplicated across engines, ranked by
107
+ * cross-engine consensus (how many engines returned them), then by best
108
+ * per-engine rank.
109
+ *
110
+ * The fan-out races three exits and returns at the earliest: every engine
111
+ * settled; the soft deadline elapsed with at least one success in hand; the
112
+ * hard deadline elapsed regardless. If the soft deadline fires before any
113
+ * engine has delivered, the aggregate keeps waiting (up to the hard cap) for
114
+ * the first success, so a slow field degrades to fewer engines rather than
115
+ * an empty answer. Stragglers are aborted once the race resolves. Individual
116
+ * engine failures (bot challenges, timeouts) are tolerated; the call fails
117
+ * only when every engine fails.
118
+ */
119
+ export async function searchPublicWeb(
120
+ params: SearchParams,
121
+ deadlines: PublicWebDeadlines = {},
122
+ ): Promise<SearchResponse> {
123
+ const softMs = deadlines.softMs ?? SOFT_DEADLINE_MS;
124
+ const hardMs = deadlines.hardMs ?? HARD_DEADLINE_MS;
125
+ const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
126
+ const engineIds = PUBLIC_ENGINE_IDS.filter(id => !isSearchProviderExcluded(id));
127
+ if (engineIds.length === 0) {
128
+ throw new SearchProviderError("public", "Every credential-free engine is excluded by settings.", 400);
129
+ }
130
+
131
+ // Each engine composes its own per-request ceiling on top of the shared
132
+ // hard deadline; the straggler controller lets the aggregate cancel
133
+ // still-running engines once it decides to return.
134
+ const straggler = new AbortController();
135
+ const signal = AbortSignal.any([withHardTimeout(params.signal), straggler.signal]);
136
+
137
+ const responses: (SearchResponse | undefined)[] = new Array(engineIds.length);
138
+ const failures: { provider: { id: SearchProviderId; label: string }; error: unknown }[] = [];
139
+ const firstSuccess = Promise.withResolvers<void>();
140
+ const all = Promise.all(
141
+ engineIds.map(async (id, index) => {
142
+ try {
143
+ const provider = await getSearchProvider(id);
144
+ responses[index] = await provider.search({ ...params, signal });
145
+ firstSuccess.resolve();
146
+ } catch (error) {
147
+ failures.push({ provider: { id, label: id }, error });
148
+ }
149
+ }),
150
+ );
151
+
152
+ await Promise.race([all, Bun.sleep(softMs)]);
153
+ if (!responses.some(response => response !== undefined) && failures.length < engineIds.length) {
154
+ await Promise.race([all, firstSuccess.promise, Bun.sleep(Math.max(0, hardMs - softMs))]);
155
+ }
156
+ straggler.abort();
157
+
158
+ // Merge in engine-priority order (not settlement order) so ranking
159
+ // tiebreaks stay deterministic.
160
+ const merged = new Map<string, MergedSource>();
161
+ for (const response of responses) {
162
+ if (response) mergeSources(merged, response.sources);
163
+ }
164
+
165
+ if (merged.size === 0 && failures.length === engineIds.length) {
166
+ throw new SearchProviderError(
167
+ "public",
168
+ `All public engines failed: ${formatSearchProviderFailures(failures)}`,
169
+ 503,
170
+ );
171
+ }
172
+
173
+ const sources = [...merged.values()]
174
+ .sort((a, b) => b.engines - a.engines || a.bestRank - b.bestRank || a.order - b.order)
175
+ .slice(0, numResults)
176
+ .map(entry => entry.source);
177
+
178
+ return { provider: "public", sources };
179
+ }
180
+
181
+ /**
182
+ * Aggregate meta-provider over every credential-free engine. Explicit-only:
183
+ * the auto chain already walks the individual engines sequentially, so
184
+ * fanning out to all of them is a deliberate user choice, not a fallback.
185
+ */
186
+ export class PublicWebProvider extends SearchProvider {
187
+ readonly id = "public";
188
+ readonly label = "Public Web";
189
+
190
+ isAvailable(_authStorage: AuthStorage): boolean {
191
+ return false;
192
+ }
193
+
194
+ isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
195
+ return true;
196
+ }
197
+
198
+ search(params: SearchParams): Promise<SearchResponse> {
199
+ return searchPublicWeb(params);
200
+ }
201
+ }
@@ -0,0 +1,213 @@
1
+ import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
2
+ import { parseHTML } from "linkedom";
3
+ import type { SearchResponse, SearchSource } from "../../../web/search/types";
4
+ import { SearchProviderError } from "../../../web/search/types";
5
+ import { clampNumResults } from "../utils";
6
+ import type { SearchParams } from "./base";
7
+ import { SearchProvider } from "./base";
8
+ import type { LoadedHtmlPage } from "./browser-page";
9
+ import { browserFetch } from "./browser-page";
10
+ import { classifyProviderHttpError, withHardTimeout } from "./utils";
11
+
12
+ /**
13
+ * Startpage proxies Google's index behind a privacy frontend and serves fully
14
+ * server-rendered result pages — no JS challenge on the happy path. Its bot
15
+ * defense keys on requests that skip the homepage handshake: the search form
16
+ * carries a session token (`sc`) plus sibling hidden inputs, and posting the
17
+ * form with a stale/absent token 302s to the `/en/errors/` CAPTCHA shell.
18
+ * The robust flow is therefore the same dance a real browser performs: GET
19
+ * the homepage, lift the form's hidden inputs, POST them back with the query.
20
+ */
21
+ const STARTPAGE_HOME_URL = "https://www.startpage.com/";
22
+ const STARTPAGE_SEARCH_URL = "https://www.startpage.com/sp/search";
23
+ const DEFAULT_NUM_RESULTS = 10;
24
+ const MAX_NUM_RESULTS = 20;
25
+
26
+ /**
27
+ * Recency → Startpage `with_date` param. Accepts single letters; an absent
28
+ * value returns the unfiltered default.
29
+ */
30
+ const RECENCY_TO_STARTPAGE_WITH_DATE: Record<NonNullable<SearchParams["recency"]>, string> = {
31
+ day: "d",
32
+ week: "w",
33
+ month: "m",
34
+ year: "y",
35
+ };
36
+
37
+ /** One organic result lifted from the Startpage results page. */
38
+ interface ParsedResult {
39
+ title: string;
40
+ url: string;
41
+ snippet?: string;
42
+ }
43
+
44
+ function normalizeText(value: string | null | undefined): string {
45
+ return (value ?? "").replace(/\s+/g, " ").trim();
46
+ }
47
+
48
+ /**
49
+ * `true` when Startpage answered with its CAPTCHA/error shell instead of
50
+ * results. Rejected requests 302 to `/en/errors/` (legacy: `/sp/captcha`), a
51
+ * Gatsby SPA whose chunk map names the captcha page components; the body
52
+ * marker matters because mocked fetch responses carry no final URL. A bare
53
+ * "captcha" substring is deliberately not used — result snippets for
54
+ * captcha-related queries would false-positive.
55
+ */
56
+ function isChallengeResponse(page: LoadedHtmlPage): boolean {
57
+ if (/\/(?:errors|captcha)\//.test(page.url) || page.url.includes("/sp/captcha")) return true;
58
+ return page.html.includes("component---src-pages-captcha") || page.html.includes("/sp/captcha");
59
+ }
60
+
61
+ /**
62
+ * Lift the hidden inputs from the homepage's `/sp/search` form. Returns
63
+ * `undefined` when the form or its `sc` anti-bot token cannot be found so the
64
+ * caller can degrade to a tokenless GET instead of posting a doomed form.
65
+ */
66
+ function parseSearchFormInputs(html: string): Record<string, string> | undefined {
67
+ const { document } = parseHTML(html);
68
+ const form = document.querySelector('form[action="/sp/search"]');
69
+ if (!form) return undefined;
70
+ const inputs: Record<string, string> = {};
71
+ for (const input of form.querySelectorAll('input[type="hidden"]')) {
72
+ const name = input.getAttribute("name");
73
+ if (name) inputs[name] = input.getAttribute("value") ?? "";
74
+ }
75
+ return inputs.sc ? inputs : undefined;
76
+ }
77
+
78
+ /** Accept only http(s) result targets that point away from Startpage itself. */
79
+ function sanitizeResultUrl(href: string | null | undefined): string | undefined {
80
+ if (!href) return undefined;
81
+ let url: URL;
82
+ try {
83
+ url = new URL(href, STARTPAGE_HOME_URL);
84
+ } catch {
85
+ return undefined;
86
+ }
87
+ if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
88
+ if (url.hostname === "startpage.com" || url.hostname.endsWith(".startpage.com")) return undefined;
89
+ return url.href;
90
+ }
91
+
92
+ /**
93
+ * Walk the server-rendered results page in document order.
94
+ *
95
+ * Each organic hit lives in a `div.result` container holding the title
96
+ * anchor `a.result-link` (with an `h2.wgl-title` heading) and an optional
97
+ * `p.description` snippet. Hrefs are direct target URLs — Startpage does not
98
+ * wrap outbound clicks. The offscreen adblock-honeypot div uses the class
99
+ * token `a-bg-result`, which a CSS class selector correctly ignores, and
100
+ * sponsored placements render outside `div.result` containers.
101
+ */
102
+ function parseHtmlResults(html: string): ParsedResult[] {
103
+ const { document } = parseHTML(html);
104
+ const results: ParsedResult[] = [];
105
+ for (const block of document.querySelectorAll("div.result")) {
106
+ const anchor = block.querySelector("a.result-link");
107
+ if (!anchor) continue;
108
+ const url = sanitizeResultUrl(anchor.getAttribute("href"));
109
+ if (!url) continue;
110
+ const title = normalizeText(anchor.querySelector("h2, h3")?.textContent ?? anchor.textContent);
111
+ if (!title) continue;
112
+ const snippet = normalizeText(block.querySelector("p.description")?.textContent);
113
+ results.push({ title, url, snippet: snippet || undefined });
114
+ }
115
+ return results;
116
+ }
117
+
118
+ /**
119
+ * Fetch the homepage and lift the search form's hidden inputs. Best effort:
120
+ * any failure (network, non-OK status, challenge shell, markup drift) yields
121
+ * `undefined` and the caller falls back to a direct GET.
122
+ */
123
+ async function fetchFormInputs(fetchImpl: FetchImpl, signal: AbortSignal): Promise<Record<string, string> | undefined> {
124
+ let page: LoadedHtmlPage;
125
+ try {
126
+ page = await browserFetch(STARTPAGE_HOME_URL, { fetch: fetchImpl, signal });
127
+ } catch (error) {
128
+ if (signal.aborted) throw error;
129
+ return undefined;
130
+ }
131
+ if (page.status < 200 || page.status >= 300 || isChallengeResponse(page)) return undefined;
132
+ return parseSearchFormInputs(page.html);
133
+ }
134
+
135
+ async function callStartpageHtml(params: SearchParams): Promise<string> {
136
+ const fetchImpl = params.fetch ?? fetch;
137
+ const signal = withHardTimeout(params.signal);
138
+ const withDate = params.recency ? RECENCY_TO_STARTPAGE_WITH_DATE[params.recency] : undefined;
139
+
140
+ const formInputs = await fetchFormInputs(fetchImpl, signal);
141
+ let page: LoadedHtmlPage;
142
+ if (formInputs) {
143
+ const form = new URLSearchParams(formInputs);
144
+ form.set("query", params.query);
145
+ if (withDate) form.set("with_date", withDate);
146
+ page = await browserFetch(STARTPAGE_SEARCH_URL, {
147
+ fetch: fetchImpl,
148
+ signal,
149
+ referer: STARTPAGE_HOME_URL,
150
+ init: { method: "POST", body: form.toString() },
151
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
152
+ });
153
+ } else {
154
+ const url = new URL(STARTPAGE_SEARCH_URL);
155
+ url.searchParams.set("query", params.query);
156
+ if (withDate) url.searchParams.set("with_date", withDate);
157
+ page = await browserFetch(url.href, {
158
+ fetch: fetchImpl,
159
+ signal,
160
+ referer: STARTPAGE_HOME_URL,
161
+ });
162
+ }
163
+
164
+ if (isChallengeResponse(page)) {
165
+ throw new SearchProviderError(
166
+ "startpage",
167
+ "Startpage blocked the request with a CAPTCHA challenge. Startpage rate-limits automated searches from datacenter/shared-egress IPs; try another provider such as DuckDuckGo or Mojeek, or retry later.",
168
+ 429,
169
+ );
170
+ }
171
+ if (page.status < 200 || page.status >= 300) {
172
+ const classified = classifyProviderHttpError("startpage", page.status, page.html);
173
+ if (classified) throw classified;
174
+ throw new SearchProviderError("startpage", `Startpage HTML error (${page.status})`, page.status);
175
+ }
176
+ return page.html;
177
+ }
178
+
179
+ /** Execute a Startpage web search via the homepage-token form flow. */
180
+ export async function searchStartpage(params: SearchParams): Promise<SearchResponse> {
181
+ const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
182
+ const html = await callStartpageHtml(params);
183
+ const parsed = parseHtmlResults(html);
184
+
185
+ const sources: SearchSource[] = [];
186
+ const seen = new Set<string>();
187
+ for (const result of parsed) {
188
+ if (seen.has(result.url)) continue;
189
+ seen.add(result.url);
190
+ sources.push({ title: result.title, url: result.url, snippet: result.snippet });
191
+ if (sources.length >= numResults) break;
192
+ }
193
+
194
+ return { provider: "startpage", sources };
195
+ }
196
+
197
+ /** Search provider for Startpage (no API key required). */
198
+ export class StartpageProvider extends SearchProvider {
199
+ readonly id = "startpage";
200
+ readonly label = "Startpage";
201
+
202
+ isAvailable(_authStorage: AuthStorage): boolean {
203
+ return true;
204
+ }
205
+
206
+ isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
207
+ return true;
208
+ }
209
+
210
+ search(params: SearchParams): Promise<SearchResponse> {
211
+ return searchStartpage(params);
212
+ }
213
+ }
@@ -0,0 +1,179 @@
1
+ import type { AuthStorage } from "@oh-my-pi/pi-ai";
2
+ import { parseHTML } from "linkedom";
3
+ import type { SearchResponse, SearchSource } from "../../../web/search/types";
4
+ import { SearchProviderError } from "../../../web/search/types";
5
+ import { clampNumResults } from "../utils";
6
+ import type { SearchParams } from "./base";
7
+ import { SearchProvider } from "./base";
8
+ import { browserFetch } from "./browser-page";
9
+ import { classifyProviderHttpError, withHardTimeout } from "./utils";
10
+
11
+ /**
12
+ * Yahoo Search's server-rendered results page. A plain GET with browser
13
+ * navigation headers returns the full SERP without any JavaScript challenge,
14
+ * so no headless-browser fallback is needed (verified live 2026-07).
15
+ */
16
+ const YAHOO_HOME_URL = "https://search.yahoo.com/";
17
+ const YAHOO_SEARCH_URL = "https://search.yahoo.com/search";
18
+ const DEFAULT_NUM_RESULTS = 10;
19
+ const MAX_NUM_RESULTS = 20;
20
+
21
+ /**
22
+ * Recency → Yahoo `btf` query param. Yahoo's time filter only offers
23
+ * day/week/month; `year` has no equivalent and is silently dropped per the
24
+ * {@link SearchParams.recency} contract.
25
+ */
26
+ const RECENCY_TO_YAHOO_BTF: Partial<Record<NonNullable<SearchParams["recency"]>, string>> = {
27
+ day: "d",
28
+ week: "w",
29
+ month: "m",
30
+ };
31
+
32
+ interface ParsedResult {
33
+ title: string;
34
+ url: string;
35
+ snippet?: string;
36
+ }
37
+
38
+ /**
39
+ * Resolve a Yahoo result href back to the underlying target URL.
40
+ *
41
+ * Organic hrefs are wrapped through the click tracker
42
+ * `https://r.search.yahoo.com/_ylt=…/RU=<percent-encoded-target>/RK=…/RS=…`;
43
+ * the `/RU=` path segment carries the destination. Older layouts emit plain
44
+ * absolute hrefs, so both shapes are handled. Tracker links without a
45
+ * recoverable target and Yahoo-internal navigation are rejected.
46
+ */
47
+ function unwrapResultUrl(href: string): string | undefined {
48
+ let url: URL;
49
+ try {
50
+ url = new URL(href, YAHOO_HOME_URL);
51
+ } catch {
52
+ return undefined;
53
+ }
54
+ if (url.protocol !== "http:" && url.protocol !== "https:") return undefined;
55
+
56
+ const wrapped = /\/RU=([^/]+)/.exec(url.pathname);
57
+ if (wrapped) {
58
+ let target: string;
59
+ try {
60
+ target = decodeURIComponent(wrapped[1]);
61
+ } catch {
62
+ return undefined;
63
+ }
64
+ return target.startsWith("http://") || target.startsWith("https://") ? target : undefined;
65
+ }
66
+ // A tracker link without an RU segment has no recoverable destination.
67
+ if (url.hostname === "r.search.yahoo.com") return undefined;
68
+ // Relative hrefs resolve against the search host: internal navigation.
69
+ if (url.hostname === "search.yahoo.com") return undefined;
70
+ return url.href;
71
+ }
72
+
73
+ /**
74
+ * Walk the SERP and pull organic result blocks in document order.
75
+ *
76
+ * Organics render as `<div class="… algo …">` blocks (inside `#web`'s
77
+ * `<ol>`): the title `<h3>` sits inside the tracker `<a>` in the current
78
+ * layout, while legacy layouts nested the `<a>` inside `<h3 class="title">`
79
+ * — both are handled. The preview text lives in a sibling
80
+ * `<div class="compText">`. Module headers ("Videos", "People also ask")
81
+ * carry `<h3>`s outside `.algo` blocks and are excluded by construction.
82
+ */
83
+ function parseHtmlResults(html: string): ParsedResult[] {
84
+ const { document } = parseHTML(html);
85
+ const results: ParsedResult[] = [];
86
+ for (const block of document.querySelectorAll("div.algo")) {
87
+ const heading = block.querySelector("h3");
88
+ if (!heading) continue;
89
+ const anchor = heading.querySelector("a") ?? heading.closest("a");
90
+ const href = anchor?.getAttribute("href");
91
+ if (!href) continue;
92
+ const url = unwrapResultUrl(href);
93
+ if (!url) continue;
94
+ const title = (heading.textContent ?? "").replace(/\s+/g, " ").trim();
95
+ if (!title) continue;
96
+ const snippet = (block.querySelector(".compText")?.textContent ?? "").replace(/\s+/g, " ").trim() || undefined;
97
+ results.push({ title, url, snippet });
98
+ }
99
+ return results;
100
+ }
101
+
102
+ /**
103
+ * `true` when Yahoo answered with its EU consent interstitial instead of
104
+ * results: either the request was redirected to consent.yahoo.com /
105
+ * guce.yahoo.com, or the body carries the consent form. The normal SERP
106
+ * mentions guce.yahoo.com only in a meta tag, so detection keys on the
107
+ * consent-host redirect and the `collectConsent` form action.
108
+ */
109
+ function isConsentInterstitial(finalUrl: string, html: string): boolean {
110
+ if (/^https?:\/\/(?:[^/]*\.)?(?:consent|guce)\.yahoo\.com\//i.test(finalUrl)) return true;
111
+ return html.includes("consent.yahoo.com") || html.includes("collectConsent");
112
+ }
113
+
114
+ async function callYahooHtml(params: SearchParams, numResults: number): Promise<string> {
115
+ const url = new URL(YAHOO_SEARCH_URL);
116
+ url.searchParams.set("p", params.query);
117
+ url.searchParams.set("n", String(numResults));
118
+ const btf = params.recency ? RECENCY_TO_YAHOO_BTF[params.recency] : undefined;
119
+ if (btf) url.searchParams.set("btf", btf);
120
+
121
+ const page = await browserFetch(url.href, {
122
+ fetch: params.fetch ?? fetch,
123
+ signal: withHardTimeout(params.signal),
124
+ referer: YAHOO_HOME_URL,
125
+ });
126
+
127
+ const body = page.html;
128
+ if (page.status < 200 || page.status >= 300) {
129
+ const classified = classifyProviderHttpError("yahoo", page.status, body);
130
+ if (classified) throw classified;
131
+ throw new SearchProviderError("yahoo", `Yahoo HTML error (${page.status})`, page.status);
132
+ }
133
+
134
+ if (isConsentInterstitial(page.url, body)) {
135
+ throw new SearchProviderError(
136
+ "yahoo",
137
+ "Yahoo served its GDPR consent interstitial instead of search results. This typically affects EU egress IPs; use another web search provider such as DuckDuckGo, Brave, or Mojeek.",
138
+ 429,
139
+ );
140
+ }
141
+
142
+ return body;
143
+ }
144
+
145
+ /** Execute a Yahoo web search via the server-rendered HTML results page. */
146
+ export async function searchYahoo(params: SearchParams): Promise<SearchResponse> {
147
+ const numResults = clampNumResults(params.numSearchResults ?? params.limit, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
148
+ const html = await callYahooHtml(params, numResults);
149
+ const parsed = parseHtmlResults(html);
150
+
151
+ const sources: SearchSource[] = [];
152
+ const seen = new Set<string>();
153
+ for (const result of parsed) {
154
+ if (seen.has(result.url)) continue;
155
+ seen.add(result.url);
156
+ sources.push({ title: result.title, url: result.url, snippet: result.snippet });
157
+ if (sources.length >= numResults) break;
158
+ }
159
+
160
+ return { provider: "yahoo", sources };
161
+ }
162
+
163
+ /** Search provider for Yahoo (no API key required). */
164
+ export class YahooProvider extends SearchProvider {
165
+ readonly id = "yahoo";
166
+ readonly label = "Yahoo";
167
+
168
+ isAvailable(_authStorage: AuthStorage): boolean {
169
+ return true;
170
+ }
171
+
172
+ isExplicitlyAvailable(_authStorage: AuthStorage): boolean {
173
+ return true;
174
+ }
175
+
176
+ search(params: SearchParams): Promise<SearchResponse> {
177
+ return searchYahoo(params);
178
+ }
179
+ }
@@ -43,11 +43,46 @@ export const SEARCH_PROVIDER_OPTIONS = [
43
43
  { value: "parallel", label: "Parallel", description: "Requires PARALLEL_API_KEY" },
44
44
  { value: "synthetic", label: "Synthetic", description: "Requires SYNTHETIC_API_KEY" },
45
45
  { value: "searxng", label: "SearXNG", description: "Requires SEARXNG_ENDPOINT or searxng.endpoint" },
46
+ {
47
+ value: "startpage",
48
+ label: "Startpage",
49
+ description: "Credential-free scrape of Startpage (Google-backed) results; may be bot-challenged",
50
+ },
46
51
  {
47
52
  value: "duckduckgo",
48
53
  label: "DuckDuckGo",
49
54
  description: "Credential-free best-effort fallback; may be bot-challenged on datacenter/shared-egress IPs",
50
55
  },
56
+ {
57
+ value: "bing",
58
+ label: "Bing",
59
+ description: "Credential-free HTML scrape of Bing results; may be bot-challenged",
60
+ },
61
+ {
62
+ value: "yahoo",
63
+ label: "Yahoo",
64
+ description: "Credential-free HTML scrape of Yahoo (Bing-backed) results",
65
+ },
66
+ {
67
+ value: "ecosia",
68
+ label: "Ecosia",
69
+ description: "Credential-free browser-backed scrape of Ecosia (Google-backed) results",
70
+ },
71
+ {
72
+ value: "google",
73
+ label: "Google",
74
+ description: "Credential-free browser-backed fallback; slower and may be bot-challenged",
75
+ },
76
+ {
77
+ value: "mojeek",
78
+ label: "Mojeek",
79
+ description: "Credential-free browser-backed scrape of Mojeek's independent index",
80
+ },
81
+ {
82
+ value: "public",
83
+ label: "Public Web",
84
+ description: "Queries every credential-free engine in parallel and consolidates deduplicated results",
85
+ },
51
86
  ] as const;
52
87
 
53
88
  /** Supported web search providers (every option except `auto`). */
@@ -1,10 +0,0 @@
1
- /**
2
- * Canonical keys served by the `omp-legacy-pi-bundled:` virtual namespace.
3
- * Mirrors `Object.keys(BUNDLED_PI_REGISTRY)` from
4
- * `legacy-pi-bundled-keys.ts`'s sibling registry file. `legacy-pi-compat.ts`
5
- * statically imports this set to seed `LEGACY_PI_PACKAGE_ROOT_OVERRIDES` in
6
- * compiled-binary mode without dragging the heavy registry's transitive
7
- * graph into dev/test runs (the registry itself stays behind a dynamic
8
- * import — see `ensureBundledRegistryLoaded` in `legacy-pi-compat.ts`).
9
- */
10
- export declare const BUNDLED_PI_REGISTRY_KEYS: ReadonlySet<string>;
@@ -1,10 +0,0 @@
1
- /**
2
- * Canonical specifier → live module namespace. Keys MUST match the right-hand
3
- * side of `bundledRegistryVirtualSpecifier(...)` calls in
4
- * `legacy-pi-compat.ts`; the synthesizer enumerates each namespace's own
5
- * enumerable exports at extension load time. The companion
6
- * `legacy-pi-bundled-keys.ts` mirrors `Object.keys(BUNDLED_PI_REGISTRY)` and
7
- * is statically imported by `legacy-pi-compat.ts` to seed the override map
8
- * without paying the cascade above.
9
- */
10
- export declare const BUNDLED_PI_REGISTRY: Readonly<Record<string, Readonly<Record<string, unknown>>>>;