@oh-my-pi/pi-coding-agent 17.1.1 → 17.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/dist/cli.js +6216 -3984
- package/dist/types/cli/bench-cli.d.ts +1 -0
- package/dist/types/config/model-resolver.d.ts +6 -3
- package/dist/types/config/settings-schema.d.ts +4 -0
- package/dist/types/launch/broker-list-order.test.d.ts +1 -0
- package/dist/types/modes/interactive-mode.d.ts +2 -6
- package/dist/types/modes/types.d.ts +8 -5
- package/dist/types/modes/utils/ui-helpers.d.ts +1 -6
- package/dist/types/task/executor.d.ts +3 -1
- package/dist/types/task/structured-subagent.d.ts +3 -0
- package/dist/types/task/types.d.ts +11 -11
- package/dist/types/thinking.d.ts +13 -0
- package/dist/types/tools/eval-format/index.d.ts +7 -0
- package/dist/types/tools/eval-format/javascript.d.ts +2 -0
- package/dist/types/tools/eval-format/julia.d.ts +2 -0
- package/dist/types/tools/eval-format/python.d.ts +2 -0
- package/dist/types/tools/eval-format/ruby.d.ts +2 -0
- package/dist/types/tools/resolve.d.ts +7 -0
- package/dist/types/tools/todo.d.ts +4 -1
- package/dist/types/web/search/providers/base.d.ts +16 -0
- package/dist/types/web/search/providers/brave.d.ts +2 -0
- package/dist/types/web/search/providers/firecrawl.d.ts +2 -0
- package/dist/types/web/search/providers/gemini.d.ts +3 -0
- package/dist/types/web/search/providers/jina.d.ts +2 -0
- package/dist/types/web/search/providers/kagi.d.ts +2 -0
- package/dist/types/web/search/providers/kimi.d.ts +2 -0
- package/dist/types/web/search/providers/parallel.d.ts +2 -0
- package/dist/types/web/search/providers/perplexity.d.ts +3 -0
- package/dist/types/web/search/providers/searxng.d.ts +10 -0
- package/dist/types/web/search/providers/tavily.d.ts +8 -0
- package/dist/types/web/search/query.d.ts +190 -0
- package/package.json +12 -12
- package/src/cli/bench-cli.ts +12 -1
- package/src/cli/web-search-cli.ts +7 -0
- package/src/config/model-resolver.ts +17 -6
- package/src/config/settings-schema.ts +5 -0
- package/src/debug/raw-sse-buffer.ts +157 -30
- package/src/export/share.ts +4 -3
- package/src/launch/broker-list-order.test.ts +89 -0
- package/src/launch/broker.ts +49 -8
- package/src/modes/controllers/input-controller.ts +8 -8
- package/src/modes/interactive-mode.ts +60 -5
- package/src/modes/types.ts +9 -4
- package/src/modes/utils/ui-helpers.ts +7 -8
- package/src/prompts/system/resolve-device-reminder.md +1 -1
- package/src/prompts/system/workflow-notice.md +1 -1
- package/src/prompts/tools/ast-edit.md +1 -0
- package/src/prompts/tools/bash.md +1 -0
- package/src/prompts/tools/eval.md +2 -2
- package/src/prompts/tools/task.md +5 -2
- package/src/prompts/tools/web-search.md +2 -0
- package/src/sdk.ts +8 -4
- package/src/session/agent-session.ts +14 -0
- package/src/session/queued-messages.ts +7 -1
- package/src/task/executor.ts +17 -9
- package/src/task/index.ts +14 -34
- package/src/task/structured-subagent.ts +4 -0
- package/src/task/types.ts +15 -13
- package/src/thinking.ts +27 -0
- package/src/tools/ast-edit.ts +4 -1
- package/src/tools/bash.ts +13 -0
- package/src/tools/browser/render.ts +2 -1
- package/src/tools/eval-format/index.ts +24 -0
- package/src/tools/eval-format/javascript.ts +952 -0
- package/src/tools/eval-format/julia.ts +446 -0
- package/src/tools/eval-format/python.ts +544 -0
- package/src/tools/eval-format/ruby.ts +380 -0
- package/src/tools/eval-render.ts +12 -6
- package/src/tools/resolve.ts +10 -1
- package/src/tools/todo.ts +58 -6
- package/src/web/search/index.ts +28 -5
- package/src/web/search/providers/anthropic.ts +62 -4
- package/src/web/search/providers/base.ts +16 -0
- package/src/web/search/providers/brave.ts +30 -3
- package/src/web/search/providers/codex.ts +14 -1
- package/src/web/search/providers/duckduckgo.ts +23 -1
- package/src/web/search/providers/ecosia.ts +5 -1
- package/src/web/search/providers/exa.ts +28 -1
- package/src/web/search/providers/firecrawl.ts +37 -3
- package/src/web/search/providers/gemini.ts +12 -2
- package/src/web/search/providers/google.ts +2 -1
- package/src/web/search/providers/jina.ts +31 -8
- package/src/web/search/providers/kagi.ts +10 -1
- package/src/web/search/providers/kimi.ts +16 -1
- package/src/web/search/providers/mojeek.ts +14 -1
- package/src/web/search/providers/parallel.ts +48 -2
- package/src/web/search/providers/perplexity.ts +94 -6
- package/src/web/search/providers/searxng.ts +145 -9
- package/src/web/search/providers/startpage.ts +8 -2
- package/src/web/search/providers/synthetic.ts +7 -1
- package/src/web/search/providers/tavily.ts +52 -3
- package/src/web/search/providers/tinyfish.ts +6 -1
- package/src/web/search/providers/xai.ts +49 -2
- package/src/web/search/providers/zai.ts +19 -1
- package/src/web/search/query.ts +850 -0
|
@@ -12,6 +12,7 @@ import { $env } from "@oh-my-pi/pi-utils";
|
|
|
12
12
|
|
|
13
13
|
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
14
14
|
import { SearchProviderError } from "../../../web/search/types";
|
|
15
|
+
import { formatQuery, parseSearchQuery, type QuerySyntax, type StructuredQuery } from "../query";
|
|
15
16
|
import { clampNumResults, dateToAgeSeconds } from "../utils";
|
|
16
17
|
import type { SearchParams } from "./base";
|
|
17
18
|
import { SearchProvider } from "./base";
|
|
@@ -25,8 +26,19 @@ const DEFAULT_NUM_RESULTS = 10;
|
|
|
25
26
|
const MAX_NUM_RESULTS = 20;
|
|
26
27
|
const DEFAULT_TIMEOUT_SECONDS = 30;
|
|
27
28
|
|
|
29
|
+
/** Kimi Code search is Bing-flavored: re-emit the operators Bing parses; dates/lang stay with the central filter. */
|
|
30
|
+
const KIMI_QUERY_SYNTAX: QuerySyntax = {
|
|
31
|
+
phrases: true,
|
|
32
|
+
negation: true,
|
|
33
|
+
site: true,
|
|
34
|
+
inTitle: true,
|
|
35
|
+
inUrl: true,
|
|
36
|
+
filetype: true,
|
|
37
|
+
};
|
|
38
|
+
|
|
28
39
|
export interface KimiSearchParams {
|
|
29
40
|
query: string;
|
|
41
|
+
parsedQuery?: StructuredQuery;
|
|
30
42
|
num_results?: number;
|
|
31
43
|
include_content?: boolean;
|
|
32
44
|
signal?: AbortSignal;
|
|
@@ -138,12 +150,14 @@ export async function searchKimi(params: KimiSearchParams): Promise<SearchRespon
|
|
|
138
150
|
);
|
|
139
151
|
}
|
|
140
152
|
|
|
153
|
+
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
|
154
|
+
const query = parsed.hasDirectives ? formatQuery(parsed, KIMI_QUERY_SYNTAX) : params.query;
|
|
141
155
|
const limit = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
|
142
156
|
const { response, requestId } = await withAuth(
|
|
143
157
|
keyOrResolver,
|
|
144
158
|
key =>
|
|
145
159
|
callKimiSearch(key, {
|
|
146
|
-
query
|
|
160
|
+
query,
|
|
147
161
|
limit,
|
|
148
162
|
includeContent: params.include_content ?? false,
|
|
149
163
|
signal: params.signal,
|
|
@@ -192,6 +206,7 @@ export class KimiProvider extends SearchProvider {
|
|
|
192
206
|
|
|
193
207
|
return searchKimi({
|
|
194
208
|
query: params.query,
|
|
209
|
+
parsedQuery: params.parsedQuery,
|
|
195
210
|
num_results: params.numSearchResults ?? params.limit,
|
|
196
211
|
signal: params.signal,
|
|
197
212
|
authStorage: params.authStorage,
|
|
@@ -4,6 +4,7 @@ import { parseHTML } from "linkedom";
|
|
|
4
4
|
import type { Page } from "puppeteer-core";
|
|
5
5
|
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
6
6
|
import { SearchProviderError } from "../../../web/search/types";
|
|
7
|
+
import { formatScraperQuery, type QuerySyntax } from "../query";
|
|
7
8
|
import { clampNumResults } from "../utils";
|
|
8
9
|
import type { SearchParams } from "./base";
|
|
9
10
|
import { SearchProvider } from "./base";
|
|
@@ -81,9 +82,21 @@ function parseHtmlResults(html: string): ParsedResult[] {
|
|
|
81
82
|
return results;
|
|
82
83
|
}
|
|
83
84
|
|
|
85
|
+
/**
|
|
86
|
+
* Syntax re-emitted to Mojeek for directive-carrying queries. Mojeek's
|
|
87
|
+
* support page (mojeek.com/support/search-operators.html) confirms `site:`,
|
|
88
|
+
* and the community docs confirm quoted phrases and `-` exclusions. Mojeek
|
|
89
|
+
* also parses `in*:` operators and its own date syntax (`since:`/`before:`
|
|
90
|
+
* with YYYYMMDD), but the latter differs from Google's `after:`/`before:`
|
|
91
|
+
* ISO form and `since` is already claimed by `recency`, so date bounds and
|
|
92
|
+
* `in*` constraints are conservatively left to the pipeline's lenient
|
|
93
|
+
* post-filter instead.
|
|
94
|
+
*/
|
|
95
|
+
const MOJEEK_QUERY_SYNTAX: QuerySyntax = { phrases: true, negation: true, site: true };
|
|
96
|
+
|
|
84
97
|
function buildSearchUrl(params: SearchParams, numResults: number): string {
|
|
85
98
|
const url = new URL(MOJEEK_SEARCH_URL);
|
|
86
|
-
url.searchParams.set("q", params.query);
|
|
99
|
+
url.searchParams.set("q", formatScraperQuery(params.query, params.parsedQuery, MOJEEK_QUERY_SYNTAX));
|
|
87
100
|
url.searchParams.set("t", String(numResults));
|
|
88
101
|
url.searchParams.set("arc", "none");
|
|
89
102
|
url.searchParams.set("lang", "en");
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
parseParallelErrorResponse,
|
|
10
10
|
parseParallelSearchPayload,
|
|
11
11
|
} from "../../parallel";
|
|
12
|
+
import { formatQuery, parseSearchQuery, type StructuredQuery } from "../query";
|
|
12
13
|
import { clampNumResults } from "../utils";
|
|
13
14
|
import type { SearchParams } from "./base";
|
|
14
15
|
import { SearchProvider } from "./base";
|
|
@@ -17,6 +18,42 @@ import { classifyProviderHttpError, toSearchSources, withHardTimeout } from "./u
|
|
|
17
18
|
const DEFAULT_NUM_RESULTS = 10;
|
|
18
19
|
const MAX_NUM_RESULTS = 40;
|
|
19
20
|
|
|
21
|
+
/** Query-string caps for Parallel: natural-language objective, no field operators. */
|
|
22
|
+
const PARALLEL_QUERY_SYNTAX = { phrases: true, negation: true, or: true } as const;
|
|
23
|
+
|
|
24
|
+
/** Parallel `source_policy` (beta Search API): bare-host allow/deny lists + freshness floor. */
|
|
25
|
+
interface ParallelSourcePolicy {
|
|
26
|
+
include_domains?: string[];
|
|
27
|
+
exclude_domains?: string[];
|
|
28
|
+
after_date?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Site values may carry paths (`github.com/anthropics`); Parallel takes bare hosts. */
|
|
32
|
+
function toHosts(sites: readonly string[]): string[] {
|
|
33
|
+
const hosts = new Set<string>();
|
|
34
|
+
for (const site of sites) {
|
|
35
|
+
const host = site.split("/", 1)[0];
|
|
36
|
+
if (host) hosts.add(host);
|
|
37
|
+
}
|
|
38
|
+
return [...hosts];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Map parsed `site:`/`-site:`/`after:` directives onto Parallel's
|
|
43
|
+
* `source_policy`. Per Parallel docs, `exclude_domains` is ignored when
|
|
44
|
+
* `include_domains` is set, so exclusions are only sent without an allow
|
|
45
|
+
* list (the central lenient filter enforces them regardless).
|
|
46
|
+
*/
|
|
47
|
+
function toSourcePolicy(parsed: StructuredQuery): ParallelSourcePolicy | undefined {
|
|
48
|
+
const policy: ParallelSourcePolicy = {};
|
|
49
|
+
const include = toHosts(parsed.sites);
|
|
50
|
+
const exclude = toHosts(parsed.excludedSites);
|
|
51
|
+
if (include.length) policy.include_domains = include;
|
|
52
|
+
else if (exclude.length) policy.exclude_domains = exclude;
|
|
53
|
+
if (parsed.after) policy.after_date = parsed.after;
|
|
54
|
+
return Object.keys(policy).length ? policy : undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
20
57
|
async function searchWithAuthStorage(
|
|
21
58
|
objective: string,
|
|
22
59
|
queries: string[],
|
|
@@ -26,6 +63,7 @@ async function searchWithAuthStorage(
|
|
|
26
63
|
},
|
|
27
64
|
authStorage: AuthStorage,
|
|
28
65
|
sessionId?: string,
|
|
66
|
+
sourcePolicy?: ParallelSourcePolicy,
|
|
29
67
|
): Promise<ParallelSearchResult> {
|
|
30
68
|
const apiKey = await authStorage.getApiKey("parallel", sessionId, { signal: params.signal });
|
|
31
69
|
if (!apiKey) {
|
|
@@ -57,6 +95,7 @@ async function searchWithAuthStorage(
|
|
|
57
95
|
excerpts: {
|
|
58
96
|
max_chars_per_result: 10_000,
|
|
59
97
|
},
|
|
98
|
+
...(sourcePolicy && { source_policy: sourcePolicy }),
|
|
60
99
|
}),
|
|
61
100
|
signal: withHardTimeout(params.signal),
|
|
62
101
|
});
|
|
@@ -78,22 +117,28 @@ export async function searchParallel(
|
|
|
78
117
|
num_results?: number;
|
|
79
118
|
signal?: AbortSignal;
|
|
80
119
|
fetch?: FetchImpl;
|
|
120
|
+
parsedQuery?: StructuredQuery;
|
|
81
121
|
},
|
|
82
122
|
authStorage: AuthStorage,
|
|
83
123
|
sessionId?: string,
|
|
84
124
|
): Promise<SearchResponse> {
|
|
85
125
|
const numResults = clampNumResults(params.num_results, DEFAULT_NUM_RESULTS, MAX_NUM_RESULTS);
|
|
126
|
+
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
|
127
|
+
// Back-compat: without directives the upstream request is byte-identical.
|
|
128
|
+
const query = parsed.hasDirectives ? formatQuery(parsed, PARALLEL_QUERY_SYNTAX) : params.query;
|
|
129
|
+
const sourcePolicy = parsed.hasDirectives ? toSourcePolicy(parsed) : undefined;
|
|
86
130
|
|
|
87
131
|
try {
|
|
88
132
|
const result = await searchWithAuthStorage(
|
|
89
|
-
|
|
90
|
-
[
|
|
133
|
+
query,
|
|
134
|
+
[query],
|
|
91
135
|
{
|
|
92
136
|
signal: params.signal,
|
|
93
137
|
fetch: params.fetch,
|
|
94
138
|
},
|
|
95
139
|
authStorage,
|
|
96
140
|
sessionId,
|
|
141
|
+
sourcePolicy,
|
|
97
142
|
);
|
|
98
143
|
|
|
99
144
|
return {
|
|
@@ -128,6 +173,7 @@ export class ParallelProvider extends SearchProvider {
|
|
|
128
173
|
num_results: params.numSearchResults ?? params.limit,
|
|
129
174
|
signal: params.signal,
|
|
130
175
|
fetch: params.fetch,
|
|
176
|
+
parsedQuery: params.parsedQuery,
|
|
131
177
|
},
|
|
132
178
|
params.authStorage,
|
|
133
179
|
params.sessionId,
|
|
@@ -30,6 +30,7 @@ import type {
|
|
|
30
30
|
SearchSource,
|
|
31
31
|
} from "../../../web/search/types";
|
|
32
32
|
import { SearchProviderError } from "../../../web/search/types";
|
|
33
|
+
import { formatQuery, parseSearchQuery, type QuerySyntax, type StructuredQuery } from "../query";
|
|
33
34
|
import { dateToAgeSeconds } from "../utils";
|
|
34
35
|
import type { SearchParams } from "./base";
|
|
35
36
|
import { SearchProvider } from "./base";
|
|
@@ -46,6 +47,71 @@ const OAUTH_USER_AGENT = "Perplexity/641 CFNetwork/1568 Darwin/25.2.0";
|
|
|
46
47
|
const ANONYMOUS_USER_AGENT =
|
|
47
48
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36";
|
|
48
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Query-string operators Perplexity's search backend tolerates as text signal.
|
|
52
|
+
* `site:`/date/`lang:` directives are excluded: they map onto native request
|
|
53
|
+
* fields (`search_domain_filter`, `search_*_date_filter`,
|
|
54
|
+
* `search_language_filter`) and must be stripped from the query so the engine
|
|
55
|
+
* is not double-constrained.
|
|
56
|
+
*/
|
|
57
|
+
const PERPLEXITY_QUERY_SYNTAX: QuerySyntax = {
|
|
58
|
+
phrases: true,
|
|
59
|
+
negation: true,
|
|
60
|
+
or: true,
|
|
61
|
+
inUrl: true,
|
|
62
|
+
inTitle: true,
|
|
63
|
+
filetype: true,
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** Native Perplexity search filters derived from parsed query directives. */
|
|
67
|
+
interface PerplexityNativeFilters {
|
|
68
|
+
/** Query rebuilt without natively-mapped directives. */
|
|
69
|
+
query: string;
|
|
70
|
+
/** `search_domain_filter`: allow entries as bare hosts, deny entries as `-host`. */
|
|
71
|
+
domainFilter?: string[];
|
|
72
|
+
/** `search_after_date_filter`, `%m/%d/%Y`. */
|
|
73
|
+
afterDate?: string;
|
|
74
|
+
/** `search_before_date_filter`, `%m/%d/%Y`. */
|
|
75
|
+
beforeDate?: string;
|
|
76
|
+
/** `search_language_filter`: ISO 639-1 two-letter codes. */
|
|
77
|
+
languageFilter?: string[];
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Bare host of a `site:` value (`github.com/anthropics` → `github.com`);
|
|
82
|
+
* Perplexity's domain filter takes hosts only, the path part is enforced by
|
|
83
|
+
* the central lenient post-filter.
|
|
84
|
+
*/
|
|
85
|
+
function siteHost(site: string): string {
|
|
86
|
+
const slash = site.indexOf("/");
|
|
87
|
+
return slash === -1 ? site : site.slice(0, slash);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** ISO `YYYY-MM-DD` → Perplexity's documented `%m/%d/%Y` date-filter format (e.g. `3/1/2025`). */
|
|
91
|
+
function toPerplexityDate(iso: string): string {
|
|
92
|
+
const [year, month, day] = iso.split("-");
|
|
93
|
+
return `${Number(month)}/${Number(day)}/${year}`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Map parsed query directives onto native Perplexity search filters. */
|
|
97
|
+
function buildNativeFilters(parsed: StructuredQuery, rawQuery: string): PerplexityNativeFilters {
|
|
98
|
+
if (!parsed.hasDirectives) return { query: rawQuery };
|
|
99
|
+
// Allow + deny share one array; the API caps it at 20 entries.
|
|
100
|
+
const domains = [
|
|
101
|
+
...new Set([...parsed.sites.map(siteHost), ...parsed.excludedSites.map(site => `-${siteHost(site)}`)]),
|
|
102
|
+
].slice(0, 20);
|
|
103
|
+
// search_language_filter takes ISO 639-1 two-letter codes; pass `en-us` as
|
|
104
|
+
// `en`, and leave anything else to the central post-filter.
|
|
105
|
+
const langCode = parsed.lang ? /^([a-z]{2})(?:[-_]|$)/.exec(parsed.lang)?.[1] : undefined;
|
|
106
|
+
return {
|
|
107
|
+
query: formatQuery(parsed, PERPLEXITY_QUERY_SYNTAX),
|
|
108
|
+
domainFilter: domains.length > 0 ? domains : undefined,
|
|
109
|
+
afterDate: parsed.after ? toPerplexityDate(parsed.after) : undefined,
|
|
110
|
+
beforeDate: parsed.before ? toPerplexityDate(parsed.before) : undefined,
|
|
111
|
+
languageFilter: langCode ? [langCode] : undefined,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
49
115
|
interface PerplexityOAuthStreamMarkdownBlock {
|
|
50
116
|
answer?: string;
|
|
51
117
|
chunks?: string[];
|
|
@@ -258,6 +324,8 @@ export interface PerplexitySearchParams {
|
|
|
258
324
|
signal?: AbortSignal;
|
|
259
325
|
query: string;
|
|
260
326
|
system_prompt?: string;
|
|
327
|
+
/** Pre-parsed view of `query` from the search pipeline; parsed locally when absent. */
|
|
328
|
+
parsedQuery?: StructuredQuery;
|
|
261
329
|
search_recency_filter?: "hour" | "day" | "week" | "month" | "year";
|
|
262
330
|
num_results?: number;
|
|
263
331
|
/** Maximum output tokens. Defaults to 8192. */
|
|
@@ -352,6 +420,10 @@ function buildPerplexityExtraBody(request: PerplexityRequest): Record<string, un
|
|
|
352
420
|
language_preference: request.language_preference,
|
|
353
421
|
return_related_questions: request.return_related_questions,
|
|
354
422
|
search_recency_filter: request.search_recency_filter,
|
|
423
|
+
search_domain_filter: request.search_domain_filter,
|
|
424
|
+
search_after_date_filter: request.search_after_date_filter,
|
|
425
|
+
search_before_date_filter: request.search_before_date_filter,
|
|
426
|
+
search_language_filter: request.search_language_filter,
|
|
355
427
|
};
|
|
356
428
|
}
|
|
357
429
|
|
|
@@ -531,6 +603,7 @@ function buildOAuthAnswer(event: PerplexityOAuthStreamEvent): string {
|
|
|
531
603
|
async function callPerplexityAsk(
|
|
532
604
|
auth: { type: "oauth"; token: string } | { type: "cookies"; cookies: string } | { type: "anonymous" },
|
|
533
605
|
params: PerplexitySearchParams,
|
|
606
|
+
filters: PerplexityNativeFilters,
|
|
534
607
|
): Promise<{ answer: string; sources: SearchSource[]; model?: string; requestId?: string }> {
|
|
535
608
|
const requestId = crypto.randomUUID();
|
|
536
609
|
// The consumer `perplexity_ask` endpoint is itself a research assistant and
|
|
@@ -539,7 +612,7 @@ async function callPerplexityAsk(
|
|
|
539
612
|
// "I don't have access to web-search tools in this turn", so ask-endpoint
|
|
540
613
|
// searches send the bare query. (The API-key path still uses system_prompt
|
|
541
614
|
// as a proper `system` message.)
|
|
542
|
-
const effectiveQuery =
|
|
615
|
+
const effectiveQuery = filters.query;
|
|
543
616
|
|
|
544
617
|
const headers: Record<string, string> = {
|
|
545
618
|
"Content-Type": "application/json",
|
|
@@ -578,7 +651,9 @@ async function callPerplexityAsk(
|
|
|
578
651
|
version: OAUTH_API_VERSION,
|
|
579
652
|
language: "en-US",
|
|
580
653
|
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
581
|
-
|
|
654
|
+
// Recency cannot be combined with absolute date filters; explicit
|
|
655
|
+
// before:/after: bounds take precedence.
|
|
656
|
+
search_recency_filter: filters.afterDate || filters.beforeDate ? null : (params.search_recency_filter ?? null),
|
|
582
657
|
is_incognito: true,
|
|
583
658
|
use_schematized_api: true,
|
|
584
659
|
// `true` (the native app's default) lets the backend classifier skip
|
|
@@ -603,6 +678,10 @@ async function callPerplexityAsk(
|
|
|
603
678
|
if (auth.type === "anonymous") {
|
|
604
679
|
requestParams.send_back_text_in_streaming_api = true;
|
|
605
680
|
}
|
|
681
|
+
if (filters.domainFilter) requestParams.search_domain_filter = filters.domainFilter;
|
|
682
|
+
if (filters.afterDate) requestParams.search_after_date_filter = filters.afterDate;
|
|
683
|
+
if (filters.beforeDate) requestParams.search_before_date_filter = filters.beforeDate;
|
|
684
|
+
if (filters.languageFilter) requestParams.search_language_filter = filters.languageFilter;
|
|
606
685
|
|
|
607
686
|
const requestInit = {
|
|
608
687
|
method: "POST",
|
|
@@ -781,12 +860,14 @@ function applySourceLimit(result: SearchResponse, limit?: number): SearchRespons
|
|
|
781
860
|
|
|
782
861
|
/** Execute Perplexity web search */
|
|
783
862
|
export async function searchPerplexity(params: PerplexitySearchParams): Promise<SearchResponse> {
|
|
863
|
+
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
|
864
|
+
const filters = buildNativeFilters(parsed, params.query);
|
|
784
865
|
const systemPrompt = params.system_prompt;
|
|
785
866
|
const messages: PerplexityRequest["messages"] = [];
|
|
786
867
|
if (systemPrompt) {
|
|
787
868
|
messages.push({ role: "system", content: systemPrompt });
|
|
788
869
|
}
|
|
789
|
-
messages.push({ role: "user", content:
|
|
870
|
+
messages.push({ role: "user", content: filters.query });
|
|
790
871
|
|
|
791
872
|
const request: PerplexityRequest = {
|
|
792
873
|
model: "sonar-pro",
|
|
@@ -805,7 +886,13 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise<
|
|
|
805
886
|
return_related_questions: true,
|
|
806
887
|
};
|
|
807
888
|
|
|
808
|
-
if (
|
|
889
|
+
if (filters.domainFilter) request.search_domain_filter = filters.domainFilter;
|
|
890
|
+
if (filters.afterDate) request.search_after_date_filter = filters.afterDate;
|
|
891
|
+
if (filters.beforeDate) request.search_before_date_filter = filters.beforeDate;
|
|
892
|
+
if (filters.languageFilter) request.search_language_filter = filters.languageFilter;
|
|
893
|
+
// The API rejects search_recency_filter combined with absolute date
|
|
894
|
+
// filters; explicit before:/after: bounds take precedence.
|
|
895
|
+
if (params.search_recency_filter && !filters.afterDate && !filters.beforeDate) {
|
|
809
896
|
request.search_recency_filter = params.search_recency_filter;
|
|
810
897
|
}
|
|
811
898
|
|
|
@@ -830,10 +917,10 @@ export async function searchPerplexity(params: PerplexitySearchParams): Promise<
|
|
|
830
917
|
? await withOAuthAccess(
|
|
831
918
|
params.authStorage,
|
|
832
919
|
"perplexity",
|
|
833
|
-
access => callPerplexityAsk({ type: "oauth", token: access.accessToken }, params),
|
|
920
|
+
access => callPerplexityAsk({ type: "oauth", token: access.accessToken }, params, filters),
|
|
834
921
|
{ sessionId: params.sessionId, signal: params.signal, seed: auth.access },
|
|
835
922
|
)
|
|
836
|
-
: await callPerplexityAsk(auth, params);
|
|
923
|
+
: await callPerplexityAsk(auth, params, filters);
|
|
837
924
|
return applySourceLimit(
|
|
838
925
|
{
|
|
839
926
|
provider: "perplexity",
|
|
@@ -892,6 +979,7 @@ export class PerplexityProvider extends SearchProvider {
|
|
|
892
979
|
return searchPerplexity({
|
|
893
980
|
signal: params.signal,
|
|
894
981
|
query: params.query,
|
|
982
|
+
parsedQuery: params.parsedQuery,
|
|
895
983
|
temperature: params.temperature,
|
|
896
984
|
max_tokens: params.maxOutputTokens,
|
|
897
985
|
num_search_results: params.numSearchResults,
|
|
@@ -14,6 +14,9 @@
|
|
|
14
14
|
* searxng.basicUsername - Optional RFC 7617 Basic auth username
|
|
15
15
|
* searxng.basicPassword - Optional RFC 7617 Basic auth password
|
|
16
16
|
* searxng.categories - Optional comma-separated categories filter
|
|
17
|
+
* searxng.engines - Optional comma-separated engine names or shortcuts
|
|
18
|
+
* (e.g. "duckduckgo, br, sp"); shortcuts resolve via
|
|
19
|
+
* the instance's /config endpoint
|
|
17
20
|
* searxng.language - Optional language code (e.g. en, zh-CN)
|
|
18
21
|
*
|
|
19
22
|
* Environment variable fallbacks:
|
|
@@ -22,6 +25,11 @@
|
|
|
22
25
|
* SEARXNG_BASIC_USERNAME - Optional RFC 7617 Basic auth username
|
|
23
26
|
* SEARXNG_BASIC_PASSWORD - Optional RFC 7617 Basic auth password
|
|
24
27
|
*
|
|
28
|
+
* Bang syntax in queries is passed through: `!ddg foo` selects an engine or
|
|
29
|
+
* category server-side and the bang token is stripped from the upstream query.
|
|
30
|
+
* External bangs (`!!g`) are removed client-side because SearXNG answers them
|
|
31
|
+
* with an HTTP redirect even for JSON requests.
|
|
32
|
+
*
|
|
25
33
|
* Reference: https://docs.searxng.org/dev/search_api.html
|
|
26
34
|
*/
|
|
27
35
|
|
|
@@ -30,6 +38,8 @@ import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
|
|
|
30
38
|
import { settings } from "../../../config/settings";
|
|
31
39
|
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
32
40
|
import { SearchProviderError } from "../../../web/search/types";
|
|
41
|
+
import type { StructuredQuery } from "../query";
|
|
42
|
+
import { formatScraperQuery, parseSearchQuery } from "../query";
|
|
33
43
|
import { clampNumResults, dateToAgeSeconds } from "../utils";
|
|
34
44
|
import type { SearchParams } from "./base";
|
|
35
45
|
import { SearchProvider } from "./base";
|
|
@@ -73,6 +83,11 @@ interface SearXNGAuth {
|
|
|
73
83
|
value: string;
|
|
74
84
|
}
|
|
75
85
|
|
|
86
|
+
/** Subset of the SearXNG /config payload used for engine shortcut resolution. */
|
|
87
|
+
interface SearXNGConfig {
|
|
88
|
+
engines?: Array<{ name?: string; shortcut?: string }>;
|
|
89
|
+
}
|
|
90
|
+
|
|
76
91
|
/** Find SearXNG endpoint from settings or environment. */
|
|
77
92
|
function findEndpoint(): string | null {
|
|
78
93
|
try {
|
|
@@ -150,6 +165,110 @@ function findAuth(): SearXNGAuth | null {
|
|
|
150
165
|
return token ? { type: "bearer", value: token } : null;
|
|
151
166
|
}
|
|
152
167
|
|
|
168
|
+
/** Find configured engine names/shortcuts from settings. */
|
|
169
|
+
function findEngines(): string | null {
|
|
170
|
+
try {
|
|
171
|
+
const engines = settings.get("searxng.engines");
|
|
172
|
+
if (engines) return engines;
|
|
173
|
+
} catch {
|
|
174
|
+
// Settings not initialized yet
|
|
175
|
+
}
|
|
176
|
+
return null;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Build request headers including authentication. */
|
|
180
|
+
function buildHeaders(auth: SearXNGAuth | null): Record<string, string> {
|
|
181
|
+
const headers: Record<string, string> = { Accept: "application/json" };
|
|
182
|
+
if (auth?.type === "basic") {
|
|
183
|
+
headers.Authorization = `Basic ${auth.value}`;
|
|
184
|
+
} else if (auth?.type === "bearer") {
|
|
185
|
+
headers.Authorization = `Bearer ${auth.value}`;
|
|
186
|
+
}
|
|
187
|
+
return headers;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** Per-endpoint cache of shortcut/name → canonical engine name maps. */
|
|
191
|
+
const engineNameMapCache = new Map<string, Promise<Map<string, string> | null>>();
|
|
192
|
+
|
|
193
|
+
/** Fetch the instance's /config and build a lookup of lowercased engine names
|
|
194
|
+
* and shortcuts to canonical engine names. Returns null on any failure. */
|
|
195
|
+
async function fetchEngineNameMap(
|
|
196
|
+
base: string,
|
|
197
|
+
auth: SearXNGAuth | null,
|
|
198
|
+
fetchImpl: FetchImpl | undefined,
|
|
199
|
+
signal: AbortSignal | undefined,
|
|
200
|
+
): Promise<Map<string, string> | null> {
|
|
201
|
+
try {
|
|
202
|
+
const response = await (fetchImpl ?? fetch)(`${base}/config`, {
|
|
203
|
+
headers: buildHeaders(auth),
|
|
204
|
+
signal: withHardTimeout(signal),
|
|
205
|
+
});
|
|
206
|
+
if (!response.ok) return null;
|
|
207
|
+
const config = (await response.json()) as SearXNGConfig;
|
|
208
|
+
const map = new Map<string, string>();
|
|
209
|
+
for (const engine of config.engines ?? []) {
|
|
210
|
+
if (!engine.name) continue;
|
|
211
|
+
map.set(engine.name.toLowerCase(), engine.name);
|
|
212
|
+
if (engine.shortcut) map.set(engine.shortcut.toLowerCase(), engine.name);
|
|
213
|
+
}
|
|
214
|
+
return map.size ? map : null;
|
|
215
|
+
} catch {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Get the engine name map for an endpoint, cached for the process lifetime.
|
|
221
|
+
* Failures are not cached so a transient error retries on the next search. */
|
|
222
|
+
function getEngineNameMap(
|
|
223
|
+
endpoint: string,
|
|
224
|
+
auth: SearXNGAuth | null,
|
|
225
|
+
fetchImpl: FetchImpl | undefined,
|
|
226
|
+
signal: AbortSignal | undefined,
|
|
227
|
+
): Promise<Map<string, string> | null> {
|
|
228
|
+
const base = endpoint.replace(/\/+$/, "");
|
|
229
|
+
let cached = engineNameMapCache.get(base);
|
|
230
|
+
if (!cached) {
|
|
231
|
+
cached = fetchEngineNameMap(base, auth, fetchImpl, signal).then(map => {
|
|
232
|
+
if (!map) engineNameMapCache.delete(base);
|
|
233
|
+
return map;
|
|
234
|
+
});
|
|
235
|
+
engineNameMapCache.set(base, cached);
|
|
236
|
+
}
|
|
237
|
+
return cached;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Resolve configured engine entries (canonical names or shortcuts like `ddg`)
|
|
241
|
+
* to canonical names for SearXNG's `engines=` parameter, which accepts names
|
|
242
|
+
* only — shortcuts resolve exclusively through bang syntax. Unknown entries
|
|
243
|
+
* pass through verbatim; the server drops them and falls back to categories. */
|
|
244
|
+
async function resolveEngineNames(
|
|
245
|
+
raw: string,
|
|
246
|
+
endpoint: string,
|
|
247
|
+
auth: SearXNGAuth | null,
|
|
248
|
+
fetchImpl: FetchImpl | undefined,
|
|
249
|
+
signal: AbortSignal | undefined,
|
|
250
|
+
): Promise<string | undefined> {
|
|
251
|
+
const entries = raw
|
|
252
|
+
.split(",")
|
|
253
|
+
.map(entry => entry.trim())
|
|
254
|
+
.filter(Boolean);
|
|
255
|
+
if (!entries.length) return undefined;
|
|
256
|
+
const map = await getEngineNameMap(endpoint, auth, fetchImpl, signal);
|
|
257
|
+
if (!map) return entries.join(",");
|
|
258
|
+
return entries.map(entry => map.get(entry.toLowerCase()) ?? entry).join(",");
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Strip external bang tokens (`!!g`, bare `!!`): SearXNG answers them with an
|
|
262
|
+
* HTTP redirect even for JSON requests, which breaks response parsing.
|
|
263
|
+
* Single-bang engine/category selectors (`!ddg`, `!images`) are kept — the
|
|
264
|
+
* instance resolves and removes them server-side. */
|
|
265
|
+
function stripExternalBangs(query: string): string {
|
|
266
|
+
return query
|
|
267
|
+
.split(/\s+/)
|
|
268
|
+
.filter(part => !part.startsWith("!!"))
|
|
269
|
+
.join(" ");
|
|
270
|
+
}
|
|
271
|
+
|
|
153
272
|
/** Build the search URL and headers for a SearXNG request */
|
|
154
273
|
function buildRequest(
|
|
155
274
|
endpoint: string,
|
|
@@ -158,6 +277,7 @@ function buildRequest(
|
|
|
158
277
|
num_results?: number;
|
|
159
278
|
recency?: "day" | "week" | "month" | "year";
|
|
160
279
|
categories?: string;
|
|
280
|
+
engines?: string;
|
|
161
281
|
language?: string;
|
|
162
282
|
signal?: AbortSignal;
|
|
163
283
|
},
|
|
@@ -181,19 +301,15 @@ function buildRequest(
|
|
|
181
301
|
url.searchParams.set("categories", params.categories);
|
|
182
302
|
}
|
|
183
303
|
|
|
304
|
+
if (params.engines) {
|
|
305
|
+
url.searchParams.set("engines", params.engines);
|
|
306
|
+
}
|
|
307
|
+
|
|
184
308
|
if (params.language) {
|
|
185
309
|
url.searchParams.set("language", params.language);
|
|
186
310
|
}
|
|
187
311
|
|
|
188
|
-
const headers
|
|
189
|
-
Accept: "application/json",
|
|
190
|
-
};
|
|
191
|
-
|
|
192
|
-
if (auth?.type === "basic") {
|
|
193
|
-
headers.Authorization = `Basic ${auth.value}`;
|
|
194
|
-
} else if (auth?.type === "bearer") {
|
|
195
|
-
headers.Authorization = `Bearer ${auth.value}`;
|
|
196
|
-
}
|
|
312
|
+
const headers = buildHeaders(auth);
|
|
197
313
|
|
|
198
314
|
return { url, headers };
|
|
199
315
|
}
|
|
@@ -205,6 +321,7 @@ async function callSearXNGSearch(
|
|
|
205
321
|
num_results?: number;
|
|
206
322
|
recency?: "day" | "week" | "month" | "year";
|
|
207
323
|
categories?: string;
|
|
324
|
+
engines?: string;
|
|
208
325
|
language?: string;
|
|
209
326
|
signal?: AbortSignal;
|
|
210
327
|
fetch?: FetchImpl;
|
|
@@ -231,6 +348,7 @@ async function callSearXNGSearch(
|
|
|
231
348
|
/** Execute SearXNG web search. */
|
|
232
349
|
export async function searchSearXNG(params: {
|
|
233
350
|
query: string;
|
|
351
|
+
parsedQuery?: StructuredQuery;
|
|
234
352
|
num_results?: number;
|
|
235
353
|
recency?: "day" | "week" | "month" | "year";
|
|
236
354
|
signal?: AbortSignal;
|
|
@@ -255,12 +373,29 @@ export async function searchSearXNG(params: {
|
|
|
255
373
|
} catch {
|
|
256
374
|
// Settings not initialized yet
|
|
257
375
|
}
|
|
376
|
+
const configuredEngines = findEngines();
|
|
377
|
+
|
|
378
|
+
// SearXNG forwards `q` to downstream engines, so build it with the shared
|
|
379
|
+
// scraper formatter: operators are canonicalized and scraper-hostile ones
|
|
380
|
+
// (path-carrying `site:`, `inurl:`) are structurally demoted to plain
|
|
381
|
+
// terms before formatting, so paren-grouped `site:` filters are covered
|
|
382
|
+
// too. `lang:` maps onto the native `language` param (overriding the
|
|
383
|
+
// configured default).
|
|
384
|
+
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
|
385
|
+
const query = formatScraperQuery(params.query, parsed);
|
|
386
|
+
if (parsed.lang) language = parsed.lang;
|
|
387
|
+
|
|
388
|
+
const engines = configuredEngines
|
|
389
|
+
? await resolveEngineNames(configuredEngines, endpoint, auth, params.fetch, params.signal)
|
|
390
|
+
: undefined;
|
|
258
391
|
|
|
259
392
|
const response = await callSearXNGSearch(
|
|
260
393
|
endpoint,
|
|
261
394
|
{
|
|
262
395
|
...params,
|
|
396
|
+
query: stripExternalBangs(query),
|
|
263
397
|
categories,
|
|
398
|
+
engines,
|
|
264
399
|
language,
|
|
265
400
|
fetch: params.fetch,
|
|
266
401
|
},
|
|
@@ -315,6 +450,7 @@ export class SearXNGProvider extends SearchProvider {
|
|
|
315
450
|
|
|
316
451
|
search(params: SearchParams): Promise<SearchResponse> {
|
|
317
452
|
return searchSearXNG({
|
|
453
|
+
parsedQuery: params.parsedQuery,
|
|
318
454
|
query: params.query,
|
|
319
455
|
num_results: params.numSearchResults ?? params.limit,
|
|
320
456
|
recency: params.recency,
|
|
@@ -2,6 +2,7 @@ import type { AuthStorage, FetchImpl } from "@oh-my-pi/pi-ai";
|
|
|
2
2
|
import { parseHTML } from "linkedom";
|
|
3
3
|
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
4
4
|
import { SearchProviderError } from "../../../web/search/types";
|
|
5
|
+
import { formatScraperQuery } from "../query";
|
|
5
6
|
import { clampNumResults } from "../utils";
|
|
6
7
|
import type { SearchParams } from "./base";
|
|
7
8
|
import { SearchProvider } from "./base";
|
|
@@ -136,12 +137,17 @@ async function callStartpageHtml(params: SearchParams): Promise<string> {
|
|
|
136
137
|
const fetchImpl = params.fetch ?? fetch;
|
|
137
138
|
const signal = withHardTimeout(params.signal);
|
|
138
139
|
const withDate = params.recency ? RECENCY_TO_STARTPAGE_WITH_DATE[params.recency] : undefined;
|
|
140
|
+
// Startpage proxies Google, so the operator set works inline; rebuild via
|
|
141
|
+
// the shared scraper formatter to canonicalize aliases (domain: → site:,
|
|
142
|
+
// since: → after:, …) and demote scraper-hostile operators. Directive-free
|
|
143
|
+
// queries pass through byte-identical.
|
|
144
|
+
const query = formatScraperQuery(params.query, params.parsedQuery);
|
|
139
145
|
|
|
140
146
|
const formInputs = await fetchFormInputs(fetchImpl, signal);
|
|
141
147
|
let page: LoadedHtmlPage;
|
|
142
148
|
if (formInputs) {
|
|
143
149
|
const form = new URLSearchParams(formInputs);
|
|
144
|
-
form.set("query",
|
|
150
|
+
form.set("query", query);
|
|
145
151
|
if (withDate) form.set("with_date", withDate);
|
|
146
152
|
page = await browserFetch(STARTPAGE_SEARCH_URL, {
|
|
147
153
|
fetch: fetchImpl,
|
|
@@ -152,7 +158,7 @@ async function callStartpageHtml(params: SearchParams): Promise<string> {
|
|
|
152
158
|
});
|
|
153
159
|
} else {
|
|
154
160
|
const url = new URL(STARTPAGE_SEARCH_URL);
|
|
155
|
-
url.searchParams.set("query",
|
|
161
|
+
url.searchParams.set("query", query);
|
|
156
162
|
if (withDate) url.searchParams.set("with_date", withDate);
|
|
157
163
|
page = await browserFetch(url.href, {
|
|
158
164
|
fetch: fetchImpl,
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { type ApiKey, type AuthStorage, type FetchImpl, getEnvApiKey, withAuth } from "@oh-my-pi/pi-ai";
|
|
9
9
|
import type { SearchResponse, SearchSource } from "../../../web/search/types";
|
|
10
10
|
import { SearchProviderError } from "../../../web/search/types";
|
|
11
|
+
import { formatQuery, parseSearchQuery } from "../query";
|
|
11
12
|
import type { SearchParams } from "./base";
|
|
12
13
|
import { SearchProvider } from "./base";
|
|
13
14
|
import { classifyProviderHttpError, withHardTimeout } from "./utils";
|
|
@@ -73,8 +74,13 @@ export async function searchSynthetic(params: SearchParamsWithFetch): Promise<Se
|
|
|
73
74
|
sessionId: params.sessionId,
|
|
74
75
|
});
|
|
75
76
|
|
|
77
|
+
const parsed = params.parsedQuery ?? parseSearchQuery(params.query);
|
|
78
|
+
const query = parsed.hasDirectives
|
|
79
|
+
? formatQuery(parsed, { phrases: true, negation: true, site: true })
|
|
80
|
+
: params.query;
|
|
81
|
+
|
|
76
82
|
const fetchImpl = params.fetch;
|
|
77
|
-
const data = await withAuth(keyOrResolver, key => callSyntheticSearch(key,
|
|
83
|
+
const data = await withAuth(keyOrResolver, key => callSyntheticSearch(key, query, params.signal, fetchImpl), {
|
|
78
84
|
signal: params.signal,
|
|
79
85
|
missingKeyMessage: "Synthetic credentials not found. Set SYNTHETIC_API_KEY or login with 'omp /login synthetic'.",
|
|
80
86
|
});
|