@nite07/search-core 0.0.0-stage → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +100 -2
  3. package/dist/adapters.d.ts +22 -0
  4. package/dist/adapters.js +286 -0
  5. package/dist/browser-backends.d.ts +7 -0
  6. package/dist/browser-backends.js +24 -0
  7. package/dist/browser.d.ts +46 -0
  8. package/dist/browser.js +130 -0
  9. package/dist/catalog.d.ts +3 -0
  10. package/dist/catalog.js +31 -0
  11. package/dist/config.d.ts +12 -0
  12. package/dist/config.js +166 -0
  13. package/dist/engine.d.ts +27 -0
  14. package/dist/engine.js +258 -0
  15. package/dist/index.d.ts +27 -0
  16. package/dist/index.js +20 -0
  17. package/dist/locale-map.d.ts +18 -0
  18. package/dist/locale-map.js +38 -0
  19. package/dist/mcp.d.ts +4 -0
  20. package/dist/mcp.js +57 -0
  21. package/dist/media.d.ts +39 -0
  22. package/dist/media.js +84 -0
  23. package/dist/reddit/auth.d.ts +77 -0
  24. package/dist/reddit/auth.js +257 -0
  25. package/dist/reddit/device.d.ts +16 -0
  26. package/dist/reddit/device.js +201 -0
  27. package/dist/reddit/gql.d.ts +77 -0
  28. package/dist/reddit/gql.js +162 -0
  29. package/dist/reddit/index.d.ts +42 -0
  30. package/dist/reddit/index.js +248 -0
  31. package/dist/reddit/shape.d.ts +114 -0
  32. package/dist/reddit/shape.js +317 -0
  33. package/dist/static.d.ts +9 -0
  34. package/dist/static.js +8 -0
  35. package/dist/tencent.d.ts +3 -0
  36. package/dist/tencent.js +111 -0
  37. package/dist/transport.d.ts +20 -0
  38. package/dist/transport.js +97 -0
  39. package/dist/types.d.ts +162 -0
  40. package/dist/types.js +1 -0
  41. package/dist/ua.d.ts +6 -0
  42. package/dist/ua.js +34 -0
  43. package/dist/version.d.ts +3 -0
  44. package/dist/version.js +13 -0
  45. package/dist/x/auth.d.ts +28 -0
  46. package/dist/x/auth.js +68 -0
  47. package/dist/x/discovery.d.ts +47 -0
  48. package/dist/x/discovery.js +180 -0
  49. package/dist/x/index.d.ts +32 -0
  50. package/dist/x/index.js +45 -0
  51. package/dist/x/official.d.ts +23 -0
  52. package/dist/x/official.js +77 -0
  53. package/dist/x/parse.d.ts +53 -0
  54. package/dist/x/parse.js +219 -0
  55. package/dist/x/session.d.ts +25 -0
  56. package/dist/x/session.js +103 -0
  57. package/dist/x/thread.d.ts +30 -0
  58. package/dist/x/thread.js +88 -0
  59. package/dist/xhs/index.d.ts +94 -0
  60. package/dist/xhs/index.js +160 -0
  61. package/dist/xhs/scripts.d.ts +29 -0
  62. package/dist/xhs/scripts.js +127 -0
  63. package/package.json +55 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 nite
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,101 @@
1
- # Temporary Holding Version
1
+ # @nite07/search-core
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ A provider-agnostic multi-backend web search engine in TypeScript. It ships the backend catalog,
4
+ the per-backend adapters, the HTTP transport and a single `SearchEngine`, with no dependency on any
5
+ host framework — the DeepSeek Harness plugin and the MCP server are both thin shells over this.
6
+
7
+ ## What it guarantees
8
+
9
+ - **Explicit access mode per backend.** A backend is used in exactly the mode the caller selected:
10
+ anonymous, key, OAuth or a session cookie. There is no fallback between modes, keys or endpoints.
11
+ - **Bounded retries.** Only the configured HTTP statuses are retried, `Retry-After` is honoured, and
12
+ a retry never changes the provider, the mode or the credential.
13
+ - **Honest failures.** A partial failure returns what worked and names what failed; a total failure
14
+ says the search failed instead of returning an empty result. Failures are never cached.
15
+ - **Cost visibility.** Every access mode carries a cost tier and a dated quota note, so a caller can
16
+ show a user the price before enabling anything.
17
+
18
+ ## Install
19
+
20
+ ```sh
21
+ npm install @nite07/search-core
22
+ ```
23
+
24
+ ## Use
25
+
26
+ ```ts
27
+ import { SearchEngine, catalog, defaultConfig } from '@nite07/search-core';
28
+
29
+ const config = defaultConfig();
30
+ config.backends.exa!.enabled = true; // anonymous mode works without a credential
31
+ config.backends.keenable!.enabled = true;
32
+
33
+ const engine = new SearchEngine(
34
+ config,
35
+ async reference => process.env[reference], // resolve credentials however your host stores them
36
+ );
37
+
38
+ const result = await engine.search({ query: 'cordis plugin loader', maxResults: 8 });
39
+ console.log(result.sources); // citeable sources
40
+ console.log(result.content); // failure report, when anything failed
41
+ ```
42
+
43
+ Credential names are fixed in code, not configuration fields. Credentialed HTTP modes resolve
44
+ `backendCredentialRef(id)` (`DSH_SEARCH_<BACKEND_ID>_KEY`, with hyphens replaced by underscores and
45
+ the ID uppercased); browser modes resolve only `browserCredentialRef()`
46
+ (`DSH_SEARCH_BROWSER_TOKEN`). Both helpers are exported for hosts that store credentials. Anonymous
47
+ modes never resolve a credential. `parseConfig` drops saved `credentialRef` fields rather than using
48
+ them to redirect a lookup.
49
+
50
+ `search(input, signal?, onlyBackend?, overrides?)` accepts per-call strategy overrides —
51
+ `mode`, `backends`, `exclude`, `backend`, `compact`, `maxResults` — without touching the saved
52
+ configuration.
53
+
54
+ ## Backends
55
+
56
+ 23 entries across four access modes: anonymous tiers, free allowances, trial credits, paid APIs, and
57
+ two session-cookie backends (X, Reddit) that use the account you already have. `catalog` carries the
58
+ mode list, cost tier, quota note and options for each one. Reddit threads are available through
59
+ `fetchRedditThread`, X searches through `searchX`, and X conversations through `fetchXThread`.
60
+
61
+ ### Xiaohongshu
62
+
63
+ Xiaohongshu only answers its own page runtime, so this backend drives a browser you host rather than
64
+ an HTTP endpoint. Set the endpoint in the shared `browser` block and store the token printed by
65
+ the container under `DSH_SEARCH_BROWSER_TOKEN` in the host's credential store. No credential name
66
+ is configured on the browser or backend. Signing in happens inside the browser, over VNC:
67
+
68
+ ```ts
69
+ config.browser.endpoint = 'ws://127.0.0.1:3000/ws';
70
+ config.backends.xiaohongshu!.enabled = true;
71
+ config.backends.xiaohongshu!.mode = 'browser';
72
+ ```
73
+
74
+ Every backend in `browser` mode uses that shared endpoint when it is set, so one container can
75
+ serve several of them. If the shared endpoint is empty, the backend's own `endpoint` is used.
76
+ Token lookup always uses `DSH_SEARCH_BROWSER_TOKEN`; there is no per-backend token fallback. A
77
+ missing token does not block attachment to a browser endpoint that accepts unauthenticated clients.
78
+
79
+ The repository ships [`image/`](./image/README.md): a container with Patchright-driven Google
80
+ Chrome, a VNC surface to sign in once, a token-protected Playwright endpoint and a profile volume. Login and
81
+ profile management stay with whoever owns that browser — this library only attaches, and only reads.
82
+
83
+ `patchright-core` is a direct dependency: a patched Playwright that closes the CDP leaks detection
84
+ systems probe for. Both sides of the connection must share a major.minor, so the browser image pins
85
+ the same version; Playwright's protocol refuses a mismatch with `428 Precondition Required`. Reads
86
+ available: `searchXhs`, `fetchXhsNote`, `fetchXhsComments`.
87
+
88
+ X is read the way the site itself reads it: the operations the web app embeds in a page (search and
89
+ per-post conversation) are mined from that document, then called directly with the cookies. No
90
+ transaction-id signing, no bundle crawling, and a stale operation id is rediscovered once.
91
+
92
+ ## Development
93
+
94
+ ```sh
95
+ npm install
96
+ npm run check # typecheck, tests, build
97
+ ```
98
+
99
+ ## License
100
+
101
+ MIT.
@@ -0,0 +1,22 @@
1
+ import type { AdapterInput, AdapterResult, Source } from './types.js';
2
+ import { type TransportOptions } from './transport.js';
3
+ /**
4
+ * Backends that read through the shared browser. A backend added here gets the endpoint, the session
5
+ * file and the timeout from the shared browser configuration, and needs no other wiring.
6
+ */
7
+ export interface BrowserRead {
8
+ /** Browser reads get the endpoint, the optional session file and the caller's signal. */
9
+ search(input: {
10
+ query: string;
11
+ maxResults: number;
12
+ options: Record<string, unknown>;
13
+ key?: string;
14
+ }, ctx: {
15
+ wsEndpoint: string;
16
+ statePath?: string;
17
+ timeoutMs: number;
18
+ signal: AbortSignal;
19
+ }): Promise<Source[]>;
20
+ }
21
+ export declare const browserBackends: Record<string, BrowserRead>;
22
+ export declare function runAdapter(id: string, i: AdapterInput, t: TransportOptions): Promise<AdapterResult>;
@@ -0,0 +1,286 @@
1
+ import { requestJson, requestText, SearchError } from './transport.js';
2
+ import { callMcp } from './mcp.js';
3
+ import { languageFor, regionFor } from './locale-map.js';
4
+ import { searchReddit } from './reddit/index.js';
5
+ import { parseXCredentials, searchX } from './x/index.js';
6
+ const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : {};
7
+ const array = (v) => Array.isArray(v) ? v : [];
8
+ const string = (v) => typeof v === 'string' ? v : undefined;
9
+ function sources(rows, keys = { title: 'title', url: 'url', snippet: 'snippet', date: 'published_at' }) {
10
+ if (!Array.isArray(rows))
11
+ throw new SearchError('protocol', 'Expected a results array from provider.');
12
+ return rows.flatMap(v => {
13
+ const r = obj(v);
14
+ const url = string(r[keys.url]);
15
+ if (!url)
16
+ return [];
17
+ try {
18
+ const u = new URL(url);
19
+ if (!['https:', 'http:'].includes(u.protocol) || u.username || u.password)
20
+ return [];
21
+ }
22
+ catch {
23
+ return [];
24
+ }
25
+ // Every field is optional, and an explicit undefined is not valid JSON: the tool layer refuses
26
+ // such a value outright, so absent fields are simply left out.
27
+ const title = string(r[keys.title]);
28
+ const snippet = string(r[keys.snippet]);
29
+ const date = string(r[keys.date]);
30
+ return [{ url, ...(title ? { title } : {}), ...(snippet ? { snippet } : {}), ...(date ? { publishedAt: date } : {}) }];
31
+ });
32
+ }
33
+ export const browserBackends = {};
34
+ import { registerBrowserBackends } from './browser-backends.js';
35
+ registerBrowserBackends(browserBackends);
36
+ export async function runAdapter(id, i, t) {
37
+ const b = i.backend;
38
+ const o = b.options;
39
+ const c = i.config;
40
+ const headers = { 'Content-Type': 'application/json' };
41
+ const browser = browserBackends[id];
42
+ if (browser) {
43
+ // The browser belongs to whoever runs the container, so there is no attempt budget to reserve:
44
+ // one page read is one read on their own infrastructure.
45
+ const endpoint = String(c.browser?.endpoint || o.wsEndpoint || b.endpoint || '');
46
+ if (!endpoint)
47
+ throw new SearchError('configuration', `${id} reads through a browser, but no endpoint is configured. Set the shared browser endpoint, or this backend's own.`);
48
+ const statePath = String(o.statePath || c.browser?.statePath || '');
49
+ const timeoutMs = Number(c.browser?.timeoutMs || b.timeoutMs);
50
+ const sources = await browser.search({ query: i.query, maxResults: i.maxResults, options: o, key: i.key }, { wsEndpoint: endpoint, ...(statePath ? { statePath } : {}), timeoutMs, signal: t.signal });
51
+ return { sources };
52
+ }
53
+ if (id === 'x') {
54
+ const credentials = parseXCredentials(i.key);
55
+ if (b.mode !== 'key' && b.mode !== 'session')
56
+ throw new SearchError('configuration', 'X supports only the key and session access modes.');
57
+ // The X adapters own their HTTP calls, so reserve the attempt budget here.
58
+ await t.beforeAttempt();
59
+ t.onAttempt();
60
+ const postsPerSearch = Number(o.postsPerSearch ?? 0);
61
+ return { sources: await searchX({ query: i.query, maxResults: i.maxResults }, { mode: b.mode === 'key' ? 'key' : 'session', credentials, signal: t.signal, timeoutMs: b.timeoutMs, postsPerSearch: postsPerSearch > 0 ? postsPerSearch : undefined, fetch: t.fetch }) };
62
+ }
63
+ if (id === 'reddit') {
64
+ let credentials = {};
65
+ if (b.mode === 'oauth') {
66
+ try {
67
+ const parsed = JSON.parse(i.key ?? '');
68
+ if (!parsed || typeof parsed !== 'object')
69
+ throw 0;
70
+ credentials = parsed;
71
+ }
72
+ catch {
73
+ throw new SearchError('configuration', 'Reddit OAuth credentials must be JSON with clientId and clientSecret.');
74
+ }
75
+ if (!credentials.clientId || !credentials.clientSecret)
76
+ throw new SearchError('configuration', 'Reddit OAuth credentials require clientId and clientSecret.');
77
+ }
78
+ // This adapter owns its HTTP calls (token flow + GraphQL), so reserve the attempt budget here.
79
+ await t.beforeAttempt();
80
+ t.onAttempt();
81
+ return { sources: await searchReddit({ query: i.query, maxResults: i.maxResults, sort: String(o.sort || 'relevance'), time: String(o.time || 'all'), subreddit: String(o.subreddit || '') || undefined }, { mode: b.mode === 'oauth' ? 'oauth' : 'anonymous', credentials, signal: t.signal, timeoutMs: b.timeoutMs, fetch: t.fetch }) };
82
+ }
83
+ const key = b.mode === 'key' ? i.key : undefined;
84
+ const json = (body, h = headers) => requestJson(b.endpoint, { method: 'POST', headers: h, body: JSON.stringify(body) }, t);
85
+ const get = (params, h = headers) => { const url = new URL(b.endpoint); for (const [k, v] of Object.entries(params))
86
+ if (v !== undefined && v !== '')
87
+ url.searchParams.set(k, String(v)); return requestJson(url.href, { headers: h }, t); };
88
+ const bearer = () => { if (key)
89
+ headers.Authorization = `Bearer ${key}`; };
90
+ if (id === 'keenable') {
91
+ headers['X-Keenable-Title'] = String(o.clientTitle);
92
+ if (key)
93
+ headers['X-API-Key'] = key;
94
+ const r = obj(await json({ query: i.query, max_results: i.maxResults, snippet_max_length: o.snippet_max_length, ...(c.includeDomains.length ? { site: c.includeDomains.join(',') } : {}) }));
95
+ return { sources: sources(r.results) };
96
+ }
97
+ if (id === 'anysearch') {
98
+ bearer();
99
+ const r = obj(await json({ query: i.query, max_results: Math.min(i.maxResults, 10), zone: o.zone, format: 'json', ...(languageFor('anysearch', c.language) ? { language: languageFor('anysearch', c.language) } : {}) }));
100
+ if (r.code !== undefined && r.code !== 0 && r.code !== 200)
101
+ throw new SearchError('protocol', 'AnySearch reported an application-level error; inspect the account console.');
102
+ return { sources: sources(obj(r.data).results) };
103
+ }
104
+ if (id === 'linkup') {
105
+ bearer();
106
+ const r = obj(await json({ q: i.query, depth: o.depth, outputType: 'searchResults', maxResults: i.maxResults }));
107
+ return { sources: sources(r.results, { title: 'name', url: 'url', snippet: 'content', date: 'publishedAt' }) };
108
+ }
109
+ if (id === 'tavily') {
110
+ bearer();
111
+ const r = obj(await json({ query: i.query, max_results: Math.min(i.maxResults, 20), search_depth: o.search_depth, auto_parameters: false, topic: o.topic, include_answer: o.include_answer, include_domains: c.includeDomains, exclude_domains: c.excludeDomains, ...(c.recency !== 'any' ? { time_range: c.recency } : {}) }));
112
+ return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'content', date: 'published_date' }), content: string(r.answer) };
113
+ }
114
+ if (id === 'brave') {
115
+ if (key)
116
+ headers['X-Subscription-Token'] = key;
117
+ const r = obj(await get({ q: i.query, count: Math.min(i.maxResults, 20), search_lang: languageFor('brave', c.language), country: regionFor('brave', c.region), safesearch: c.safeSearch ? 'strict' : 'off', freshness: { any: undefined, day: 'pd', week: 'pw', month: 'pm', year: 'py' }[c.recency] }));
118
+ return { sources: sources(obj(r.web).results, { title: 'title', url: 'url', snippet: 'description', date: 'page_age' }) };
119
+ }
120
+ if (id === 'serper') {
121
+ if (key)
122
+ headers['X-API-KEY'] = key;
123
+ const r = obj(await json({ q: i.query, num: i.maxResults, hl: languageFor('serper', c.language), gl: regionFor('serper', c.region), tbs: { any: undefined, day: 'qdr:d', week: 'qdr:w', month: 'qdr:m', year: 'qdr:y' }[c.recency] }));
124
+ return { sources: sources(r.organic, { title: 'title', url: 'link', snippet: 'snippet', date: 'date' }) };
125
+ }
126
+ if (id === 'searxng') {
127
+ bearer();
128
+ const r = obj(await get({ q: i.query, format: 'json', language: languageFor('searxng', c.language) || 'auto', time_range: c.recency === 'any' ? undefined : c.recency, safesearch: c.safeSearch ? 2 : 0, engines: o.engines, categories: o.categories }));
129
+ return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'content', date: 'publishedDate' }) };
130
+ }
131
+ if (id === 'exa' && b.mode === 'key') {
132
+ headers['x-api-key'] = key;
133
+ const date = c.recency === 'any' ? undefined : new Date(Date.now() - ({ day: 1, week: 7, month: 30, year: 365 }[c.recency]) * 86400000).toISOString();
134
+ const r = obj(await json({ query: i.query, numResults: i.maxResults, type: o.type, includeDomains: c.includeDomains, excludeDomains: c.excludeDomains, startPublishedDate: date, contents: { text: { maxCharacters: o.textMaxCharacters } } }));
135
+ return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'text', date: 'publishedDate' }) };
136
+ }
137
+ if ((id === 'exa' || id === 'parallel' || id === 'you') && b.mode === 'anonymous') {
138
+ const tool = id === 'exa' ? 'web_search_exa' : id === 'you' ? 'you-search' : 'web_search';
139
+ const args = id === 'exa' ? { query: i.query, objective: i.query, numResults: i.maxResults } : id === 'you' ? { query: i.query, count: i.maxResults } : { objective: i.query, search_queries: [i.query] };
140
+ const endpoint = new URL(b.endpoint);
141
+ if (id === 'you')
142
+ endpoint.searchParams.set('profile', 'free');
143
+ if (id === 'parallel')
144
+ endpoint.searchParams.set('tools', 'web_search');
145
+ const result = await callMcp(endpoint.href, tool, args, t);
146
+ const structured = obj(result.structuredContent);
147
+ if (Array.isArray(structured.results))
148
+ return { sources: sources(structured.results) };
149
+ const text = array(result.content).map(v => string(obj(v).text) || '').join('\n');
150
+ if (!text.trim())
151
+ throw new SearchError('protocol', 'MCP search response has no recognized result payload.');
152
+ try {
153
+ const parsed = obj(JSON.parse(text));
154
+ const list = [parsed.results, obj(parsed.data).results, obj(parsed.results).web].find(Array.isArray);
155
+ if (Array.isArray(list))
156
+ return { sources: sources(list) };
157
+ }
158
+ catch (e) {
159
+ if (e instanceof SearchError)
160
+ throw e;
161
+ }
162
+ const found = [];
163
+ const blocks = text.split(/(?=Title:)/);
164
+ for (const block of blocks) {
165
+ const url = block.match(/URL:\s*(https?:\/\/\S+)/)?.[1];
166
+ if (url)
167
+ found.push({ url, title: block.match(/Title:\s*(.+)/)?.[1], snippet: block.split(/(?:Text|Content):\s*/).slice(1).join(' ').slice(0, 4000) });
168
+ }
169
+ if (!found.length && text.trim())
170
+ throw new SearchError('protocol', 'MCP result format contains no verifiable source URLs.');
171
+ return { sources: found };
172
+ }
173
+ if (id === 'parallel') {
174
+ if (key)
175
+ headers['x-api-key'] = key;
176
+ const r = obj(await json({ search_queries: [i.query], mode: o.mode, advanced_settings: { max_results: i.maxResults } }));
177
+ const rows = array(r.results).map(v => { const row = obj(v); return { ...row, excerpt: array(row.excerpts).filter(x => typeof x === 'string').join('\n').slice(0, Number(o.maxCharactersPerResult)) }; });
178
+ return { sources: sources(rows, { title: 'title', url: 'url', snippet: 'excerpt', date: 'publish_date' }) };
179
+ }
180
+ if (id === 'you') {
181
+ if (key)
182
+ headers['X-API-Key'] = key;
183
+ const r = obj(await json({ query: i.query, count: i.maxResults, strict_operators: true, ...(languageFor('you', c.language) ? { language: languageFor('you', c.language) } : {}), ...(regionFor('you', c.region) ? { country: regionFor('you', c.region) } : {}) }));
184
+ return { sources: sources(obj(r.results).web, { title: 'title', url: 'url', snippet: 'description', date: 'page_age' }) };
185
+ }
186
+ if (id === 'baidu') {
187
+ bearer();
188
+ const r = obj(await json({ messages: [{ role: 'user', content: i.query }], search_source: 'baidu_search_v2', edition: o.edition, resource_type_filter: [{ type: 'web', top_k: i.maxResults }] }));
189
+ return { sources: sources(r.references, { title: 'title', url: 'url', snippet: 'snippet', date: 'date' }) };
190
+ }
191
+ if (id === 'doubao') {
192
+ bearer();
193
+ const r = obj(await json({ Query: i.query, SearchType: 'web', Count: i.maxResults, Filter: { NeedUrl: true }, QueryControl: { QueryRewrite: o.queryRewrite } }));
194
+ if (obj(r.ResponseMetadata).Error)
195
+ throw new SearchError('protocol', 'Doubao returned an application-level error.');
196
+ return { sources: sources(obj(r.Result).WebResults, { title: 'Title', url: 'Url', snippet: 'Snippet', date: 'PublishTime' }) };
197
+ }
198
+ if (id === 'zhihu-web' || id === 'zhihu-site') {
199
+ bearer();
200
+ headers['X-Request-Timestamp'] = String(Math.floor(Date.now() / 1000));
201
+ const r = obj(await get({ Query: i.query, Count: Math.min(i.maxResults, id === 'zhihu-web' ? 20 : 10), ...(id === 'zhihu-web' ? { SearchDB: o.SearchDB, Filter: o.Filter } : {}) }));
202
+ if (r.Code !== 0)
203
+ throw new SearchError('protocol', 'Zhihu reported an application-level failure; check Access Secret and account permissions.');
204
+ return { sources: sources(obj(r.Data).Items, { title: 'Title', url: 'Url', snippet: 'ContentText', date: 'PublishTime' }) };
205
+ }
206
+ if (id === 'open-websearch') {
207
+ const r = obj(await json({ query: i.query, limit: i.maxResults, engines: [o.engine], searchMode: o.searchMode }));
208
+ if (r.status !== 'ok')
209
+ throw new SearchError('protocol', 'Self-hosted search daemon reported failure.');
210
+ return { sources: sources(r.data, { title: 'title', url: 'url', snippet: 'description', date: 'date' }) };
211
+ }
212
+ if (id === 'serpex') {
213
+ bearer();
214
+ const r = obj(await json({ q: i.query }));
215
+ return { sources: sources(r.results), warnings: ['Serpex controls its own internal upstream routing; the plugin did not switch providers.'] };
216
+ }
217
+ if (id === 'mojeek') {
218
+ const r = obj(await get({ q: i.query, api_key: key, fmt: 'json' }));
219
+ const response = obj(r.response);
220
+ if (response.status !== 'OK' && response.status !== 200 && response.status !== '200')
221
+ throw new SearchError('protocol', 'Mojeek reported an application-level error.');
222
+ return { sources: sources(response.results, { title: 'title', url: 'url', snippet: 'desc', date: 'date' }) };
223
+ }
224
+ if (id === 'deepseek') {
225
+ bearer();
226
+ const r = obj(await json({ query: i.query, count: i.maxResults, need_content: o.need_content }));
227
+ return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'snippet', date: 'date' }) };
228
+ }
229
+ if (id === 'anthropic') {
230
+ if (key)
231
+ headers['x-api-key'] = key;
232
+ headers['anthropic-version'] = '2023-06-01';
233
+ const r = obj(await json({ model: o.model, max_tokens: o.maxTokens, tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: o.maxUses, ...(c.includeDomains.length ? { allowed_domains: c.includeDomains } : {}), ...(c.excludeDomains.length ? { blocked_domains: c.excludeDomains } : {}) }], messages: [{ role: 'user', content: i.query }] }));
234
+ const result = [];
235
+ for (const block of array(r.content)) {
236
+ const v = obj(block);
237
+ if (v.type === 'web_search_tool_result') {
238
+ if (obj(v.content).type === 'web_search_tool_result_error')
239
+ throw new SearchError('protocol', 'Anthropic search tool reported failure.');
240
+ for (const x of array(v.content)) {
241
+ const s = obj(x);
242
+ if (s.type === 'web_search_result')
243
+ result.push(...sources([s], { title: 'title', url: 'url', snippet: 'page_content', date: 'page_age' }));
244
+ }
245
+ }
246
+ }
247
+ return { sources: result };
248
+ }
249
+ if (id === 'openai') {
250
+ bearer();
251
+ const r = obj(await json({ model: o.model, input: i.query, tools: [{ type: 'web_search' }], tool_choice: 'required', include: ['web_search_call.action.sources'], max_output_tokens: o.maxOutputTokens }));
252
+ if (r.error || r.status === 'failed' || r.status === 'incomplete')
253
+ throw new SearchError('protocol', 'OpenAI response did not complete.');
254
+ const result = [];
255
+ for (const output of array(r.output))
256
+ for (const block of array(obj(output).content))
257
+ for (const a of array(obj(block).annotations)) {
258
+ const citation = obj(a);
259
+ if (citation.type === 'url_citation')
260
+ result.push(...sources([citation]));
261
+ }
262
+ return { sources: result };
263
+ }
264
+ if (id === 'ddg') {
265
+ const raw = await requestText(b.endpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ q: i.query }).toString() }, t);
266
+ if (/anomaly|challenge-form|bots use DuckDuckGo/i.test(raw))
267
+ throw new SearchError('rate-limit', 'DuckDuckGo returned a challenge; no bypass was attempted.');
268
+ const decode = (s) => s.replace(/<[^>]*>/g, ' ').replace(/&amp;/g, '&').replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/\s+/g, ' ').trim();
269
+ const result = [];
270
+ for (const match of raw.matchAll(/<a\b[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g)) {
271
+ let url = decode(match[1]);
272
+ try {
273
+ const u = new URL(url, 'https://duckduckgo.com');
274
+ url = u.searchParams.get('uddg') || u.href;
275
+ }
276
+ catch {
277
+ continue;
278
+ }
279
+ result.push(...sources([{ url, title: decode(match[2]) }]));
280
+ }
281
+ if (!result.length && !/No results found|result--no-result/.test(raw))
282
+ throw new SearchError('protocol', 'DuckDuckGo HTML layout was not recognized.');
283
+ return { sources: result };
284
+ }
285
+ throw new SearchError('unverified', 'Backend adapter is not verified and cannot be called.');
286
+ }
@@ -0,0 +1,7 @@
1
+ import type { BrowserRead } from './adapters.js';
2
+ /**
3
+ * Every backend that needs the shared browser registers here, once. The point of the indirection is
4
+ * that adding the second one touches this file and nothing else: the adapter dispatch, the shared
5
+ * endpoint, the session file and the timeout all come from the same place.
6
+ */
7
+ export declare function registerBrowserBackends(registry: Record<string, BrowserRead>): void;
@@ -0,0 +1,24 @@
1
+ import { searchXhs } from './xhs/index.js';
2
+ /**
3
+ * Every backend that needs the shared browser registers here, once. The point of the indirection is
4
+ * that adding the second one touches this file and nothing else: the adapter dispatch, the shared
5
+ * endpoint, the session file and the timeout all come from the same place.
6
+ */
7
+ export function registerBrowserBackends(registry) {
8
+ registry.xiaohongshu = {
9
+ async search(input, ctx) {
10
+ const options = input.options;
11
+ return searchXhs({
12
+ query: input.query,
13
+ maxResults: input.maxResults,
14
+ ...(typeof options.sort === 'string' && options.sort !== '' ? { sort: options.sort } : {}),
15
+ }, {
16
+ wsEndpoint: ctx.wsEndpoint,
17
+ ...(input.key ? { token: input.key } : {}),
18
+ ...(ctx.statePath ? { statePath: ctx.statePath } : {}),
19
+ timeoutMs: ctx.timeoutMs,
20
+ signal: ctx.signal,
21
+ });
22
+ },
23
+ };
24
+ }
@@ -0,0 +1,46 @@
1
+ import { type Page } from 'patchright-core';
2
+ import { SearchError } from './transport.js';
3
+ /**
4
+ * One connection to the browser the user runs. The browser belongs to them: this class never
5
+ * launches anything, never installs anything and never touches the profile — it attaches, reuses
6
+ * the page that is already there, and evaluates scripts inside it.
7
+ */
8
+ export interface BrowserSessionOptions {
9
+ /** Playwright server endpoint, for example ws://127.0.0.1:3000/xhs. */
10
+ wsEndpoint: string;
11
+ /** Bearer token checked by the proxy in front of that endpoint. */
12
+ token?: string;
13
+ /** Page the browser is asked to open when it is not on the site yet. */
14
+ siteUrl: string;
15
+ timeoutMs: number;
16
+ /**
17
+ * Where the signed-in session is kept. The login itself happens in that browser over VNC; this
18
+ * file only lets a fresh connection start already signed in.
19
+ */
20
+ statePath?: string;
21
+ }
22
+ export declare class BrowserSession {
23
+ private readonly options;
24
+ private browser;
25
+ private page;
26
+ private savedAt;
27
+ private context;
28
+ constructor(options: BrowserSessionOptions);
29
+ private connect;
30
+ /** Reuse the page the browser already has; open one only when it has none. */
31
+ target(): Promise<Page>;
32
+ private ensureOnSite;
33
+ evaluate<T>(script: string): Promise<T>;
34
+ /**
35
+ * Persist the session so the next process starts signed in. Writing on every read would be
36
+ * wasteful, so it happens at most once a minute and only after a call actually succeeded.
37
+ */
38
+ persist(): Promise<void>;
39
+ close(): Promise<void>;
40
+ }
41
+ /** A refused endpoint, a missing one and a version mismatch each need a different fix. */
42
+ export declare function connectFailure(error: unknown, options: BrowserSessionOptions): SearchError;
43
+ /** The session for an endpoint, created on first use and reused for the life of the process. */
44
+ export declare function sessionFor(key: string, options: BrowserSessionOptions): BrowserSession;
45
+ /** Close every kept session; the host calls this when it shuts down. */
46
+ export declare function closeBrowserSessions(): Promise<void>;
@@ -0,0 +1,130 @@
1
+ // patchright-core is a patched Playwright: same protocol and API, with the CDP leaks closed.
2
+ // Both sides of the connection must share a major.minor, so the browser image pins the same version.
3
+ import { chromium } from 'patchright-core';
4
+ import { existsSync } from 'node:fs';
5
+ import { SearchError } from './transport.js';
6
+ const SITE_HOST = 'xiaohongshu.com';
7
+ export class BrowserSession {
8
+ options;
9
+ browser;
10
+ page;
11
+ savedAt = 0;
12
+ context;
13
+ constructor(options) {
14
+ this.options = options;
15
+ }
16
+ async connect() {
17
+ if (this.browser?.isConnected())
18
+ return this.browser;
19
+ try {
20
+ this.browser = await chromium.connect(this.options.wsEndpoint, {
21
+ timeout: this.options.timeoutMs,
22
+ ...(this.options.token ? { headers: { authorization: `Bearer ${this.options.token}` } } : {}),
23
+ });
24
+ return this.browser;
25
+ }
26
+ catch (error) {
27
+ throw connectFailure(error, this.options);
28
+ }
29
+ }
30
+ /** Reuse the page the browser already has; open one only when it has none. */
31
+ async target() {
32
+ if (this.page && !this.page.isClosed())
33
+ return this.ensureOnSite(this.page);
34
+ const browser = await this.connect();
35
+ // Contexts belong to the client that created them, so this process owns the only one: it is
36
+ // also the window the VNC viewer sees. A saved session is loaded here, if one exists.
37
+ const statePath = this.options.statePath;
38
+ const saved = statePath && existsSync(statePath) ? statePath : undefined;
39
+ const context = browser.contexts().find(candidate => candidate.pages().every(page => !page.isClosed()))
40
+ // viewport: null keeps the browser window at its real size instead of emulating one, which is
41
+ // what makes the window visible on the container's display for the one-time sign-in.
42
+ ?? await browser.newContext({ locale: 'zh-CN', viewport: null, ...(saved ? { storageState: saved } : {}) });
43
+ const existing = context.pages().find(page => !page.isClosed());
44
+ this.page = existing ?? await context.newPage();
45
+ this.context = context;
46
+ const ready = await this.ensureOnSite(this.page);
47
+ // Launching a browser server suppresses Chrome's startup window, so the tab has to ask for the
48
+ // window: without this the page runs, but the display the VNC viewer looks at stays empty.
49
+ await ready.bringToFront().catch(() => undefined);
50
+ return ready;
51
+ }
52
+ async ensureOnSite(page) {
53
+ if (page.url().includes(SITE_HOST))
54
+ return page;
55
+ try {
56
+ await page.goto(this.options.siteUrl, { waitUntil: 'domcontentloaded', timeout: this.options.timeoutMs });
57
+ }
58
+ catch (error) {
59
+ throw new SearchError('network', `Could not open ${this.options.siteUrl} in the browser: ${message(error)}`);
60
+ }
61
+ return page;
62
+ }
63
+ async evaluate(script) {
64
+ const page = await this.target();
65
+ try {
66
+ // Playwright treats a string argument as an expression and never calls it, so the generated
67
+ // source becomes a function here and is serialised to the page like any other page function.
68
+ const pageFunction = new Function(`return (${script});`)();
69
+ // The site's own client lives in the main world, so this call opts out of patchright's
70
+ // isolated-context default; isolation would hide the very runtime we need to reach.
71
+ return await page.evaluate(pageFunction, undefined, {}, false);
72
+ }
73
+ catch (error) {
74
+ throw new SearchError('protocol', `The Xiaohongshu page refused to run the read: ${message(error)}`);
75
+ }
76
+ }
77
+ /**
78
+ * Persist the session so the next process starts signed in. Writing on every read would be
79
+ * wasteful, so it happens at most once a minute and only after a call actually succeeded.
80
+ */
81
+ async persist() {
82
+ const statePath = this.options.statePath;
83
+ if (!statePath || !this.context)
84
+ return;
85
+ if (Date.now() - this.savedAt < 60_000)
86
+ return;
87
+ this.savedAt = Date.now();
88
+ await this.context.storageState({ path: statePath }).catch(() => undefined);
89
+ }
90
+ async close() {
91
+ // Only the connection is dropped; the user's browser keeps running with whatever page state it had.
92
+ await this.persist();
93
+ await this.browser?.close().catch(() => undefined);
94
+ this.browser = undefined;
95
+ this.page = undefined;
96
+ this.context = undefined;
97
+ }
98
+ }
99
+ const message = (error) => (error instanceof Error ? error.message : String(error));
100
+ /** A refused endpoint, a missing one and a version mismatch each need a different fix. */
101
+ export function connectFailure(error, options) {
102
+ const text = message(error);
103
+ if (/version mismatch/i.test(text) || /\b428\b/.test(text)) {
104
+ return new SearchError('protocol', `The Playwright server and this client disagree on protocol version, so no request was sent. Update whichever side is older; both must share the same major.minor. (${text.slice(0, 200)})`);
105
+ }
106
+ if (/\b401\b|unauthori[sz]ed/i.test(text)) {
107
+ return new SearchError('authentication', `The Playwright endpoint at ${options.wsEndpoint} rejected the token. Check the token the server printed at startup.`);
108
+ }
109
+ if (/ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|ETIMEDOUT/i.test(text)) {
110
+ return new SearchError('network', `No Playwright server answered at ${options.wsEndpoint}. Start the browser container, or fix the endpoint address.`);
111
+ }
112
+ return new SearchError('network', `Could not attach to the Playwright endpoint at ${options.wsEndpoint}: ${text.slice(0, 200)}`);
113
+ }
114
+ /** Sessions are shared by endpoint: one connection, many readers, one page per site. */
115
+ const sessions = new Map();
116
+ /** The session for an endpoint, created on first use and reused for the life of the process. */
117
+ export function sessionFor(key, options) {
118
+ let session = sessions.get(key);
119
+ if (!session) {
120
+ session = new BrowserSession(options);
121
+ sessions.set(key, session);
122
+ }
123
+ return session;
124
+ }
125
+ /** Close every kept session; the host calls this when it shuts down. */
126
+ export async function closeBrowserSessions() {
127
+ const open = [...sessions.values()];
128
+ sessions.clear();
129
+ await Promise.all(open.map(session => session.close().catch(() => undefined)));
130
+ }
@@ -0,0 +1,3 @@
1
+ import type { BackendDefinition } from './types.js';
2
+ export declare const catalog: BackendDefinition[];
3
+ export declare const backendById: (id: string) => BackendDefinition | undefined;