@nite07/search-core 0.1.0 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/adapters.d.ts +20 -1
- package/dist/adapters.js +21 -4
- package/dist/browser-backends.d.ts +7 -0
- package/dist/browser-backends.js +24 -0
- package/dist/browser.d.ts +46 -0
- package/dist/browser.js +130 -0
- package/dist/catalog.js +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.js +50 -17
- package/dist/engine.js +29 -3
- package/dist/index.d.ts +5 -1
- package/dist/index.js +4 -1
- package/dist/media.d.ts +39 -0
- package/dist/media.js +84 -0
- package/dist/reddit/shape.d.ts +20 -0
- package/dist/reddit/shape.js +50 -0
- package/dist/types.d.ts +12 -2
- package/dist/x/parse.d.ts +4 -0
- package/dist/x/parse.js +25 -2
- package/dist/xhs/index.d.ts +94 -0
- package/dist/xhs/index.js +160 -0
- package/dist/xhs/scripts.d.ts +29 -0
- package/dist/xhs/scripts.js +127 -0
- package/package.json +4 -1
package/README.md
CHANGED
|
@@ -40,6 +40,13 @@ console.log(result.sources); // citeable sources
|
|
|
40
40
|
console.log(result.content); // failure report, when anything failed
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
Credential names are fixed in code, not configuration fields. Credentialed HTTP modes resolve
|
|
44
|
+
`backendCredentialRef(id)` (`DSH_SEARCH_<BACKEND_ID>_KEY`, with hyphens replaced by underscores and
|
|
45
|
+
the ID uppercased); browser modes resolve only `browserCredentialRef()`
|
|
46
|
+
(`DSH_SEARCH_BROWSER_TOKEN`). Both helpers are exported for hosts that store credentials. Anonymous
|
|
47
|
+
modes never resolve a credential. `parseConfig` drops saved `credentialRef` fields rather than using
|
|
48
|
+
them to redirect a lookup.
|
|
49
|
+
|
|
43
50
|
`search(input, signal?, onlyBackend?, overrides?)` accepts per-call strategy overrides —
|
|
44
51
|
`mode`, `backends`, `exclude`, `backend`, `compact`, `maxResults` — without touching the saved
|
|
45
52
|
configuration.
|
|
@@ -51,6 +58,33 @@ two session-cookie backends (X, Reddit) that use the account you already have. `
|
|
|
51
58
|
mode list, cost tier, quota note and options for each one. Reddit threads are available through
|
|
52
59
|
`fetchRedditThread`, X searches through `searchX`, and X conversations through `fetchXThread`.
|
|
53
60
|
|
|
61
|
+
### Xiaohongshu
|
|
62
|
+
|
|
63
|
+
Xiaohongshu only answers its own page runtime, so this backend drives a browser you host rather than
|
|
64
|
+
an HTTP endpoint. Set the endpoint in the shared `browser` block and store the token printed by
|
|
65
|
+
the container under `DSH_SEARCH_BROWSER_TOKEN` in the host's credential store. No credential name
|
|
66
|
+
is configured on the browser or backend. Signing in happens inside the browser, over VNC:
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
config.browser.endpoint = 'ws://127.0.0.1:3000/ws';
|
|
70
|
+
config.backends.xiaohongshu!.enabled = true;
|
|
71
|
+
config.backends.xiaohongshu!.mode = 'browser';
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Every backend in `browser` mode uses that shared endpoint when it is set, so one container can
|
|
75
|
+
serve several of them. If the shared endpoint is empty, the backend's own `endpoint` is used.
|
|
76
|
+
Token lookup always uses `DSH_SEARCH_BROWSER_TOKEN`; there is no per-backend token fallback. A
|
|
77
|
+
missing token does not block attachment to a browser endpoint that accepts unauthenticated clients.
|
|
78
|
+
|
|
79
|
+
The repository ships [`image/`](./image/README.md): a container with Patchright-driven Google
|
|
80
|
+
Chrome, a VNC surface to sign in once, a token-protected Playwright endpoint and a profile volume. Login and
|
|
81
|
+
profile management stay with whoever owns that browser — this library only attaches, and only reads.
|
|
82
|
+
|
|
83
|
+
`patchright-core` is a direct dependency: a patched Playwright that closes the CDP leaks detection
|
|
84
|
+
systems probe for. Both sides of the connection must share a major.minor, so the browser image pins
|
|
85
|
+
the same version; Playwright's protocol refuses a mismatch with `428 Precondition Required`. Reads
|
|
86
|
+
available: `searchXhs`, `fetchXhsNote`, `fetchXhsComments`.
|
|
87
|
+
|
|
54
88
|
X is read the way the site itself reads it: the operations the web app embeds in a page (search and
|
|
55
89
|
per-post conversation) are mined from that document, then called directly with the cookies. No
|
|
56
90
|
transaction-id signing, no bundle crawling, and a stale operation id is rediscovered once.
|
package/dist/adapters.d.ts
CHANGED
|
@@ -1,3 +1,22 @@
|
|
|
1
|
-
import type { AdapterInput, AdapterResult } from './types.js';
|
|
1
|
+
import type { AdapterInput, AdapterResult, Source } from './types.js';
|
|
2
2
|
import { type TransportOptions } from './transport.js';
|
|
3
|
+
/**
|
|
4
|
+
* Backends that read through the shared browser. A backend added here gets the endpoint, the session
|
|
5
|
+
* file and the timeout from the shared browser configuration, and needs no other wiring.
|
|
6
|
+
*/
|
|
7
|
+
export interface BrowserRead {
|
|
8
|
+
/** Browser reads get the endpoint, the optional session file and the caller's signal. */
|
|
9
|
+
search(input: {
|
|
10
|
+
query: string;
|
|
11
|
+
maxResults: number;
|
|
12
|
+
options: Record<string, unknown>;
|
|
13
|
+
key?: string;
|
|
14
|
+
}, ctx: {
|
|
15
|
+
wsEndpoint: string;
|
|
16
|
+
statePath?: string;
|
|
17
|
+
timeoutMs: number;
|
|
18
|
+
signal: AbortSignal;
|
|
19
|
+
}): Promise<Source[]>;
|
|
20
|
+
}
|
|
21
|
+
export declare const browserBackends: Record<string, BrowserRead>;
|
|
3
22
|
export declare function runAdapter(id: string, i: AdapterInput, t: TransportOptions): Promise<AdapterResult>;
|
package/dist/adapters.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { requestJson, requestText, SearchError } from './transport.js';
|
|
2
2
|
import { callMcp } from './mcp.js';
|
|
3
3
|
import { languageFor, regionFor } from './locale-map.js';
|
|
4
|
-
import { searchTencent } from './tencent.js';
|
|
5
4
|
import { searchReddit } from './reddit/index.js';
|
|
6
5
|
import { parseXCredentials, searchX } from './x/index.js';
|
|
7
6
|
const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : {};
|
|
@@ -23,16 +22,34 @@ function sources(rows, keys = { title: 'title', url: 'url', snippet: 'snippet',
|
|
|
23
22
|
catch {
|
|
24
23
|
return [];
|
|
25
24
|
}
|
|
26
|
-
|
|
25
|
+
// Every field is optional, and an explicit undefined is not valid JSON: the tool layer refuses
|
|
26
|
+
// such a value outright, so absent fields are simply left out.
|
|
27
|
+
const title = string(r[keys.title]);
|
|
28
|
+
const snippet = string(r[keys.snippet]);
|
|
29
|
+
const date = string(r[keys.date]);
|
|
30
|
+
return [{ url, ...(title ? { title } : {}), ...(snippet ? { snippet } : {}), ...(date ? { publishedAt: date } : {}) }];
|
|
27
31
|
});
|
|
28
32
|
}
|
|
33
|
+
export const browserBackends = {};
|
|
34
|
+
import { registerBrowserBackends } from './browser-backends.js';
|
|
35
|
+
registerBrowserBackends(browserBackends);
|
|
29
36
|
export async function runAdapter(id, i, t) {
|
|
30
37
|
const b = i.backend;
|
|
31
38
|
const o = b.options;
|
|
32
39
|
const c = i.config;
|
|
33
40
|
const headers = { 'Content-Type': 'application/json' };
|
|
34
|
-
|
|
35
|
-
|
|
41
|
+
const browser = browserBackends[id];
|
|
42
|
+
if (browser) {
|
|
43
|
+
// The browser belongs to whoever runs the container, so there is no attempt budget to reserve:
|
|
44
|
+
// one page read is one read on their own infrastructure.
|
|
45
|
+
const endpoint = String(c.browser?.endpoint || o.wsEndpoint || b.endpoint || '');
|
|
46
|
+
if (!endpoint)
|
|
47
|
+
throw new SearchError('configuration', `${id} reads through a browser, but no endpoint is configured. Set the shared browser endpoint, or this backend's own.`);
|
|
48
|
+
const statePath = String(o.statePath || c.browser?.statePath || '');
|
|
49
|
+
const timeoutMs = Number(c.browser?.timeoutMs || b.timeoutMs);
|
|
50
|
+
const sources = await browser.search({ query: i.query, maxResults: i.maxResults, options: o, key: i.key }, { wsEndpoint: endpoint, ...(statePath ? { statePath } : {}), timeoutMs, signal: t.signal });
|
|
51
|
+
return { sources };
|
|
52
|
+
}
|
|
36
53
|
if (id === 'x') {
|
|
37
54
|
const credentials = parseXCredentials(i.key);
|
|
38
55
|
if (b.mode !== 'key' && b.mode !== 'session')
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { BrowserRead } from './adapters.js';
|
|
2
|
+
/**
|
|
3
|
+
* Every backend that needs the shared browser registers here, once. The point of the indirection is
|
|
4
|
+
* that adding the second one touches this file and nothing else: the adapter dispatch, the shared
|
|
5
|
+
* endpoint, the session file and the timeout all come from the same place.
|
|
6
|
+
*/
|
|
7
|
+
export declare function registerBrowserBackends(registry: Record<string, BrowserRead>): void;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { searchXhs } from './xhs/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Every backend that needs the shared browser registers here, once. The point of the indirection is
|
|
4
|
+
* that adding the second one touches this file and nothing else: the adapter dispatch, the shared
|
|
5
|
+
* endpoint, the session file and the timeout all come from the same place.
|
|
6
|
+
*/
|
|
7
|
+
export function registerBrowserBackends(registry) {
|
|
8
|
+
registry.xiaohongshu = {
|
|
9
|
+
async search(input, ctx) {
|
|
10
|
+
const options = input.options;
|
|
11
|
+
return searchXhs({
|
|
12
|
+
query: input.query,
|
|
13
|
+
maxResults: input.maxResults,
|
|
14
|
+
...(typeof options.sort === 'string' && options.sort !== '' ? { sort: options.sort } : {}),
|
|
15
|
+
}, {
|
|
16
|
+
wsEndpoint: ctx.wsEndpoint,
|
|
17
|
+
...(input.key ? { token: input.key } : {}),
|
|
18
|
+
...(ctx.statePath ? { statePath: ctx.statePath } : {}),
|
|
19
|
+
timeoutMs: ctx.timeoutMs,
|
|
20
|
+
signal: ctx.signal,
|
|
21
|
+
});
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { type Page } from 'patchright-core';
|
|
2
|
+
import { SearchError } from './transport.js';
|
|
3
|
+
/**
|
|
4
|
+
* One connection to the browser the user runs. The browser belongs to them: this class never
|
|
5
|
+
* launches anything, never installs anything and never touches the profile — it attaches, reuses
|
|
6
|
+
* the page that is already there, and evaluates scripts inside it.
|
|
7
|
+
*/
|
|
8
|
+
export interface BrowserSessionOptions {
|
|
9
|
+
/** Playwright server endpoint, for example ws://127.0.0.1:3000/xhs. */
|
|
10
|
+
wsEndpoint: string;
|
|
11
|
+
/** Bearer token checked by the proxy in front of that endpoint. */
|
|
12
|
+
token?: string;
|
|
13
|
+
/** Page the browser is asked to open when it is not on the site yet. */
|
|
14
|
+
siteUrl: string;
|
|
15
|
+
timeoutMs: number;
|
|
16
|
+
/**
|
|
17
|
+
* Where the signed-in session is kept. The login itself happens in that browser over VNC; this
|
|
18
|
+
* file only lets a fresh connection start already signed in.
|
|
19
|
+
*/
|
|
20
|
+
statePath?: string;
|
|
21
|
+
}
|
|
22
|
+
export declare class BrowserSession {
|
|
23
|
+
private readonly options;
|
|
24
|
+
private browser;
|
|
25
|
+
private page;
|
|
26
|
+
private savedAt;
|
|
27
|
+
private context;
|
|
28
|
+
constructor(options: BrowserSessionOptions);
|
|
29
|
+
private connect;
|
|
30
|
+
/** Reuse the page the browser already has; open one only when it has none. */
|
|
31
|
+
target(): Promise<Page>;
|
|
32
|
+
private ensureOnSite;
|
|
33
|
+
evaluate<T>(script: string): Promise<T>;
|
|
34
|
+
/**
|
|
35
|
+
* Persist the session so the next process starts signed in. Writing on every read would be
|
|
36
|
+
* wasteful, so it happens at most once a minute and only after a call actually succeeded.
|
|
37
|
+
*/
|
|
38
|
+
persist(): Promise<void>;
|
|
39
|
+
close(): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
/** A refused endpoint, a missing one and a version mismatch each need a different fix. */
|
|
42
|
+
export declare function connectFailure(error: unknown, options: BrowserSessionOptions): SearchError;
|
|
43
|
+
/** The session for an endpoint, created on first use and reused for the life of the process. */
|
|
44
|
+
export declare function sessionFor(key: string, options: BrowserSessionOptions): BrowserSession;
|
|
45
|
+
/** Close every kept session; the host calls this when it shuts down. */
|
|
46
|
+
export declare function closeBrowserSessions(): Promise<void>;
|
package/dist/browser.js
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// patchright-core is a patched Playwright: same protocol and API, with the CDP leaks closed.
|
|
2
|
+
// Both sides of the connection must share a major.minor, so the browser image pins the same version.
|
|
3
|
+
import { chromium } from 'patchright-core';
|
|
4
|
+
import { existsSync } from 'node:fs';
|
|
5
|
+
import { SearchError } from './transport.js';
|
|
6
|
+
const SITE_HOST = 'xiaohongshu.com';
|
|
7
|
+
export class BrowserSession {
|
|
8
|
+
options;
|
|
9
|
+
browser;
|
|
10
|
+
page;
|
|
11
|
+
savedAt = 0;
|
|
12
|
+
context;
|
|
13
|
+
constructor(options) {
|
|
14
|
+
this.options = options;
|
|
15
|
+
}
|
|
16
|
+
async connect() {
|
|
17
|
+
if (this.browser?.isConnected())
|
|
18
|
+
return this.browser;
|
|
19
|
+
try {
|
|
20
|
+
this.browser = await chromium.connect(this.options.wsEndpoint, {
|
|
21
|
+
timeout: this.options.timeoutMs,
|
|
22
|
+
...(this.options.token ? { headers: { authorization: `Bearer ${this.options.token}` } } : {}),
|
|
23
|
+
});
|
|
24
|
+
return this.browser;
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
throw connectFailure(error, this.options);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Reuse the page the browser already has; open one only when it has none. */
|
|
31
|
+
async target() {
|
|
32
|
+
if (this.page && !this.page.isClosed())
|
|
33
|
+
return this.ensureOnSite(this.page);
|
|
34
|
+
const browser = await this.connect();
|
|
35
|
+
// Contexts belong to the client that created them, so this process owns the only one: it is
|
|
36
|
+
// also the window the VNC viewer sees. A saved session is loaded here, if one exists.
|
|
37
|
+
const statePath = this.options.statePath;
|
|
38
|
+
const saved = statePath && existsSync(statePath) ? statePath : undefined;
|
|
39
|
+
const context = browser.contexts().find(candidate => candidate.pages().every(page => !page.isClosed()))
|
|
40
|
+
// viewport: null keeps the browser window at its real size instead of emulating one, which is
|
|
41
|
+
// what makes the window visible on the container's display for the one-time sign-in.
|
|
42
|
+
?? await browser.newContext({ locale: 'zh-CN', viewport: null, ...(saved ? { storageState: saved } : {}) });
|
|
43
|
+
const existing = context.pages().find(page => !page.isClosed());
|
|
44
|
+
this.page = existing ?? await context.newPage();
|
|
45
|
+
this.context = context;
|
|
46
|
+
const ready = await this.ensureOnSite(this.page);
|
|
47
|
+
// Launching a browser server suppresses Chrome's startup window, so the tab has to ask for the
|
|
48
|
+
// window: without this the page runs, but the display the VNC viewer looks at stays empty.
|
|
49
|
+
await ready.bringToFront().catch(() => undefined);
|
|
50
|
+
return ready;
|
|
51
|
+
}
|
|
52
|
+
async ensureOnSite(page) {
|
|
53
|
+
if (page.url().includes(SITE_HOST))
|
|
54
|
+
return page;
|
|
55
|
+
try {
|
|
56
|
+
await page.goto(this.options.siteUrl, { waitUntil: 'domcontentloaded', timeout: this.options.timeoutMs });
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
throw new SearchError('network', `Could not open ${this.options.siteUrl} in the browser: ${message(error)}`);
|
|
60
|
+
}
|
|
61
|
+
return page;
|
|
62
|
+
}
|
|
63
|
+
async evaluate(script) {
|
|
64
|
+
const page = await this.target();
|
|
65
|
+
try {
|
|
66
|
+
// Playwright treats a string argument as an expression and never calls it, so the generated
|
|
67
|
+
// source becomes a function here and is serialised to the page like any other page function.
|
|
68
|
+
const pageFunction = new Function(`return (${script});`)();
|
|
69
|
+
// The site's own client lives in the main world, so this call opts out of patchright's
|
|
70
|
+
// isolated-context default; isolation would hide the very runtime we need to reach.
|
|
71
|
+
return await page.evaluate(pageFunction, undefined, {}, false);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
throw new SearchError('protocol', `The Xiaohongshu page refused to run the read: ${message(error)}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Persist the session so the next process starts signed in. Writing on every read would be
|
|
79
|
+
* wasteful, so it happens at most once a minute and only after a call actually succeeded.
|
|
80
|
+
*/
|
|
81
|
+
async persist() {
|
|
82
|
+
const statePath = this.options.statePath;
|
|
83
|
+
if (!statePath || !this.context)
|
|
84
|
+
return;
|
|
85
|
+
if (Date.now() - this.savedAt < 60_000)
|
|
86
|
+
return;
|
|
87
|
+
this.savedAt = Date.now();
|
|
88
|
+
await this.context.storageState({ path: statePath }).catch(() => undefined);
|
|
89
|
+
}
|
|
90
|
+
async close() {
|
|
91
|
+
// Only the connection is dropped; the user's browser keeps running with whatever page state it had.
|
|
92
|
+
await this.persist();
|
|
93
|
+
await this.browser?.close().catch(() => undefined);
|
|
94
|
+
this.browser = undefined;
|
|
95
|
+
this.page = undefined;
|
|
96
|
+
this.context = undefined;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const message = (error) => (error instanceof Error ? error.message : String(error));
|
|
100
|
+
/** A refused endpoint, a missing one and a version mismatch each need a different fix. */
|
|
101
|
+
export function connectFailure(error, options) {
|
|
102
|
+
const text = message(error);
|
|
103
|
+
if (/version mismatch/i.test(text) || /\b428\b/.test(text)) {
|
|
104
|
+
return new SearchError('protocol', `The Playwright server and this client disagree on protocol version, so no request was sent. Update whichever side is older; both must share the same major.minor. (${text.slice(0, 200)})`);
|
|
105
|
+
}
|
|
106
|
+
if (/\b401\b|unauthori[sz]ed/i.test(text)) {
|
|
107
|
+
return new SearchError('authentication', `The Playwright endpoint at ${options.wsEndpoint} rejected the token. Check the token the server printed at startup.`);
|
|
108
|
+
}
|
|
109
|
+
if (/ECONNREFUSED|ENOTFOUND|EHOSTUNREACH|ETIMEDOUT/i.test(text)) {
|
|
110
|
+
return new SearchError('network', `No Playwright server answered at ${options.wsEndpoint}. Start the browser container, or fix the endpoint address.`);
|
|
111
|
+
}
|
|
112
|
+
return new SearchError('network', `Could not attach to the Playwright endpoint at ${options.wsEndpoint}: ${text.slice(0, 200)}`);
|
|
113
|
+
}
|
|
114
|
+
/** Sessions are shared by endpoint: one connection, many readers, one page per site. */
|
|
115
|
+
const sessions = new Map();
|
|
116
|
+
/** The session for an endpoint, created on first use and reused for the life of the process. */
|
|
117
|
+
export function sessionFor(key, options) {
|
|
118
|
+
let session = sessions.get(key);
|
|
119
|
+
if (!session) {
|
|
120
|
+
session = new BrowserSession(options);
|
|
121
|
+
sessions.set(key, session);
|
|
122
|
+
}
|
|
123
|
+
return session;
|
|
124
|
+
}
|
|
125
|
+
/** Close every kept session; the host calls this when it shuts down. */
|
|
126
|
+
export async function closeBrowserSessions() {
|
|
127
|
+
const open = [...sessions.values()];
|
|
128
|
+
sessions.clear();
|
|
129
|
+
await Promise.all(open.map(session => session.close().catch(() => undefined)));
|
|
130
|
+
}
|
package/dist/catalog.js
CHANGED
|
@@ -25,6 +25,7 @@ export const catalog = [
|
|
|
25
25
|
backend('serpex', 'Serpex', 'https://serpex.dev/docs/api/search', [mode('key', 'https://api.serpex.dev/api/search', 'trial', '200 one-time credits; paid top-ups start around $0.80/1,000. Vendor routes internally; no fixed upstream engine.', '一次性 200 credits;充值约 $0.80/千次起;供应商内部路由,不能固定其上游引擎。', 'https://serpex.dev/docs/getting-started/pricing')]),
|
|
26
26
|
backend('x', 'X (Twitter)', 'https://docs.x.com/x-api/posts/search/quickstart/recent-search', [mode('session', 'https://x.com/i/api/graphql', 'account', 'Uses the browser session cookies (auth_token, ct0) of your own X account. No fee, but X changes its request signing and query ids regularly, so this mode can break until the adapter is updated.', '使用你自己 X 账号的网页 cookie(auth_token、ct0)。不产生费用,但 X 会不定期改动前端签名与 queryId,失效后需要维护者更新适配器。', 'https://github.com/bestnite/x-mcp', { en: '{"authToken":"...","ct0":"..."}, or paste auth_token=...; ct0=... straight from DevTools', zh: '{"authToken":"...","ct0":"..."},也可以直接粘贴浏览器里的 auth_token=...; ct0=...' }), mode('key', 'https://api.x.com/2/tweets/search/recent', 'paid', 'Official API with an app-only bearer token as the key. Pay per post read: about $0.005 each, so a 10-result search costs roughly $0.05. Recent search covers the last 7 days; reads are capped by the posts-per-search option.', '官方 API,密钥填 app-only bearer token。按读取帖数计费:约 $0.005/条,取 10 条约 $0.05。近期搜索只覆盖最近 7 天,读取条数受「每次搜索帖数」参数限制。', 'https://docs.x.com/x-api/getting-started/pricing', { en: '{"bearerToken":"<app-only bearer token>"}', zh: '{"bearerToken":"<app-only bearer 令牌>"}' })], [option('postsPerSearch', 'Posts read per search', '每次搜索读取帖数', 10)], ['recency']),
|
|
27
27
|
backend('reddit', 'Reddit', 'https://github.com/reddit-archive/reddit/wiki/API', [mode('anonymous', 'https://gql-fed.reddit.com/', 'free', 'No credentials. Uses the Reddit Android app endpoint, because the public REST search and comments endpoints answer 403 to anonymous tokens since 2026-10. No published quota; shared rate limits.', '无需凭证。走 Reddit 安卓客户端端点——自 2026-10 起公开 REST 的搜索与评论对匿名 token 一律返回 403。额度未公布,属共享限流。', 'https://github.com/bestnite/reddit-readonly-mcp'), mode('oauth', 'https://oauth.reddit.com/search', 'allowance', 'Official OAuth API: about 100 queries/minute per client for non-commercial use. client_credentials reads public data; add a username and password for a script grant.', '官方 OAuth API:非商业用途约每客户端 100 次/分钟。client_credentials 即可读公开数据;补用户名密码则用 script 授权。', 'https://support.reddithelp.com/hc/en-us/articles/16160319875092-Reddit-Data-API-Wiki', { en: '{"clientId":"...","clientSecret":"..."}; add "username" and "password" for a script grant', zh: '{"clientId":"...","clientSecret":"..."};需要脚本授权就再加 "username" 与 "password"' })], [option('sort', 'Sort', '排序', 'relevance', ['relevance', 'hot', 'top', 'new', 'comments']), option('time', 'Time window', '时间范围', 'all', ['all', 'hour', 'day', 'week', 'month', 'year']), option('subreddit', 'Limit to subreddit', '限定版块', '')], ['recency']),
|
|
28
|
+
backend('xiaohongshu', 'Xiaohongshu', 'https://git.nite07.com/nite/search-core/tree/main/image', [mode('browser', 'ws://127.0.0.1:3000/ws', 'self-hosted', 'Reads run through a browser you host, so nothing is billed per call and no public quota applies. The browser must be signed in to Xiaohongshu; this project ships an image that provides that browser, a VNC login surface and a token-protected endpoint.', '读取走你自己托管的浏览器,不按次计费、无公开额度。浏览器需要处于小红书登录状态;本项目提供一个镜像,内含浏览器、VNC 登录界面和带令牌的端点。', 'https://git.nite07.com/nite/search-core/tree/main/image', { en: 'Bearer token printed by the browser container at startup. Set it once in the shared browser settings; it protects the browser endpoint, not a site account', zh: '浏览器容器启动时打印的 Bearer 令牌。在共享浏览器设置中填写一次即可;它保护浏览器端点,不是网站账户凭证' })], [option('sort', 'Sort', '排序', 'general', ['general', 'time_descending', 'popularity_descending']), { key: 'statePath', type: 'string', label: text('Session state file', '会话状态文件'), help: text('Where the signed-in session is read from. Leave empty to use the container volume default (~/.config/dsh-search/xhs/state.json).', '登录态从哪个文件读取。留空则用容器卷的默认位置(~/.config/dsh-search/xhs/state.json)。'), default: '' }], [], 'documented', 'live'),
|
|
28
29
|
backend('tencent-wsa', 'Tencent Cloud WSA', 'https://cloud.tencent.com/document/product/1806/121811', [mode('key', 'https://wsa.tencentcloudapi.com/', 'paid', '¥14/¥20/¥26 per 1,000 by tier. Store a JSON credential {secretId,secretKey}; no free allowance verified.', '分档 ¥14/¥20/¥26 每千次。凭证填写 JSON {secretId,secretKey};免费额度未核实。', 'https://cloud.tencent.com/product/wsa', { en: '{"secretId":"...","secretKey":"..."}', zh: '{"secretId":"...","secretKey":"..."}' })], [{ ...option('Mode', 'Search mode', '搜索模式', 0), min: 0, max: 2 }]),
|
|
29
30
|
];
|
|
30
31
|
export const backendById = (id) => catalog.find(item => item.id === id);
|
package/dist/config.d.ts
CHANGED
|
@@ -1,4 +1,12 @@
|
|
|
1
1
|
import type { ConfigIssue, SearchConfig } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The credential name this library files a backend's key under. It is fixed here and not part of the
|
|
4
|
+
* configuration on purpose: a saved row must not be able to point a lookup at another name, and the
|
|
5
|
+
* settings page never has to show or ask for a name.
|
|
6
|
+
*/
|
|
7
|
+
export declare function backendCredentialRef(id: string): string;
|
|
8
|
+
/** The single endpoint token every browser-backed reader uses. Fixed for the same reason. */
|
|
9
|
+
export declare function browserCredentialRef(): string;
|
|
2
10
|
export declare function defaultConfig(): SearchConfig;
|
|
3
11
|
export declare function validateConfig(value: unknown): ConfigIssue[];
|
|
4
12
|
export declare function parseConfig(value: unknown): SearchConfig;
|
package/dist/config.js
CHANGED
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import { catalog } from './catalog.js';
|
|
2
2
|
import { LANGUAGES, REGIONS } from './locale-map.js';
|
|
3
|
+
/**
|
|
4
|
+
* The credential name this library files a backend's key under. It is fixed here and not part of the
|
|
5
|
+
* configuration on purpose: a saved row must not be able to point a lookup at another name, and the
|
|
6
|
+
* settings page never has to show or ask for a name.
|
|
7
|
+
*/
|
|
8
|
+
export function backendCredentialRef(id) {
|
|
9
|
+
return `DSH_SEARCH_${id.replaceAll('-', '_').toUpperCase()}_KEY`;
|
|
10
|
+
}
|
|
11
|
+
/** The single endpoint token every browser-backed reader uses. Fixed for the same reason. */
|
|
12
|
+
export function browserCredentialRef() {
|
|
13
|
+
return 'DSH_SEARCH_BROWSER_TOKEN';
|
|
14
|
+
}
|
|
3
15
|
export function defaultConfig() {
|
|
4
16
|
return {
|
|
5
17
|
mode: 'parallel', singleBackend: 'keenable', maxResults: 8, concurrency: 4, overallTimeoutMs: 30000,
|
|
6
18
|
language: '', region: '', recency: 'any', safeSearch: true, includeDomains: [], excludeDomains: [], dedup: 'url', fusion: 'rrf',
|
|
19
|
+
browser: { endpoint: '', statePath: '', timeoutMs: 60000 },
|
|
7
20
|
retry: { enabled: true, statuses: [408, 429, 500, 502, 503, 504], maxRetries: 2, initialDelayMs: 500, maxDelayMs: 5000, networkErrors: false, timeouts: false },
|
|
8
21
|
cache: { enabled: true, ttlSeconds: 900, maxEntries: 100 }, breaker: { enabled: false, threshold: 3, cooldownMs: 60000 }, diagnostics: { enabled: true, maxEntries: 50 }, tools: { webSearchPlus: true, compactByDefault: false },
|
|
9
|
-
backends: Object.fromEntries(catalog.map(d => [d.id, { enabled: false, mode: d.modes[0].mode, endpoint: d.modes[0].endpoint,
|
|
22
|
+
backends: Object.fromEntries(catalog.map(d => [d.id, { enabled: false, mode: d.modes[0].mode, endpoint: d.modes[0].endpoint, timeoutMs: 15000, rateLimitPerMinute: 0, dailyAttemptLimit: 0, options: Object.fromEntries(d.options.map(o => [o.key, o.default])) }]))
|
|
10
23
|
};
|
|
11
24
|
}
|
|
12
25
|
export function validateConfig(value) {
|
|
@@ -77,29 +90,40 @@ export function validateConfig(value) {
|
|
|
77
90
|
num(c.breaker.threshold, 1, 100, 'breaker.threshold');
|
|
78
91
|
num(c.breaker.cooldownMs, 1000, 3600000, 'breaker.cooldownMs');
|
|
79
92
|
num(c.diagnostics.maxEntries, 1, 500, 'diagnostics.maxEntries');
|
|
93
|
+
// An endpoint is an HTTP URL, except for browser backends, which point at a Playwright server (a
|
|
94
|
+
// WebSocket URL); the scheme family has to match the kind, and both accept TLS. The host itself is
|
|
95
|
+
// not gated: an operator may run a search server or a browser container behind any name, including
|
|
96
|
+
// the container-runtime aliases (host.containers.internal, host.docker.internal), so plaintext is
|
|
97
|
+
// allowed wherever the operator points it.
|
|
98
|
+
const checkEndpoint = (endpoint, path) => {
|
|
99
|
+
try {
|
|
100
|
+
const url = new URL(endpoint);
|
|
101
|
+
if (url.username || url.password || url.hash || url.search)
|
|
102
|
+
throw 0;
|
|
103
|
+
const schemes = url.protocol === 'ws:' || url.protocol === 'wss:' ? ['ws:', 'wss:'] : ['http:', 'https:'];
|
|
104
|
+
if (!schemes.includes(url.protocol))
|
|
105
|
+
throw 0;
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
fail(path, 'Use an HTTP(S) or WS(S) endpoint without credentials, query, or fragment.');
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
const sharedBrowser = c.browser.endpoint;
|
|
112
|
+
if (sharedBrowser)
|
|
113
|
+
checkEndpoint(sharedBrowser, 'browser.endpoint');
|
|
114
|
+
if (!(c.browser.timeoutMs >= 1000 && c.browser.timeoutMs <= 120000))
|
|
115
|
+
fail('browser.timeoutMs', 'Expected 1000 to 120000 milliseconds.');
|
|
80
116
|
for (const d of catalog) {
|
|
81
117
|
const b = c.backends[d.id];
|
|
82
118
|
const p = `backends.${d.id}`;
|
|
83
119
|
choice(b.mode, d.modes.map(m => m.mode), `${p}.mode`);
|
|
84
120
|
if (b.enabled && d.status === 'unverified')
|
|
85
121
|
fail(`${p}.enabled`, 'Backend protocol is not verified.');
|
|
86
|
-
if (b.endpoint)
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
throw 0;
|
|
91
|
-
const local = url.hostname === 'localhost' || url.hostname === '127.0.0.1' || url.hostname === '[::1]';
|
|
92
|
-
if (url.protocol === 'http:' && !local)
|
|
93
|
-
throw 0;
|
|
94
|
-
}
|
|
95
|
-
catch {
|
|
96
|
-
fail(`${p}.endpoint`, 'Use an HTTPS endpoint without credentials, query, or fragment; HTTP is allowed only on loopback.');
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
else if (b.enabled)
|
|
122
|
+
if (b.endpoint)
|
|
123
|
+
checkEndpoint(b.endpoint, `${p}.endpoint`);
|
|
124
|
+
// A browser backend may inherit the shared endpoint instead of naming its own.
|
|
125
|
+
else if (b.enabled && !(b.mode === 'browser' && sharedBrowser))
|
|
100
126
|
fail(`${p}.endpoint`, 'An endpoint is required.');
|
|
101
|
-
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(b.credentialRef))
|
|
102
|
-
fail(`${p}.credentialRef`, 'Expected an environment-style credential reference.');
|
|
103
127
|
num(b.timeoutMs, 1000, 120000, `${p}.timeoutMs`);
|
|
104
128
|
num(b.rateLimitPerMinute, 0, 100000, `${p}.rateLimitPerMinute`);
|
|
105
129
|
num(b.dailyAttemptLimit, 0, 1000000, `${p}.dailyAttemptLimit`);
|
|
@@ -124,6 +148,15 @@ export function parseConfig(value) {
|
|
|
124
148
|
for (const [id, fallback] of Object.entries(defaults.backends))
|
|
125
149
|
if (!raw.backends[id])
|
|
126
150
|
raw.backends[id] = fallback;
|
|
151
|
+
// A credential reference used to be a configuration field. The name is fixed in code now, so a
|
|
152
|
+
// saved reference is dropped instead of being honoured: nothing in a file may redirect a lookup.
|
|
153
|
+
if (raw.browser && typeof raw.browser === 'object')
|
|
154
|
+
delete raw.browser.credentialRef;
|
|
155
|
+
for (const id of Object.keys(raw.backends)) {
|
|
156
|
+
const row = raw.backends[id];
|
|
157
|
+
if (row && typeof row === 'object')
|
|
158
|
+
delete row.credentialRef;
|
|
159
|
+
}
|
|
127
160
|
value = raw;
|
|
128
161
|
}
|
|
129
162
|
const issues = validateConfig(value);
|
package/dist/engine.js
CHANGED
|
@@ -2,7 +2,27 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { catalog } from './catalog.js';
|
|
3
3
|
import { runAdapter } from './adapters.js';
|
|
4
4
|
import { abortError, SearchError } from './transport.js';
|
|
5
|
+
import { backendCredentialRef, browserCredentialRef } from './config.js';
|
|
5
6
|
export function revision(config) { return createHash('sha256').update(JSON.stringify(config)).digest('hex'); }
|
|
7
|
+
/**
|
|
8
|
+
* A result crosses a JSON boundary before any host or tool sees it, and an explicit undefined does
|
|
9
|
+
* not survive that crossing. Adapters are meant to omit absent fields, but one gap there breaks a
|
|
10
|
+
* whole tool call, so the boundary enforces it once for every caller.
|
|
11
|
+
*/
|
|
12
|
+
function lossless(value) {
|
|
13
|
+
if (Array.isArray(value))
|
|
14
|
+
return value.map(item => lossless(item));
|
|
15
|
+
if (value && typeof value === 'object') {
|
|
16
|
+
const out = {};
|
|
17
|
+
for (const [key, item] of Object.entries(value)) {
|
|
18
|
+
if (item === undefined)
|
|
19
|
+
continue;
|
|
20
|
+
out[key] = lossless(item);
|
|
21
|
+
}
|
|
22
|
+
return out;
|
|
23
|
+
}
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
6
26
|
export class SearchEngine {
|
|
7
27
|
config;
|
|
8
28
|
resolveKey;
|
|
@@ -142,7 +162,7 @@ export class SearchEngine {
|
|
|
142
162
|
while (this.cache.size > c.cache.maxEntries)
|
|
143
163
|
this.cache.delete(this.cache.keys().next().value);
|
|
144
164
|
}
|
|
145
|
-
return result;
|
|
165
|
+
return lossless(result);
|
|
146
166
|
}
|
|
147
167
|
async call(id, query, maxResults, signal) {
|
|
148
168
|
const started = Date.now();
|
|
@@ -163,8 +183,14 @@ export class SearchEngine {
|
|
|
163
183
|
throw new SearchError('circuit-open', 'Backend circuit is open; no call or fallback was sent.');
|
|
164
184
|
// Every mode except anonymous needs a credential, including session and OAuth modes whose
|
|
165
185
|
// credential is a JSON document rather than a bare key.
|
|
166
|
-
|
|
167
|
-
|
|
186
|
+
// A browser backend authenticates against the user's own browser; its credential is an
|
|
187
|
+
// optional token, so a missing one must not block the call.
|
|
188
|
+
const needsCredential = b.mode !== 'anonymous' && b.mode !== 'browser';
|
|
189
|
+
// The endpoint token guards the browser, not one backend, so a browser read asks for the
|
|
190
|
+
// browser's single token; every other mode asks for the name this library files its key under.
|
|
191
|
+
const key = b.mode === 'browser'
|
|
192
|
+
? await this.resolveKey(browserCredentialRef()).catch(() => undefined)
|
|
193
|
+
: (needsCredential ? await this.resolveKey(backendCredentialRef(id)) : undefined);
|
|
168
194
|
if (needsCredential && !key)
|
|
169
195
|
throw new SearchError('configuration', b.mode === 'key' ? 'API-key mode selected but credential is missing. Ask the user to set the credential.' : `The ${b.mode} mode selected for this backend needs a stored credential. Ask the user to set it.`);
|
|
170
196
|
for (const [field, active] of [['language', !!this.config.language], ['region', !!this.config.region], ['recency', this.config.recency !== 'any'], ['safeSearch', this.config.safeSearch]])
|
package/dist/index.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Nothing here depends on DeepSeek Harness, so any host can drive it.
|
|
5
5
|
*/
|
|
6
6
|
export { catalog } from './catalog.js';
|
|
7
|
-
export { defaultConfig, parseConfig, validateConfig } from './config.js';
|
|
7
|
+
export { backendCredentialRef, browserCredentialRef, defaultConfig, parseConfig, validateConfig } from './config.js';
|
|
8
8
|
export { SearchEngine } from './engine.js';
|
|
9
9
|
export { revision } from './engine.js';
|
|
10
10
|
export type { ConfigIssue, SearchOverrides, SearchResult, BackendOutcome } from './types.js';
|
|
@@ -19,5 +19,9 @@ export type { RedditComment, RedditCredentials, RedditThread } from './reddit/in
|
|
|
19
19
|
export { parseXCredentials, searchX } from './x/index.js';
|
|
20
20
|
export type { XCredentials } from './x/index.js';
|
|
21
21
|
export { fetchXThread, normaliseId } from './x/thread.js';
|
|
22
|
+
export { fetchXhsComments, fetchXhsNote, readXhsSearchNotes, resolveTarget as resolveXhsTarget, searchXhs } from './xhs/index.js';
|
|
23
|
+
export type { XhsComment, XhsContext, XhsNote, XhsNoteSummary } from './xhs/index.js';
|
|
24
|
+
export { materialiseImages, mediaDir, type MediaImage, type MediaOptions } from './media.js';
|
|
25
|
+
export { BrowserSession, closeBrowserSessions, connectFailure, sessionFor, type BrowserSessionOptions } from './browser.js';
|
|
22
26
|
export type { XThread } from './x/thread.js';
|
|
23
27
|
export type { XReply, XThreadPost } from './x/parse.js';
|
package/dist/index.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Nothing here depends on DeepSeek Harness, so any host can drive it.
|
|
5
5
|
*/
|
|
6
6
|
export { catalog } from './catalog.js';
|
|
7
|
-
export { defaultConfig, parseConfig, validateConfig } from './config.js';
|
|
7
|
+
export { backendCredentialRef, browserCredentialRef, defaultConfig, parseConfig, validateConfig } from './config.js';
|
|
8
8
|
export { SearchEngine } from './engine.js';
|
|
9
9
|
export { revision } from './engine.js';
|
|
10
10
|
export { SearchError } from './transport.js';
|
|
@@ -15,3 +15,6 @@ export { runAdapter } from './adapters.js';
|
|
|
15
15
|
export { fetchRedditThread, searchReddit } from './reddit/index.js';
|
|
16
16
|
export { parseXCredentials, searchX } from './x/index.js';
|
|
17
17
|
export { fetchXThread, normaliseId } from './x/thread.js';
|
|
18
|
+
export { fetchXhsComments, fetchXhsNote, readXhsSearchNotes, resolveTarget as resolveXhsTarget, searchXhs } from './xhs/index.js';
|
|
19
|
+
export { materialiseImages, mediaDir } from './media.js';
|
|
20
|
+
export { BrowserSession, closeBrowserSessions, connectFailure, sessionFor } from './browser.js';
|
package/dist/media.d.ts
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Images are part of what a post says, but a text answer cannot show one. The read tools therefore
|
|
3
|
+
* download the pictures a post carries into a scratch directory and hand back the paths, so the
|
|
4
|
+
* host's image reader can open them.
|
|
5
|
+
*
|
|
6
|
+
* Every link a caller passes is reported and every one that fits is downloaded, because which picture
|
|
7
|
+
* matters is the reader's judgement, not this module's. What is bounded is cost: a size ceiling per
|
|
8
|
+
* file, a byte budget per call, and files older than an hour are swept before the next write, since
|
|
9
|
+
* nothing else knows when they stop being needed.
|
|
10
|
+
*/
|
|
11
|
+
export interface MediaImage {
|
|
12
|
+
url: string;
|
|
13
|
+
/** Where the downloaded file is, when the download worked. */
|
|
14
|
+
path?: string;
|
|
15
|
+
contentType?: string;
|
|
16
|
+
bytes?: number;
|
|
17
|
+
/** Why this image is not on disk, when it is not. */
|
|
18
|
+
error?: string;
|
|
19
|
+
}
|
|
20
|
+
export interface MediaOptions {
|
|
21
|
+
/** Directory to write into. Defaults to a scratch directory under the system temp dir. */
|
|
22
|
+
dir?: string;
|
|
23
|
+
/** How many images to download. Defaults to every link, capped at 24. */
|
|
24
|
+
limit?: number;
|
|
25
|
+
/** Largest file to keep. Defaults to 8 MiB. */
|
|
26
|
+
maxBytes?: number;
|
|
27
|
+
maxAgeMs?: number;
|
|
28
|
+
/** Stop downloading once this many bytes have arrived. Defaults to 48 MiB. */
|
|
29
|
+
totalBytes?: number;
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
fetch?: typeof fetch;
|
|
32
|
+
}
|
|
33
|
+
/** The scratch directory these files live in, so a caller can point a reader at it. */
|
|
34
|
+
export declare function mediaDir(): string;
|
|
35
|
+
/**
|
|
36
|
+
* Download images to disk, one entry per input, in input order. A failed download is reported in
|
|
37
|
+
* place rather than dropped, because a caller that asked for pictures should learn why it got none.
|
|
38
|
+
*/
|
|
39
|
+
export declare function materialiseImages(urls: readonly string[], options?: MediaOptions): Promise<MediaImage[]>;
|