@fffangchenghao/dsh-web-search 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +194 -33
- package/README.zh-CN.md +171 -34
- package/SECURITY.md +42 -7
- package/client.js +131 -9
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +16 -0
- package/lib/bridge.js +160 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +118 -19
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +304 -20
- package/lib/inject.d.ts +43 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `platform_search` tool.
|
|
3
|
+
*
|
|
4
|
+
* It is the community/video counterpart to `domain_search`: one call can ask
|
|
5
|
+
* Bilibili, V2EX, Reddit, YouTube, Zhihu and Juejin for pages about a topic.
|
|
6
|
+
* The transport and parser live in `platforms.ts`; this module owns only the
|
|
7
|
+
* tool contract (arguments, validation, rendering, fallback name and
|
|
8
|
+
* registration). That keeps the network adapters testable without the host
|
|
9
|
+
* registry and keeps the tool definition in the same raw JSON-Schema dialect
|
|
10
|
+
* as the other optional tools.
|
|
11
|
+
*
|
|
12
|
+
* @module @fangchenghao/dsh-web-search/platform-tool
|
|
13
|
+
*/
|
|
14
|
+
import type { Source } from './types.ts';
|
|
15
|
+
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
16
|
+
import { MAX_PLATFORM_LIMIT } from './platforms.ts';
|
|
17
|
+
import type { PlatformId } from './platforms.ts';
|
|
18
|
+
/** The name this tool prefers. */
|
|
19
|
+
export declare const PLATFORM_TOOL_NAME = "platform_search";
|
|
20
|
+
/** The name used when another plugin already registered `platform_search`. */
|
|
21
|
+
export declare const PLATFORM_TOOL_FALLBACK_NAME = "web_platform_search";
|
|
22
|
+
/** Default and maximum result count per call (merged across platforms). */
|
|
23
|
+
export declare const PLATFORM_DEFAULT_LIMIT = 5;
|
|
24
|
+
export declare const PLATFORM_MAX_LIMIT = 10;
|
|
25
|
+
/** What one platform call returns to the model. */
|
|
26
|
+
export interface PlatformToolValue {
|
|
27
|
+
query: string;
|
|
28
|
+
platforms: PlatformId[];
|
|
29
|
+
sources: Source[];
|
|
30
|
+
failures: string[];
|
|
31
|
+
via: Partial<Record<PlatformId, 'api' | 'fallback'>>;
|
|
32
|
+
truncated: boolean;
|
|
33
|
+
}
|
|
34
|
+
/** The network half, injectable so the tool definition can be tested offline. */
|
|
35
|
+
export type PlatformRunner = (query: string, ids: readonly PlatformId[], limit: number, signal?: AbortSignal) => Promise<Omit<PlatformToolValue, 'query' | 'platforms'>>;
|
|
36
|
+
/** A registry definition in the host's raw JSON-Schema dialect. */
|
|
37
|
+
export interface PlatformToolDefinition {
|
|
38
|
+
name: string;
|
|
39
|
+
description: string;
|
|
40
|
+
parameters: ToolJsonSchema;
|
|
41
|
+
output: {
|
|
42
|
+
schema: ToolJsonSchema;
|
|
43
|
+
render: (args: unknown, value: PlatformToolValue) => ToolContent[];
|
|
44
|
+
};
|
|
45
|
+
timeoutMs?: number;
|
|
46
|
+
isConcurrencySafe?: () => boolean;
|
|
47
|
+
execute: (args: unknown, exec: {
|
|
48
|
+
signal?: AbortSignal;
|
|
49
|
+
}) => Promise<PlatformToolValue>;
|
|
50
|
+
}
|
|
51
|
+
/** The slice of the tool registry this module needs. */
|
|
52
|
+
export interface PlatformToolRegistryLike {
|
|
53
|
+
register(definition: PlatformToolDefinition): unknown;
|
|
54
|
+
get?(name: string, scope?: unknown): unknown;
|
|
55
|
+
}
|
|
56
|
+
/** Parse and validate arguments. */
|
|
57
|
+
export declare function parsePlatformArgs(args: unknown): {
|
|
58
|
+
query: string;
|
|
59
|
+
ids: PlatformId[];
|
|
60
|
+
limit: number;
|
|
61
|
+
};
|
|
62
|
+
/** Format the model-facing result, inside the untrusted boundary. */
|
|
63
|
+
export declare function formatPlatformValue(value: PlatformToolValue): string;
|
|
64
|
+
/**
|
|
65
|
+
* Build the tool definition.
|
|
66
|
+
* @param name - preferred or fallback name.
|
|
67
|
+
* @param run - network runner, normally `searchPlatforms` with the live proxy
|
|
68
|
+
* and the plugin's normal search as the fallback.
|
|
69
|
+
*/
|
|
70
|
+
export declare function createPlatformToolDefinition(name: string, run: PlatformRunner): PlatformToolDefinition;
|
|
71
|
+
/** Pick the preferred name unless another tool already owns it. */
|
|
72
|
+
export declare function pickPlatformToolName(tools: PlatformToolRegistryLike): string;
|
|
73
|
+
/** Register the tool; any registry failure is contained. */
|
|
74
|
+
export declare function registerPlatformTool(tools: PlatformToolRegistryLike, run: PlatformRunner, options?: {
|
|
75
|
+
warn?: (message: string) => void;
|
|
76
|
+
}): string | undefined;
|
|
77
|
+
export { MAX_PLATFORM_LIMIT };
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { wrapUntrustedContent } from "./untrusted.js";
|
|
2
|
+
import { MAX_PLATFORM_LIMIT, PLATFORM_IDS, PLATFORM_LABELS, isPlatformId } from "./platforms.js";
|
|
3
|
+
/** The name this tool prefers. */
|
|
4
|
+
export const PLATFORM_TOOL_NAME = 'platform_search';
|
|
5
|
+
/** The name used when another plugin already registered `platform_search`. */
|
|
6
|
+
export const PLATFORM_TOOL_FALLBACK_NAME = 'web_platform_search';
|
|
7
|
+
/** Default and maximum result count per call (merged across platforms). */
|
|
8
|
+
export const PLATFORM_DEFAULT_LIMIT = 5;
|
|
9
|
+
export const PLATFORM_MAX_LIMIT = 10;
|
|
10
|
+
/** True for a value usable as a record. */
|
|
11
|
+
function isRecord(value) {
|
|
12
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
13
|
+
}
|
|
14
|
+
/** Parameter schema. */
|
|
15
|
+
function parameterSchema() {
|
|
16
|
+
return {
|
|
17
|
+
type: 'object',
|
|
18
|
+
additionalProperties: false,
|
|
19
|
+
properties: {
|
|
20
|
+
query: {
|
|
21
|
+
type: 'string',
|
|
22
|
+
description: 'What to look for on the selected community/video platforms; distinctive words work best.',
|
|
23
|
+
},
|
|
24
|
+
platforms: {
|
|
25
|
+
type: 'array',
|
|
26
|
+
items: { type: 'string', enum: [...PLATFORM_IDS] },
|
|
27
|
+
description: `Platforms to search: ${PLATFORM_IDS.join(', ')}. Omit to search all six.`,
|
|
28
|
+
},
|
|
29
|
+
limit: {
|
|
30
|
+
type: 'integer',
|
|
31
|
+
description: `Merged results wanted, 1–${PLATFORM_MAX_LIMIT}; defaults to ${PLATFORM_DEFAULT_LIMIT}.`,
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
required: ['query'],
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/** Output schema. */
|
|
38
|
+
function outputSchema() {
|
|
39
|
+
return {
|
|
40
|
+
type: 'object',
|
|
41
|
+
additionalProperties: false,
|
|
42
|
+
properties: {
|
|
43
|
+
query: { type: 'string' },
|
|
44
|
+
platforms: { type: 'array', items: { type: 'string', enum: [...PLATFORM_IDS] } },
|
|
45
|
+
sources: {
|
|
46
|
+
type: 'array',
|
|
47
|
+
items: {
|
|
48
|
+
type: 'object',
|
|
49
|
+
additionalProperties: false,
|
|
50
|
+
properties: {
|
|
51
|
+
url: { type: 'string' },
|
|
52
|
+
title: { type: 'string' },
|
|
53
|
+
snippet: { type: 'string' },
|
|
54
|
+
publishedAt: { type: 'string' },
|
|
55
|
+
},
|
|
56
|
+
required: ['url'],
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
60
|
+
via: {
|
|
61
|
+
type: 'object',
|
|
62
|
+
additionalProperties: false,
|
|
63
|
+
properties: {
|
|
64
|
+
bilibili: { type: 'string', enum: ['api', 'fallback'] },
|
|
65
|
+
v2ex: { type: 'string', enum: ['api', 'fallback'] },
|
|
66
|
+
reddit: { type: 'string', enum: ['api', 'fallback'] },
|
|
67
|
+
youtube: { type: 'string', enum: ['api', 'fallback'] },
|
|
68
|
+
zhihu: { type: 'string', enum: ['api', 'fallback'] },
|
|
69
|
+
juejin: { type: 'string', enum: ['api', 'fallback'] },
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
truncated: { type: 'boolean' },
|
|
73
|
+
},
|
|
74
|
+
required: ['query', 'platforms', 'sources', 'failures', 'via', 'truncated'],
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/** Parse and validate arguments. */
|
|
78
|
+
export function parsePlatformArgs(args) {
|
|
79
|
+
if (!isRecord(args) || typeof args['query'] !== 'string' || args['query'].trim().length === 0) {
|
|
80
|
+
throw new Error('platform_search requires a non-empty query');
|
|
81
|
+
}
|
|
82
|
+
const rawIds = args['platforms'];
|
|
83
|
+
let ids = [...PLATFORM_IDS];
|
|
84
|
+
if (rawIds !== undefined) {
|
|
85
|
+
if (!Array.isArray(rawIds) || rawIds.some((item) => !isPlatformId(item))) {
|
|
86
|
+
throw new Error(`platform_search platforms must be a subset of ${PLATFORM_IDS.join(', ')}`);
|
|
87
|
+
}
|
|
88
|
+
if (rawIds.length === 0)
|
|
89
|
+
throw new Error('platform_search platforms must not be empty');
|
|
90
|
+
ids = PLATFORM_IDS.filter((id) => rawIds.includes(id));
|
|
91
|
+
}
|
|
92
|
+
const rawLimit = args['limit'];
|
|
93
|
+
if (rawLimit !== undefined && (typeof rawLimit !== 'number' || !Number.isInteger(rawLimit) || rawLimit < 1 || rawLimit > PLATFORM_MAX_LIMIT)) {
|
|
94
|
+
throw new Error(`platform_search limit must be an integer between 1 and ${PLATFORM_MAX_LIMIT}`);
|
|
95
|
+
}
|
|
96
|
+
return { query: args['query'].trim(), ids, limit: rawLimit ?? PLATFORM_DEFAULT_LIMIT };
|
|
97
|
+
}
|
|
98
|
+
/** Format the model-facing result, inside the untrusted boundary. */
|
|
99
|
+
export function formatPlatformValue(value) {
|
|
100
|
+
const platforms = value.platforms.map((id) => PLATFORM_LABELS[id]).join(', ');
|
|
101
|
+
const head = `Platform search: ${JSON.stringify(value.query)} — ${value.sources.length} result(s) from ${platforms}${value.truncated ? ' (truncated)' : ''}.`;
|
|
102
|
+
const lines = value.sources.map((source, index) => {
|
|
103
|
+
const linesForSource = [`${index + 1}. ${source.title ?? source.url}`, ` ${source.url}`];
|
|
104
|
+
if (source.snippet !== undefined && source.snippet.length > 0)
|
|
105
|
+
linesForSource.push(` ${source.snippet}`);
|
|
106
|
+
if (source.publishedAt !== undefined && source.publishedAt.length > 0)
|
|
107
|
+
linesForSource.push(` ${source.publishedAt}`);
|
|
108
|
+
return linesForSource.join('\n');
|
|
109
|
+
});
|
|
110
|
+
if (value.failures.length > 0)
|
|
111
|
+
lines.push('', 'Unavailable platforms:', ...value.failures.map((failure) => `- ${failure}`));
|
|
112
|
+
return [head, '', ...lines].join('\n').trimEnd();
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Build the tool definition.
|
|
116
|
+
* @param name - preferred or fallback name.
|
|
117
|
+
* @param run - network runner, normally `searchPlatforms` with the live proxy
|
|
118
|
+
* and the plugin's normal search as the fallback.
|
|
119
|
+
*/
|
|
120
|
+
export function createPlatformToolDefinition(name, run) {
|
|
121
|
+
return {
|
|
122
|
+
name,
|
|
123
|
+
description: 'Search community and video platforms directly: Bilibili, V2EX, Reddit, YouTube, Zhihu and Juejin. Use it when the user asks what a community, forum or video site says; use web_search for general pages. Results include title, URL, snippet and date when the platform exposes one.',
|
|
124
|
+
parameters: parameterSchema(),
|
|
125
|
+
output: {
|
|
126
|
+
schema: outputSchema(),
|
|
127
|
+
render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatPlatformValue(value)) }],
|
|
128
|
+
},
|
|
129
|
+
timeoutMs: 40_000,
|
|
130
|
+
isConcurrencySafe: () => true,
|
|
131
|
+
async execute(args, exec) {
|
|
132
|
+
const parsed = parsePlatformArgs(args);
|
|
133
|
+
const result = await run(parsed.query, parsed.ids, parsed.limit, exec.signal);
|
|
134
|
+
return {
|
|
135
|
+
query: parsed.query,
|
|
136
|
+
platforms: [...parsed.ids],
|
|
137
|
+
sources: result.sources.slice(0, parsed.limit).map((source) => ({ ...source })),
|
|
138
|
+
failures: [...result.failures],
|
|
139
|
+
via: { ...result.via },
|
|
140
|
+
truncated: result.truncated || result.sources.length > parsed.limit,
|
|
141
|
+
};
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
/** Pick the preferred name unless another tool already owns it. */
|
|
146
|
+
export function pickPlatformToolName(tools) {
|
|
147
|
+
if (typeof tools.get !== 'function')
|
|
148
|
+
return PLATFORM_TOOL_NAME;
|
|
149
|
+
try {
|
|
150
|
+
return tools.get(PLATFORM_TOOL_NAME) === undefined ? PLATFORM_TOOL_NAME : PLATFORM_TOOL_FALLBACK_NAME;
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return PLATFORM_TOOL_NAME;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Register the tool; any registry failure is contained. */
|
|
157
|
+
export function registerPlatformTool(tools, run, options = {}) {
|
|
158
|
+
const name = pickPlatformToolName(tools);
|
|
159
|
+
try {
|
|
160
|
+
tools.register(createPlatformToolDefinition(name, run));
|
|
161
|
+
return name;
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
try {
|
|
165
|
+
options.warn?.(`platform_search registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// Diagnostics must not break mounting.
|
|
169
|
+
}
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
export { MAX_PLATFORM_LIMIT };
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keyless search across six community platforms: Bilibili, V2EX, Reddit,
|
|
3
|
+
* YouTube, Zhihu and Juejin.
|
|
4
|
+
*
|
|
5
|
+
* Each platform answers from a public endpoint first. Those endpoints are
|
|
6
|
+
* borrowed from result pages, so they are allowed to break: a JSON API may
|
|
7
|
+
* answer with an anti-bot HTML page, a page may stop embedding
|
|
8
|
+
* `ytInitialData`, V2EX's hot list is a snapshot rather than a search. When
|
|
9
|
+
* the direct attempt fails or yields nothing, the platform is retried through
|
|
10
|
+
* an injected {@link PlatformSearchOptions.fallbackSearch} (normally a
|
|
11
|
+
* `site:<domain>` web search). Zhihu has no usable keyless endpoint at all and
|
|
12
|
+
* goes straight to that fallback.
|
|
13
|
+
*
|
|
14
|
+
* One platform failing never fails the call: the reason is recorded in
|
|
15
|
+
* `failures`, the other platforms still answer, and the rows are interleaved
|
|
16
|
+
* by rank so a single prolific platform cannot crowd the rest out.
|
|
17
|
+
*
|
|
18
|
+
* @module @fangchenghao/dsh-web-search/platforms
|
|
19
|
+
*/
|
|
20
|
+
import type { Source } from './types.ts';
|
|
21
|
+
/** The platforms this module can search without a credential. */
|
|
22
|
+
export type PlatformId = 'bilibili' | 'v2ex' | 'reddit' | 'youtube' | 'zhihu' | 'juejin';
|
|
23
|
+
/** Every supported platform, in the order results are interleaved. */
|
|
24
|
+
export declare const PLATFORM_IDS: readonly PlatformId[];
|
|
25
|
+
/** Human labels, used in notes and in the model-facing tool description. */
|
|
26
|
+
export declare const PLATFORM_LABELS: Readonly<Record<PlatformId, string>>;
|
|
27
|
+
/** Results wanted when the caller names no limit. */
|
|
28
|
+
export declare const DEFAULT_PLATFORM_LIMIT = 5;
|
|
29
|
+
/** Upper bound a single call may request. */
|
|
30
|
+
export declare const MAX_PLATFORM_LIMIT = 10;
|
|
31
|
+
/** True for a value usable as a platform id. */
|
|
32
|
+
export declare function isPlatformId(value: unknown): value is PlatformId;
|
|
33
|
+
/** Options for {@link searchPlatforms}. */
|
|
34
|
+
export interface PlatformSearchOptions {
|
|
35
|
+
/** Platform ids to query; empty or absent means every id in {@link PLATFORM_IDS}. */
|
|
36
|
+
ids?: readonly string[];
|
|
37
|
+
/** Results wanted per platform, clamped to 1…10 and capped again after the merge. */
|
|
38
|
+
limit?: number;
|
|
39
|
+
/** Per-platform deadline in milliseconds. */
|
|
40
|
+
timeoutMs?: number;
|
|
41
|
+
/** Caller cancellation, propagated to every request and fallback. */
|
|
42
|
+
signal?: AbortSignal;
|
|
43
|
+
/** Tunnel URL; when set, direct requests go through {@link proxyGet}. */
|
|
44
|
+
proxy?: string;
|
|
45
|
+
/** Test seam; defaults to the platform `fetch`. */
|
|
46
|
+
fetch?: typeof fetch;
|
|
47
|
+
/**
|
|
48
|
+
* Web search used when a platform's direct endpoint fails, is blocked or
|
|
49
|
+
* returns nothing. It receives a `site:<domain> <query>` string, the same
|
|
50
|
+
* budget and the caller's signal.
|
|
51
|
+
*/
|
|
52
|
+
fallbackSearch?: (query: string, limit: number, signal?: AbortSignal) => Promise<{
|
|
53
|
+
sources: Source[];
|
|
54
|
+
failures: string[];
|
|
55
|
+
}>;
|
|
56
|
+
}
|
|
57
|
+
/** What one platform run reports. */
|
|
58
|
+
export interface PlatformOutcome {
|
|
59
|
+
/** Merged, de-duplicated sources, capped at the caller's budget. */
|
|
60
|
+
sources: Source[];
|
|
61
|
+
/** One `id: reason` line per platform that could not answer. */
|
|
62
|
+
failures: string[];
|
|
63
|
+
/** Parsed row count per platform, before the global cap. */
|
|
64
|
+
counts: Record<string, number>;
|
|
65
|
+
/** Whether rows were dropped because of the global cap. */
|
|
66
|
+
truncated: boolean;
|
|
67
|
+
/**
|
|
68
|
+
* How each platform was served: `api` for its own endpoint, `fallback` when
|
|
69
|
+
* the web-search fallback answered (or was the last route tried).
|
|
70
|
+
*/
|
|
71
|
+
via: Record<string, 'api' | 'fallback'>;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Parse Bilibili's video search envelope.
|
|
75
|
+
*
|
|
76
|
+
* Row-level drift yields fewer sources; an anti-bot envelope (HTML or a
|
|
77
|
+
* non-zero `code`) throws so the caller knows to fall back.
|
|
78
|
+
*
|
|
79
|
+
* @param payload - `api.bilibili.com/x/web-interface/search/type` JSON.
|
|
80
|
+
* @param limit - rows to keep.
|
|
81
|
+
* @returns video sources in API order.
|
|
82
|
+
* @throws `Error` on a wall or an unparsable envelope.
|
|
83
|
+
*/
|
|
84
|
+
export declare function parseBilibili(payload: string, limit?: number, query?: string): Source[];
|
|
85
|
+
/**
|
|
86
|
+
* Parse a YouTube results page through the embedded `ytInitialData` graph.
|
|
87
|
+
*
|
|
88
|
+
* `videoRenderer` is looked up at any depth because YouTube reshuffles the
|
|
89
|
+
* section wrappers between experiments; the marker and the JSON are the only
|
|
90
|
+
* two things the parser insists on.
|
|
91
|
+
*
|
|
92
|
+
* @param payload - `/results?search_query=` HTML.
|
|
93
|
+
* @param limit - rows to keep.
|
|
94
|
+
* @returns video sources in page order.
|
|
95
|
+
* @throws `Error` when the page embeds no parsable `ytInitialData`.
|
|
96
|
+
*/
|
|
97
|
+
export declare function parseYoutube(payload: string, limit?: number): Source[];
|
|
98
|
+
/**
|
|
99
|
+
* Parse a Reddit search listing.
|
|
100
|
+
*
|
|
101
|
+
* The permalink is preferred over the external target so a post's citation
|
|
102
|
+
* points at the discussion; `selftext` becomes the snippet, with a
|
|
103
|
+
* subreddit/score line used when the post is link-only.
|
|
104
|
+
*
|
|
105
|
+
* @param payload - `reddit.com/search.json` JSON.
|
|
106
|
+
* @param limit - rows to keep.
|
|
107
|
+
* @returns post sources in listing order.
|
|
108
|
+
* @throws `Error` on a wall, invalid JSON or an unexpected envelope.
|
|
109
|
+
*/
|
|
110
|
+
export declare function parseReddit(payload: string, limit?: number): Source[];
|
|
111
|
+
/**
|
|
112
|
+
* Parse Juejin's article search envelope.
|
|
113
|
+
*
|
|
114
|
+
* The documented shape is `data[].result_model.article_info`; the collector
|
|
115
|
+
* also accepts a bare `article_info` or an unwrapped `data` array because the
|
|
116
|
+
* search API has changed wrappers before.
|
|
117
|
+
*
|
|
118
|
+
* @param payload - `api.juejin.cn/search_api/v1/search` JSON.
|
|
119
|
+
* @param limit - rows to keep.
|
|
120
|
+
* @returns article sources in API order.
|
|
121
|
+
* @throws `Error` on a wall, invalid JSON or an API error code.
|
|
122
|
+
*/
|
|
123
|
+
export declare function parseJuejin(payload: string, limit?: number): Source[];
|
|
124
|
+
/**
|
|
125
|
+
* Sort Juejin rows by likes only when the query asks for popular posts. The
|
|
126
|
+
* API returns a relevance order; an explicit "高赞/热门" request should not
|
|
127
|
+
* leave a 16-like post above a 217-like one. Rows without a like count keep
|
|
128
|
+
* their relative order after measured rows.
|
|
129
|
+
*
|
|
130
|
+
* @param sources - rows from {@link parseJuejin}, or a fallback.
|
|
131
|
+
* @param query - the caller's query.
|
|
132
|
+
* @returns the original order when no popularity intent is present.
|
|
133
|
+
*/
|
|
134
|
+
export declare function normaliseJuejinSources(sources: readonly Source[], query: string): Source[];
|
|
135
|
+
/**
|
|
136
|
+
* Filter V2EX's hot-topic snapshot by the query.
|
|
137
|
+
*
|
|
138
|
+
* The public API has no search endpoint, so the hot list is the only keyless
|
|
139
|
+
* source; a query that does not appear in any title or body filters everything
|
|
140
|
+
* out and the caller is expected to fall back to `site:v2ex.com`.
|
|
141
|
+
*
|
|
142
|
+
* @param payload - `v2ex.com/api/topics/hot.json` JSON.
|
|
143
|
+
* @param query - text to look for in a title or body, case-insensitively.
|
|
144
|
+
* @param limit - rows to keep.
|
|
145
|
+
* @returns matching topics in hot-list order.
|
|
146
|
+
* @throws `Error` on a Cloudflare page, invalid JSON or an unexpected envelope.
|
|
147
|
+
*/
|
|
148
|
+
export declare function parseV2ex(payload: string, query: string, limit?: number): Source[];
|
|
149
|
+
/**
|
|
150
|
+
* Search the requested platforms and merge their answers.
|
|
151
|
+
*
|
|
152
|
+
* Every platform is asked concurrently. A direct endpoint that is blocked or
|
|
153
|
+
* unparsable is retried through `fallbackSearch`; a platform that still has
|
|
154
|
+
* nothing contributes a `failures` line instead of failing the call. Answers
|
|
155
|
+
* are interleaved by rank (one row from each platform, then the second row
|
|
156
|
+
* from each), duplicate URLs are dropped, and the merged list is capped at the
|
|
157
|
+
* caller's budget.
|
|
158
|
+
*
|
|
159
|
+
* @param query - what to search for; empty input yields an empty outcome.
|
|
160
|
+
* @param options - platform ids, budget, deadline, cancellation, tunnel, fetch
|
|
161
|
+
* seam and fallback search.
|
|
162
|
+
* @returns merged sources plus per-platform counts, channels and failures.
|
|
163
|
+
* @throws SearchError `SEARCH_ABORTED` when the caller cancelled the run.
|
|
164
|
+
*/
|
|
165
|
+
export declare function searchPlatforms(query: string, options?: PlatformSearchOptions): Promise<PlatformOutcome>;
|
|
166
|
+
/**
|
|
167
|
+
* Drop a generic fallback row that shares almost none of a specific query.
|
|
168
|
+
*
|
|
169
|
+
* Zhihu has no keyless direct endpoint. Its fallback sometimes puts a broad
|
|
170
|
+
* question such as "目前有哪些主流的 AI?" above an answer that actually names
|
|
171
|
+
* coding assistants. This is a light preference, not a hard filter: when fewer
|
|
172
|
+
* than two rows survive, the original list is returned.
|
|
173
|
+
*
|
|
174
|
+
* @param sources - fallback rows already filtered to the Zhihu host.
|
|
175
|
+
* @param query - the caller's query.
|
|
176
|
+
* @returns rows with at least two query-term matches, or the original list.
|
|
177
|
+
*/
|
|
178
|
+
export declare function normaliseZhihuFallback(sources: readonly Source[], query: string): Source[];
|