@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,286 @@
1
+ /** The name this tool prefers. */
2
+ export const SEARCH_MANY_TOOL_NAME = 'search_many';
3
+ /** The name used when another plugin already registered `search_many`. */
4
+ export const SEARCH_MANY_FALLBACK_NAME = 'web_search_many';
5
+ /** How many queries run at the same time; deliberately below the engine count. */
6
+ export const SEARCH_MANY_CONCURRENCY = 3;
7
+ /** Queries accepted in one call. */
8
+ export const SEARCH_MANY_MIN_QUERIES = 1;
9
+ /** Queries accepted in one call. */
10
+ export const SEARCH_MANY_MAX_QUERIES = 5;
11
+ /** Default result budget for a query that names none. */
12
+ export const SEARCH_MANY_DEFAULT_LIMIT = 5;
13
+ /** Largest per-query budget the tool accepts. */
14
+ export const SEARCH_MANY_MAX_LIMIT = 20;
15
+ /** True for a value usable as a record. */
16
+ function isRecord(value) {
17
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
18
+ }
19
+ /** The parameter schema, in the registry's raw JSON-Schema dialect. */
20
+ function parameterSchema() {
21
+ return {
22
+ type: 'object',
23
+ additionalProperties: false,
24
+ properties: {
25
+ queries: {
26
+ type: 'array',
27
+ items: { type: 'string' },
28
+ description: `The searches to run, ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} independent queries. Each runs as its own \`web_search\`.`,
29
+ },
30
+ limit: {
31
+ type: 'integer',
32
+ description: `Results wanted per query, 1–${SEARCH_MANY_MAX_LIMIT}; defaults to ${SEARCH_MANY_DEFAULT_LIMIT}.`,
33
+ },
34
+ withSites: {
35
+ type: 'boolean',
36
+ description: 'Attach the result hostname to every source. Defaults to false.',
37
+ },
38
+ },
39
+ required: ['queries'],
40
+ };
41
+ }
42
+ /** The output schema, in the same dialect. */
43
+ function outputSchema() {
44
+ const sourceSchema = {
45
+ type: 'object',
46
+ additionalProperties: false,
47
+ properties: {
48
+ url: { type: 'string' },
49
+ title: { type: 'string' },
50
+ snippet: { type: 'string' },
51
+ publishedAt: { type: 'string' },
52
+ site: { type: 'string' },
53
+ },
54
+ required: ['url'],
55
+ };
56
+ return {
57
+ type: 'object',
58
+ additionalProperties: false,
59
+ properties: {
60
+ results: {
61
+ type: 'array',
62
+ items: {
63
+ type: 'object',
64
+ additionalProperties: false,
65
+ properties: {
66
+ query: { type: 'string' },
67
+ sources: { type: 'array', items: sourceSchema },
68
+ failures: { type: 'array', items: { type: 'string' } },
69
+ },
70
+ required: ['query', 'sources', 'failures'],
71
+ },
72
+ },
73
+ truncated: { type: 'boolean' },
74
+ },
75
+ required: ['results', 'truncated'],
76
+ };
77
+ }
78
+ /** Parse model arguments into a validated shape, throwing on nonsense. */
79
+ function parseArgs(args) {
80
+ if (!isRecord(args))
81
+ throw new Error('search_many requires arguments');
82
+ const rawQueries = args['queries'];
83
+ if (!Array.isArray(rawQueries)) {
84
+ throw new Error(`search_many requires a queries array with ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} entries`);
85
+ }
86
+ if (rawQueries.length < SEARCH_MANY_MIN_QUERIES || rawQueries.length > SEARCH_MANY_MAX_QUERIES) {
87
+ throw new Error(`search_many accepts ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} queries, got ${rawQueries.length}`);
88
+ }
89
+ const queries = [];
90
+ for (const raw of rawQueries) {
91
+ if (typeof raw !== 'string' || raw.trim().length === 0) {
92
+ throw new Error('search_many requires every query to be a non-empty string');
93
+ }
94
+ queries.push(raw.trim());
95
+ }
96
+ const rawLimit = args['limit'];
97
+ let limit = SEARCH_MANY_DEFAULT_LIMIT;
98
+ if (rawLimit !== undefined) {
99
+ if (typeof rawLimit !== 'number' || !Number.isInteger(rawLimit)
100
+ || rawLimit < 1 || rawLimit > SEARCH_MANY_MAX_LIMIT) {
101
+ throw new Error(`search_many limit must be an integer between 1 and ${SEARCH_MANY_MAX_LIMIT}`);
102
+ }
103
+ limit = rawLimit;
104
+ }
105
+ const rawSites = args['withSites'];
106
+ if (rawSites !== undefined && typeof rawSites !== 'boolean') {
107
+ throw new Error('search_many withSites must be a boolean');
108
+ }
109
+ return { queries, limit, withSites: rawSites === true };
110
+ }
111
+ /** Return the hostname a source is served from, or `undefined` for a bad URL. */
112
+ function siteOf(url) {
113
+ try {
114
+ const host = new URL(url).hostname.trim().toLowerCase();
115
+ return host.length === 0 ? undefined : host;
116
+ }
117
+ catch {
118
+ return undefined;
119
+ }
120
+ }
121
+ /** Attach the hostname without mutating the cached source object. */
122
+ function withSite(source) {
123
+ const site = siteOf(source.url);
124
+ return site === undefined ? { ...source } : { ...source, site };
125
+ }
126
+ /**
127
+ * Run `worker` over items with at most `limit` in flight, preserving order.
128
+ * A dedicated helper keeps the concurrency contract visible instead of relying
129
+ * on an unstated promise-pool behaviour.
130
+ */
131
+ async function mapLimit(items, limit, worker) {
132
+ const results = new Array(items.length);
133
+ let next = 0;
134
+ const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
135
+ for (;;) {
136
+ const index = next;
137
+ next += 1;
138
+ if (index >= items.length)
139
+ return;
140
+ results[index] = await worker(items[index], index);
141
+ }
142
+ });
143
+ await Promise.all(runners);
144
+ return results;
145
+ }
146
+ /** Format the model-facing result text. */
147
+ export function formatSearchManyValue(value) {
148
+ const total = value.results.reduce((sum, result) => sum + result.sources.length, 0);
149
+ const failed = value.results.reduce((sum, result) => sum + result.failures.length, 0);
150
+ const head = `Batch search: ${value.results.length} quer${value.results.length === 1 ? 'y' : 'ies'}, `
151
+ + `${total} result(s)${failed === 0 ? '' : `, ${failed} failure(s)`}${value.truncated ? ' (truncated)' : ''}.`;
152
+ const sections = value.results.map((result, index) => {
153
+ const lines = [`${index + 1}. ${JSON.stringify(result.query)} — ${result.sources.length} result(s)`];
154
+ for (const [position, source] of result.sources.entries()) {
155
+ const title = source.title !== undefined && source.title.length > 0 ? source.title : source.url;
156
+ lines.push(` ${position + 1}. ${title}`);
157
+ lines.push(` ${source.url}${source.site === undefined ? '' : ` [${source.site}]`}`);
158
+ if (source.snippet !== undefined && source.snippet.length > 0)
159
+ lines.push(` ${source.snippet}`);
160
+ if (source.publishedAt !== undefined && source.publishedAt.length > 0) {
161
+ lines.push(` ${source.publishedAt}`);
162
+ }
163
+ }
164
+ if (result.failures.length > 0)
165
+ lines.push(` failures: ${result.failures.join('; ')}`);
166
+ return lines.join('\n');
167
+ });
168
+ return [head, '', ...sections].join('\n').trimEnd();
169
+ }
170
+ /**
171
+ * Build the tool definition the host registry accepts.
172
+ * @param name - `search_many`, or the fallback when that name is taken.
173
+ * @param runQuery - executes one search through the plugin's normal path.
174
+ * @returns a registry-ready definition.
175
+ */
176
+ export function createSearchManyDefinition(name, runQuery) {
177
+ return {
178
+ name,
179
+ description: 'Run 1–5 independent web searches in one call and return each query\'s results separately. Use it when one question needs several distinct queries; a failed query does not stop the others. For a single query use web_search.',
180
+ parameters: parameterSchema(),
181
+ output: {
182
+ schema: outputSchema(),
183
+ render: (_args, value) => [{ type: 'text', text: formatSearchManyValue(value) }],
184
+ },
185
+ timeoutMs: 30_000,
186
+ isConcurrencySafe: () => true,
187
+ async execute(args, exec) {
188
+ const parsed = parseArgs(args);
189
+ const outcomes = await mapLimit(parsed.queries, SEARCH_MANY_CONCURRENCY, async (query) => {
190
+ try {
191
+ const outcome = await runQuery(query, parsed.limit, exec.signal);
192
+ const mapped = parsed.withSites
193
+ ? outcome.sources.map(withSite)
194
+ : outcome.sources.map((source) => ({ ...source }));
195
+ const truncated = outcome.truncated || mapped.length > parsed.limit;
196
+ return {
197
+ result: {
198
+ query,
199
+ sources: mapped.slice(0, parsed.limit),
200
+ failures: [...outcome.failures],
201
+ },
202
+ truncated,
203
+ };
204
+ }
205
+ catch (error) {
206
+ return {
207
+ result: {
208
+ query,
209
+ sources: [],
210
+ failures: [error instanceof Error ? error.message : String(error)],
211
+ },
212
+ truncated: false,
213
+ };
214
+ }
215
+ });
216
+ return {
217
+ results: outcomes.map((entry) => entry.result),
218
+ truncated: outcomes.some((entry) => entry.truncated),
219
+ };
220
+ },
221
+ };
222
+ }
223
+ /**
224
+ * Pick the tool name this deployment can accept.
225
+ * @param tools - the live registry.
226
+ * @returns the preferred or fallback name.
227
+ */
228
+ export function pickSearchManyToolName(tools) {
229
+ if (typeof tools.get !== 'function')
230
+ return SEARCH_MANY_TOOL_NAME;
231
+ try {
232
+ return tools.get(SEARCH_MANY_TOOL_NAME) === undefined ? SEARCH_MANY_TOOL_NAME : SEARCH_MANY_FALLBACK_NAME;
233
+ }
234
+ catch {
235
+ return SEARCH_MANY_TOOL_NAME;
236
+ }
237
+ }
238
+ /** Report a registration failure without throwing out of the plugin. */
239
+ function report(options, error) {
240
+ const message = error instanceof Error ? error.message : String(error);
241
+ try {
242
+ options.warn?.(`search_many registration skipped: ${message}`);
243
+ }
244
+ catch {
245
+ // A diagnostic callback must never break plugin loading.
246
+ }
247
+ }
248
+ /**
249
+ * Register the tool when a registry is available.
250
+ *
251
+ * Every failure path is contained: a duplicate name falls back, and any
252
+ * registry rejection is reported and swallowed so `web_search` keeps working.
253
+ * The returned registration owns the host disposer, so plugin disposal removes
254
+ * exactly the definition this call added.
255
+ *
256
+ * @param tools - the live registry.
257
+ * @param runQuery - executes one search through the plugin's normal path.
258
+ * @param options - diagnostics.
259
+ * @returns the registered name and its disposer, or `undefined` when skipped.
260
+ */
261
+ export function registerSearchManyTool(tools, runQuery, options = {}) {
262
+ const name = pickSearchManyToolName(tools);
263
+ try {
264
+ const registered = tools.register(createSearchManyDefinition(name, runQuery));
265
+ const disposeHost = typeof registered === 'function' ? registered : undefined;
266
+ let disposed = false;
267
+ return {
268
+ name,
269
+ dispose: () => {
270
+ if (disposed)
271
+ return;
272
+ disposed = true;
273
+ try {
274
+ disposeHost?.();
275
+ }
276
+ catch {
277
+ // Disposal must stay contained; the plugin is already going away.
278
+ }
279
+ },
280
+ };
281
+ }
282
+ catch (error) {
283
+ report(options, error);
284
+ return undefined;
285
+ }
286
+ }
package/lib/transport.js CHANGED
@@ -1,6 +1,19 @@
1
1
  import { SearchError, parseRetryAfter } from "./errors.js";
2
2
  import { proxyGet } from "./proxy.js";
3
3
  import { echoesQuery } from "./parsers.js";
4
+ /** Markers of an interstitial anti-bot page, narrow enough not to hit prose. */
5
+ const BOT_WALL_MARKERS = ['qcaptcha', '访问异常', 'unusual traffic', 'verify you are human', '安全验证'];
6
+ /**
7
+ * Whether a payload is an anti-bot interstitial rather than a result page.
8
+ * @param payload - response body.
9
+ * @returns true when the page is a wall.
10
+ */
11
+ function looksLikeBotWall(payload) {
12
+ if (payload.length === 0)
13
+ return true;
14
+ const sample = payload.slice(0, 4000).toLowerCase();
15
+ return BOT_WALL_MARKERS.some((marker) => sample.includes(marker.toLowerCase()));
16
+ }
4
17
  /** The user agent the scraped engines see. */
5
18
  const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15';
6
19
  /**
@@ -92,6 +105,12 @@ async function attempt(host, url, signal, timeoutMs, proxy, query) {
92
105
  }
93
106
  if (status < 200 || status >= 300)
94
107
  return { ok: false, error: statusError(host.id, status, retryAfter) };
108
+ // A wall served with HTTP 200 is not an empty result set. Treating it as one
109
+ // quietly removed an engine from every search while its slot stayed reserved,
110
+ // and the cooldown never engaged because nothing had failed.
111
+ if (looksLikeBotWall(payload)) {
112
+ return { ok: false, error: new SearchError(`${host.id}: bot verification page`, 'SEARCH_BLOCKED', true, retryAfter) };
113
+ }
95
114
  // A JSON API validates itself through its envelope, so the echoed-query check
96
115
  // (which exists to catch a rate-limit page served with HTTP 200) does not
97
116
  // apply to it.
package/lib/types.d.ts CHANGED
@@ -117,13 +117,22 @@ export interface RankOptions {
117
117
  timeRange?: TimeRange;
118
118
  /** Per-engine weight in the pre-fusion interleave. */
119
119
  weights?: number[];
120
+ /**
121
+ * Evaluation instant in milliseconds for the freshness boost, defaulting to
122
+ * `Date.now()`. Benchmarks pin this to the fixture's recording time so a
123
+ * replay does not drift as the fixture ages.
124
+ */
125
+ now?: number;
120
126
  }
121
127
  /** Cache knobs. */
122
128
  export interface CacheOptions {
123
- /** Entry lifetime in milliseconds. `0` disables the cache. */
124
- ttlMs?: number;
125
- /** Maximum retained entries; the least recently used entry is evicted. */
126
- maxEntries?: number;
129
+ /**
130
+ * Entry lifetime in milliseconds. `0` disables the cache. A reader keeps
131
+ * Settings edits live without a remount.
132
+ */
133
+ ttlMs?: number | (() => number);
134
+ /** Maximum retained entries, or a reader for it; the least recently used goes. */
135
+ maxEntries?: number | (() => number);
127
136
  }
128
137
  /** Health knobs for the per-engine cooldown. */
129
138
  export interface HealthOptions {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fffangchenghao/dsh-web-search",
3
- "version": "0.11.0",
3
+ "version": "0.13.0",
4
4
  "description": "DSH bundle: a key-free web search provider. Leads with the anonymous AnySearch API and falls back to public result pages (DuckDuckGo, Bing, 360), merged by reciprocal-rank fusion with a bilingual region policy, an engine cooldown and a TTL cache.",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",