@fffangchenghao/dsh-web-search 0.19.3 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +194 -33
  3. package/README.zh-CN.md +171 -34
  4. package/SECURITY.md +42 -7
  5. package/client.js +131 -9
  6. package/docs/store-evidence.md +10 -5
  7. package/lib/advanced-search.d.ts +107 -0
  8. package/lib/advanced-search.js +255 -0
  9. package/lib/bridge.d.ts +16 -0
  10. package/lib/bridge.js +160 -0
  11. package/lib/cache.d.ts +11 -4
  12. package/lib/cache.js +9 -5
  13. package/lib/config.d.ts +19 -3
  14. package/lib/config.js +24 -0
  15. package/lib/domain-tool.d.ts +5 -0
  16. package/lib/domain-tool.js +45 -6
  17. package/lib/engine-test.d.ts +116 -0
  18. package/lib/engine-test.js +256 -0
  19. package/lib/engines.d.ts +11 -1
  20. package/lib/engines.js +118 -19
  21. package/lib/errors.d.ts +16 -2
  22. package/lib/errors.js +40 -1
  23. package/lib/index.d.ts +24 -2
  24. package/lib/index.js +304 -20
  25. package/lib/inject.d.ts +43 -0
  26. package/lib/inject.js +1 -0
  27. package/lib/markdown.js +18 -1
  28. package/lib/mirrors.d.ts +23 -0
  29. package/lib/mirrors.js +69 -0
  30. package/lib/official-feed.d.ts +114 -0
  31. package/lib/official-feed.js +288 -0
  32. package/lib/official-support.d.ts +74 -0
  33. package/lib/official-support.js +134 -0
  34. package/lib/platform-tool.d.ts +77 -0
  35. package/lib/platform-tool.js +173 -0
  36. package/lib/platforms.d.ts +178 -0
  37. package/lib/platforms.js +1035 -0
  38. package/lib/query-router.d.ts +29 -0
  39. package/lib/query-router.js +58 -0
  40. package/lib/read-page.js +2 -1
  41. package/lib/research-cache.d.ts +115 -0
  42. package/lib/research-cache.js +275 -0
  43. package/lib/research.d.ts +45 -4
  44. package/lib/research.js +505 -22
  45. package/lib/search-many.d.ts +7 -1
  46. package/lib/search-many.js +7 -2
  47. package/lib/system-prompt.d.ts +27 -0
  48. package/lib/system-prompt.js +50 -0
  49. package/lib/tool-suite.d.ts +97 -0
  50. package/lib/tool-suite.js +224 -0
  51. package/lib/transport.d.ts +33 -2
  52. package/lib/transport.js +42 -9
  53. package/lib/types.d.ts +20 -0
  54. package/lib/types.js +9 -1
  55. package/lib/untrusted.d.ts +33 -0
  56. package/lib/untrusted.js +42 -0
  57. package/lib/verticals.d.ts +3 -1
  58. package/lib/verticals.js +26 -7
  59. package/package.json +5 -3
  60. package/skills/dsh-web-search/SKILL.md +21 -6
@@ -0,0 +1,256 @@
1
+ import { wrapUntrustedContent } from "./untrusted.js";
2
+ import { HOSTS } from "./engines.js";
3
+ /** The name this tool prefers. */
4
+ export const ENGINE_TEST_TOOL_NAME = 'engine_test';
5
+ /** The name used when another plugin already registered `engine_test`. */
6
+ export const ENGINE_TEST_FALLBACK_NAME = 'web_engine_test';
7
+ /** Default query; broad enough that every engine has an answer. */
8
+ export const ENGINE_TEST_DEFAULT_QUERY = 'DeepSeek Harness';
9
+ /** How many engines one run may ask at the same time. */
10
+ const ENGINE_TEST_CONCURRENCY = 3;
11
+ /** True for a value usable as a record. */
12
+ function isRecord(value) {
13
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
14
+ }
15
+ /** One short error line for the tool output. */
16
+ function failureReason(outcome) {
17
+ return outcome.failureDetails[0]?.reason ?? outcome.failures[0];
18
+ }
19
+ /** Run `worker` over items with at most `limit` in flight, preserving order. */
20
+ async function mapLimit(items, limit, worker) {
21
+ const results = new Array(items.length);
22
+ let next = 0;
23
+ const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
24
+ for (;;) {
25
+ const index = next;
26
+ next += 1;
27
+ if (index >= items.length)
28
+ return;
29
+ results[index] = await worker(items[index], index);
30
+ }
31
+ });
32
+ await Promise.all(runners);
33
+ return results;
34
+ }
35
+ /** The parameter schema, in the registry's raw JSON-Schema dialect. */
36
+ function parameterSchema() {
37
+ return {
38
+ type: 'object',
39
+ additionalProperties: false,
40
+ properties: {
41
+ engine: {
42
+ type: 'string',
43
+ description: `Optional engine id to test alone, e.g. anysearch, exa, or ddg-html. Omit to test all ${HOSTS.length} engines.`,
44
+ },
45
+ query: {
46
+ type: 'string',
47
+ description: `Optional probe query; defaults to ${JSON.stringify(ENGINE_TEST_DEFAULT_QUERY)}.`,
48
+ },
49
+ },
50
+ };
51
+ }
52
+ /** The output schema, in the same dialect. */
53
+ function outputSchema() {
54
+ return {
55
+ type: 'object',
56
+ additionalProperties: false,
57
+ properties: {
58
+ query: { type: 'string' },
59
+ ok: { type: 'integer' },
60
+ failed: { type: 'integer' },
61
+ results: {
62
+ type: 'array',
63
+ items: {
64
+ type: 'object',
65
+ additionalProperties: false,
66
+ properties: {
67
+ engine: { type: 'string' },
68
+ ok: { type: 'boolean' },
69
+ latencyMs: { type: 'integer' },
70
+ count: { type: 'integer' },
71
+ failureKind: { type: 'string' },
72
+ code: { type: 'string' },
73
+ status: { type: 'integer' },
74
+ reason: { type: 'string' },
75
+ sample: { type: 'string' },
76
+ },
77
+ required: ['engine', 'ok', 'latencyMs', 'count'],
78
+ },
79
+ },
80
+ },
81
+ required: ['query', 'ok', 'failed', 'results'],
82
+ };
83
+ }
84
+ /** Format the model-facing result text. */
85
+ export function formatEngineTestValue(value) {
86
+ const head = `Engine test (${value.query}): ${value.ok} OK, ${value.failed} failed.`;
87
+ const lines = value.results.map((result) => {
88
+ if (result.ok) {
89
+ const sample = result.sample === undefined ? '' : ` — ${result.sample}`;
90
+ return `- ${result.engine}: OK (${result.latencyMs}ms, ${result.count} result(s))${sample}`;
91
+ }
92
+ const kind = result.failureKind === undefined ? '' : ` [${result.failureKind}]`;
93
+ const reason = result.reason === undefined ? 'no result' : result.reason;
94
+ return `- ${result.engine}: FAIL${kind} (${result.latencyMs}ms) ${reason}`;
95
+ });
96
+ return [head, '', ...lines].join('\n');
97
+ }
98
+ /** Parse model arguments into a validated shape, throwing on nonsense. */
99
+ function parseArgs(args) {
100
+ if (!isRecord(args))
101
+ throw new Error('engine_test requires arguments');
102
+ const rawEngine = args['engine'];
103
+ let engine;
104
+ if (rawEngine !== undefined) {
105
+ if (typeof rawEngine !== 'string' || rawEngine.trim().length === 0) {
106
+ throw new Error('engine_test engine must be a non-empty string');
107
+ }
108
+ engine = rawEngine.trim();
109
+ if (!HOSTS.some((host) => host.id === engine)) {
110
+ throw new Error(`engine_test unknown engine ${JSON.stringify(engine)}; expected one of ${HOSTS.map((host) => host.id).join(', ')}`);
111
+ }
112
+ }
113
+ const rawQuery = args['query'];
114
+ let query = ENGINE_TEST_DEFAULT_QUERY;
115
+ if (rawQuery !== undefined) {
116
+ if (typeof rawQuery !== 'string' || rawQuery.trim().length === 0) {
117
+ throw new Error('engine_test query must be a non-empty string');
118
+ }
119
+ query = rawQuery.trim();
120
+ }
121
+ return { engine, query };
122
+ }
123
+ /**
124
+ * Run a list of engines and assemble the tool value.
125
+ *
126
+ * Exported so the Settings bridge and the model tool share one implementation
127
+ * of latency, sample and failure reporting.
128
+ *
129
+ * @param runTest - tests one engine through the normal search path.
130
+ * @param engines - engine ids to test, in display order.
131
+ * @param query - probe query.
132
+ * @param signal - cancellation from the caller.
133
+ * @returns per-engine results plus counts.
134
+ */
135
+ export async function runEngineTestValue(runTest, engines, query, signal) {
136
+ const results = await mapLimit(engines, ENGINE_TEST_CONCURRENCY, async (engine) => {
137
+ const started = Date.now();
138
+ try {
139
+ const outcome = await runTest(engine, query, signal);
140
+ const latencyMs = Date.now() - started;
141
+ if (outcome.sources.length > 0) {
142
+ const first = outcome.sources[0];
143
+ return {
144
+ engine,
145
+ ok: true,
146
+ latencyMs,
147
+ count: outcome.sources.length,
148
+ ...(first.title === undefined || first.title.length === 0 ? {} : { sample: first.title }),
149
+ };
150
+ }
151
+ const detail = outcome.failureDetails[0];
152
+ const reason = failureReason(outcome) ?? 'no result';
153
+ return {
154
+ engine,
155
+ ok: false,
156
+ latencyMs,
157
+ count: 0,
158
+ ...(detail?.kind === undefined ? {} : { failureKind: detail.kind }),
159
+ ...(detail?.code === undefined ? {} : { code: detail.code }),
160
+ ...(detail?.status === undefined ? {} : { status: detail.status }),
161
+ reason,
162
+ };
163
+ }
164
+ catch (error) {
165
+ return {
166
+ engine,
167
+ ok: false,
168
+ latencyMs: Date.now() - started,
169
+ count: 0,
170
+ failureKind: 'unknown',
171
+ reason: error instanceof Error ? error.message : String(error),
172
+ };
173
+ }
174
+ });
175
+ const ok = results.filter((result) => result.ok).length;
176
+ return { query, results, ok, failed: results.length - ok };
177
+ }
178
+ /**
179
+ * Build the tool definition the host registry accepts.
180
+ * @param name - `engine_test`, or the fallback when that name is taken.
181
+ * @param runTest - tests one engine through the normal search path.
182
+ * @returns a registry-ready definition.
183
+ */
184
+ export function createEngineTestDefinition(name, runTest) {
185
+ return {
186
+ name,
187
+ description: 'Test the available search engines with one probe each and report which answered, their latency, and a failure class (quota, auth, bot-wall, transient, invalid-response) for the rest. Use it when search fails or is slow; it never changes the cache, fallback or cooldown state.',
188
+ parameters: parameterSchema(),
189
+ output: {
190
+ schema: outputSchema(),
191
+ render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatEngineTestValue(value)) }],
192
+ },
193
+ timeoutMs: 60_000,
194
+ isConcurrencySafe: () => true,
195
+ async execute(args, exec) {
196
+ const parsed = parseArgs(args);
197
+ const engines = parsed.engine === undefined ? HOSTS.map((host) => host.id) : [parsed.engine];
198
+ return runEngineTestValue(runTest, engines, parsed.query, exec.signal);
199
+ },
200
+ };
201
+ }
202
+ /**
203
+ * Pick the tool name this deployment can accept.
204
+ * @param tools - the live registry.
205
+ * @returns the preferred or fallback name.
206
+ */
207
+ export function pickEngineTestToolName(tools) {
208
+ if (typeof tools.get !== 'function')
209
+ return ENGINE_TEST_TOOL_NAME;
210
+ try {
211
+ return tools.get(ENGINE_TEST_TOOL_NAME) === undefined ? ENGINE_TEST_TOOL_NAME : ENGINE_TEST_FALLBACK_NAME;
212
+ }
213
+ catch {
214
+ return ENGINE_TEST_TOOL_NAME;
215
+ }
216
+ }
217
+ /** Report a registration failure without throwing out of the plugin. */
218
+ export function warnEngineTest(message) {
219
+ try {
220
+ console.warn(`[dsh-web-search-bing] ${message}`);
221
+ }
222
+ catch {
223
+ // Logging must never break plugin loading.
224
+ }
225
+ }
226
+ /**
227
+ * Register the tool and return its undo handle.
228
+ * @param tools - the host tool registry.
229
+ * @param runTest - tests one engine.
230
+ * @param options - diagnostics hook.
231
+ * @returns the registration, or `undefined` when the registry refused it.
232
+ */
233
+ export function registerEngineTestTool(tools, runTest, options = {}) {
234
+ const warn = options.warn ?? warnEngineTest;
235
+ const name = pickEngineTestToolName(tools);
236
+ const definition = createEngineTestDefinition(name, runTest);
237
+ try {
238
+ const handle = tools.register(definition);
239
+ const unregister = typeof handle === 'function' ? handle : undefined;
240
+ return {
241
+ name,
242
+ dispose: () => {
243
+ try {
244
+ unregister?.();
245
+ }
246
+ catch (error) {
247
+ warn(`engine_test unregister failed: ${error instanceof Error ? error.message : String(error)}`);
248
+ }
249
+ },
250
+ };
251
+ }
252
+ catch (error) {
253
+ warn(`engine_test registration skipped: ${error instanceof Error ? error.message : String(error)}`);
254
+ return undefined;
255
+ }
256
+ }
package/lib/engines.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  *
12
12
  * @module @fangchenghao/dsh-web-search/engines
13
13
  */
14
- import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
14
+ import type { EngineAnswer, EngineFailure, FailureKind, HostSpec, RankOptions, Region, SafeSearch, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
15
15
  import type { SearchCache } from './cache.ts';
16
16
  import type { EngineHealth } from './health.ts';
17
17
  /**
@@ -30,6 +30,12 @@ export interface EngineRunOptions extends SearchOptions {
30
30
  anysearchDomain?: string;
31
31
  /** Ranking knobs, falling back to the documented defaults. */
32
32
  rank?: RankOptions;
33
+ /** Failure classes that may trigger the fallback tier; absent means all. */
34
+ fallbackOn?: readonly FailureKind[];
35
+ /** Domains the caller never wants in a result list. */
36
+ blockedDomains?: readonly string[];
37
+ /** Content-filter level forwarded to engines that support it. */
38
+ safeSearch?: SafeSearch;
33
39
  /** Result cache to consult and fill, when one is configured. */
34
40
  cache?: SearchCache;
35
41
  /** Cooldown tracker, when one is configured. */
@@ -72,6 +78,10 @@ export interface SearchTrace {
72
78
  sources: Source[];
73
79
  /** Failure strings accumulated by this run. */
74
80
  failures: string[];
81
+ /** Structured failures, including the class used by `fallbackOn`. */
82
+ failureDetails: EngineFailure[];
83
+ /** Human-readable note when policy changed the answer. */
84
+ note?: string;
75
85
  /** Whether the result came from the cache. */
76
86
  cached: boolean;
77
87
  }
package/lib/engines.js CHANGED
@@ -1,4 +1,5 @@
1
- import { SearchError } from "./errors.js";
1
+ import { FAILURE_KINDS } from "./types.js";
2
+ import { SearchError, classifyFailure } from "./errors.js";
2
3
  import { filterByTimeRange, preferWindow } from "./freshness.js";
3
4
  import { resolveProxy } from "./proxy.js";
4
5
  import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseExaMcp, parseFirecrawl, parseSo } from "./parsers.js";
@@ -185,6 +186,69 @@ const SOFT_DEADLINE = {
185
186
  function leadsQuery(host, lead) {
186
187
  return host.region === 'both' || host.region === lead;
187
188
  }
189
+ /** Build the structured counterpart of a failure string. */
190
+ function failureDetail(id, reason, error) {
191
+ return {
192
+ id,
193
+ reason,
194
+ ...(error === undefined ? {} : { code: error.code }),
195
+ kind: error === undefined ? 'unknown' : classifyFailure(error),
196
+ ...(error?.status === undefined ? {} : { status: error.status }),
197
+ };
198
+ }
199
+ /**
200
+ * Normalise a configured domain for suffix matching.
201
+ *
202
+ * Accepts a bare domain, a `*.domain` wildcard or a pasted URL, because all
203
+ * three are things a person reasonably types into a blocklist field.
204
+ */
205
+ function normaliseDomain(value) {
206
+ let text = value.trim().toLowerCase();
207
+ if (text.includes('://')) {
208
+ try {
209
+ text = new URL(text).hostname;
210
+ }
211
+ catch {
212
+ // Fall through: the bare-domain cleanup below may still help.
213
+ }
214
+ }
215
+ return text
216
+ .replace(/^\*\./u, '')
217
+ .replace(/^\.+/u, '')
218
+ .replace(/\.+$/u, '')
219
+ .replace(/\/.*$/u, '')
220
+ .trim();
221
+ }
222
+ /** Whether a URL's host is the configured domain or a subdomain of it. */
223
+ function isBlockedHost(hostname, blocked) {
224
+ const host = hostname.toLowerCase().replace(/\.$/u, '');
225
+ return blocked.some((domain) => host === domain || host.endsWith(`.${domain}`));
226
+ }
227
+ /**
228
+ * Remove a caller-blocked domain from a result list.
229
+ *
230
+ * The filter is applied after fusion so it can never change the ranking of the
231
+ * rows that survive; it only removes rows the user explicitly excluded.
232
+ */
233
+ function removeBlocked(sources, blocked) {
234
+ if (blocked.length === 0)
235
+ return [...sources];
236
+ return sources.filter((source) => {
237
+ try {
238
+ return !isBlockedHost(new URL(source.url).hostname, blocked);
239
+ }
240
+ catch {
241
+ return true;
242
+ }
243
+ });
244
+ }
245
+ /** Whether the primary failures permit a fallback under the configured policy. */
246
+ function allowsFallback(details, fallbackOn) {
247
+ if (fallbackOn === undefined)
248
+ return true;
249
+ const allowed = new Set(fallbackOn);
250
+ return details.every((detail) => allowed.has(detail.kind));
251
+ }
188
252
  /**
189
253
  * Ask one engine, rotating to the next key when the failure blames the key.
190
254
  *
@@ -202,9 +266,9 @@ function leadsQuery(host, lead) {
202
266
  * @param domain - AnySearch vertical domain, if configured.
203
267
  * @returns the last attempt's outcome.
204
268
  */
205
- async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain) {
269
+ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain, safeSearch) {
206
270
  if (host.id !== 'anysearch' || pool.size === 0) {
207
- return searchHost(host, query, signal, timeoutMs, proxy, timeRange);
271
+ return searchHost(host, query, signal, timeoutMs, proxy, timeRange, safeSearch);
208
272
  }
209
273
  let key = pool.pick();
210
274
  let last;
@@ -213,7 +277,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
213
277
  anysearchApiKey: key,
214
278
  ...(domain === undefined ? {} : { anysearchDomain: domain }),
215
279
  });
216
- const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange);
280
+ const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange, safeSearch);
217
281
  if (outcome.ok) {
218
282
  pool.reward(key);
219
283
  return outcome;
@@ -242,7 +306,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
242
306
  * @param health - cooldown tracker to update, when one is configured.
243
307
  * @returns one answer per engine that produced sources.
244
308
  */
245
- async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits, health, pool, anysearchDomain) {
309
+ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails, proxy, limits, health, pool, anysearchDomain, safeSearch) {
246
310
  const counters = limits.lead === 'all' || !hosts.some((host) => leadsQuery(host, limits.lead))
247
311
  ? () => true
248
312
  : (host) => leadsQuery(host, limits.lead);
@@ -275,7 +339,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
275
339
  releasedEarly = early;
276
340
  released();
277
341
  };
278
- const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain).then((outcome) => {
342
+ const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain, safeSearch).then((outcome) => {
279
343
  if (outcome.ok) {
280
344
  rawCounts[index] = outcome.sources.length;
281
345
  outcomes[index] = { host, sources: filterRelevant(outcome.sources, query) };
@@ -311,13 +375,18 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
311
375
  for (const [index, outcome] of outcomes.entries()) {
312
376
  const host = hosts[index];
313
377
  if (outcome === undefined) {
314
- failures.push(releasedEarly ? `${host.id}: aborted once enough results had arrived` : `${host.id}: still running at deadline`);
378
+ const reason = releasedEarly ? `${host.id}: aborted once enough results had arrived` : `${host.id}: still running at deadline`;
379
+ failures.push(reason);
380
+ failureDetails.push(failureDetail(host.id, reason));
315
381
  }
316
382
  else if (outcome instanceof SearchError) {
317
383
  failures.push(outcome.message);
384
+ failureDetails.push(failureDetail(host.id, outcome.message, outcome));
318
385
  }
319
386
  else if (outcome.sources.length === 0) {
320
- failures.push(`${host.id}: ${(rawCounts[index] ?? 0) > 0 ? 'no relevant results' : 'no results'}`);
387
+ const reason = `${host.id}: ${(rawCounts[index] ?? 0) > 0 ? 'no relevant results' : 'no results'}`;
388
+ failures.push(reason);
389
+ failureDetails.push(failureDetail(host.id, reason));
321
390
  }
322
391
  else {
323
392
  answers.push(outcome);
@@ -388,16 +457,21 @@ export async function search(query, signal, options) {
388
457
  // a throttled engine is still better than no search.
389
458
  const now = Date.now();
390
459
  const failures = [];
460
+ const failureDetails = [];
391
461
  const rested = hosts.filter((host) => options.health?.isCooling(host.id, now) !== true);
392
462
  if (rested.length > 0 && options.health !== undefined) {
393
463
  for (const host of hosts) {
394
464
  if (rested.includes(host))
395
465
  continue;
396
- const reason = options.health.skipped(host.id, now);
397
- failures.push(`${host.id}: ${reason ?? 'cooling down'}`);
466
+ const reason = `${host.id}: ${options.health.skipped(host.id, now) ?? 'cooling down'}`;
467
+ failures.push(reason);
468
+ failureDetails.push(failureDetail(host.id, reason));
398
469
  }
399
470
  }
400
471
  const usable = rested.length > 0 ? rested : hosts;
472
+ const blocked = (options.blockedDomains ?? []).map(normaliseDomain).filter((domain) => domain.length > 0);
473
+ const safeSearch = options.safeSearch ?? 'off';
474
+ const fallbackOn = options.fallbackOn;
401
475
  const ascii = !CJK.test(query);
402
476
  const lead = ascii ? 'global' : 'cn';
403
477
  const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
@@ -428,9 +502,20 @@ export async function search(query, signal, options) {
428
502
  // the cooldown filter: an engine entering or leaving a cooldown must not
429
503
  // invalidate an entry that is still fresh, or every throttle would turn the
430
504
  // next identical query into a cache miss.
431
- const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
505
+ const key = options.cache === undefined ? undefined : cacheKey({
506
+ query,
507
+ limit,
508
+ wait,
509
+ lead,
510
+ engines: hosts.map((host) => host.id),
511
+ timeRange,
512
+ ...(safeSearch === 'off' ? {} : { safeSearch }),
513
+ ...(fallbackOn === undefined || fallbackOn.length === FAILURE_KINDS.length ? {} : { fallbackOn }),
514
+ ...(blocked.length === 0 ? {} : { blockedDomains: blocked }),
515
+ });
432
516
  const cached = key === undefined ? undefined : options.cache?.get(key);
433
- if (cached !== undefined) {
517
+ const cachedSources = cached === undefined ? undefined : removeBlocked(cached, blocked);
518
+ if (cachedSources !== undefined && cachedSources.length > 0) {
434
519
  options.trace?.({
435
520
  limit,
436
521
  lead,
@@ -439,27 +524,39 @@ export async function search(query, signal, options) {
439
524
  rank,
440
525
  tier: 'cache',
441
526
  answers: [],
442
- sources: cached,
527
+ sources: cachedSources,
443
528
  failures: [...failures],
529
+ failureDetails: [...failureDetails],
444
530
  cached: true,
445
531
  });
446
- return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
532
+ return { sources: cachedSources, failures, failureDetails, ...(proxy === undefined ? {} : { proxy }) };
447
533
  }
448
534
  const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
449
535
  const primary = ordered.filter((host) => host.tier === 'primary');
450
536
  const fallback = ordered.filter((host) => host.tier === 'fallback');
451
537
  let tier = 'none';
452
- let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
538
+ let note;
539
+ let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
453
540
  if (answers.length > 0)
454
541
  tier = 'primary';
455
542
  let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
456
543
  if (sources.length === 0 && fallback.length > 0) {
457
- tier = 'fallback';
458
- answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
459
- sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
544
+ if (allowsFallback(failureDetails, fallbackOn)) {
545
+ tier = 'fallback';
546
+ const kinds = [...new Set(failureDetails.map((detail) => detail.kind))];
547
+ answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
548
+ sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
549
+ if (answers.length > 0 && kinds.length > 0)
550
+ note = `primary engines failed (${kinds.join(', ')}); using fallback.`;
551
+ }
552
+ else {
553
+ const blockedKind = failureDetails.find((detail) => !(fallbackOn ?? []).includes(detail.kind))?.kind ?? 'unknown';
554
+ note = `fallback skipped: ${blockedKind} is not enabled in fallbackOn.`;
555
+ }
460
556
  }
461
557
  if (signal?.aborted === true)
462
558
  throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
559
+ sources = removeBlocked(sources, blocked);
463
560
  if (key !== undefined && sources.length > 0)
464
561
  options.cache?.set(key, sources);
465
562
  options.trace?.({
@@ -472,7 +569,9 @@ export async function search(query, signal, options) {
472
569
  answers,
473
570
  sources,
474
571
  failures: [...failures],
572
+ failureDetails: [...failureDetails],
573
+ ...(note === undefined ? {} : { note }),
475
574
  cached: false,
476
575
  });
477
- return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
576
+ return { sources, failures, failureDetails, ...(note === undefined ? {} : { note }), ...(proxy === undefined ? {} : { proxy }) };
478
577
  }
package/lib/errors.d.ts CHANGED
@@ -8,7 +8,7 @@
8
8
  *
9
9
  * @module @fangchenghao/dsh-web-search/errors
10
10
  */
11
- import type { SearchErrorCode } from './types.ts';
11
+ import type { FailureKind, SearchErrorCode } from './types.ts';
12
12
  /** Failure raised by one engine, carrying its business code. */
13
13
  export declare class SearchError extends Error {
14
14
  /** Business code, used for cooldown decisions and for caller diagnostics. */
@@ -17,8 +17,22 @@ export declare class SearchError extends Error {
17
17
  readonly retryable: boolean;
18
18
  /** The service's own wait hint in milliseconds, when it sent one. */
19
19
  readonly retryAfterMs: number | undefined;
20
- constructor(message: string, code: SearchErrorCode, retryable?: boolean, retryAfterMs?: number);
20
+ /** HTTP status when the failure came from a response. */
21
+ readonly status: number | undefined;
22
+ constructor(message: string, code: SearchErrorCode, retryable?: boolean, retryAfterMs?: number, status?: number);
21
23
  }
24
+ /**
25
+ * Classify one engine failure into an actionable kind.
26
+ *
27
+ * The mapping is deliberately conservative: only a response status that says
28
+ * the credential or quota is the problem becomes `auth`/`quota`; an unexplained
29
+ * 403 is treated as a bot wall, because retrying the same request cannot fix a
30
+ * missing key but a wall often clears on its own.
31
+ *
32
+ * @param error - the classified engine failure.
33
+ * @returns the kind used by `fallbackOn` and diagnostics.
34
+ */
35
+ export declare function classifyFailure(error: SearchError): FailureKind;
22
36
  /**
23
37
  * Read the `Retry-After` header of a response.
24
38
  * @param value - raw header value: delta-seconds, or an HTTP date.
package/lib/errors.js CHANGED
@@ -6,14 +6,53 @@ export class SearchError extends Error {
6
6
  retryable;
7
7
  /** The service's own wait hint in milliseconds, when it sent one. */
8
8
  retryAfterMs;
9
- constructor(message, code, retryable = true, retryAfterMs) {
9
+ /** HTTP status when the failure came from a response. */
10
+ status;
11
+ constructor(message, code, retryable = true, retryAfterMs, status) {
10
12
  super(message);
11
13
  this.name = 'SearchError';
12
14
  this.code = code;
13
15
  this.retryable = retryable;
14
16
  this.retryAfterMs = retryAfterMs;
17
+ this.status = status;
15
18
  }
16
19
  }
20
+ /**
21
+ * Classify one engine failure into an actionable kind.
22
+ *
23
+ * The mapping is deliberately conservative: only a response status that says
24
+ * the credential or quota is the problem becomes `auth`/`quota`; an unexplained
25
+ * 403 is treated as a bot wall, because retrying the same request cannot fix a
26
+ * missing key but a wall often clears on its own.
27
+ *
28
+ * @param error - the classified engine failure.
29
+ * @returns the kind used by `fallbackOn` and diagnostics.
30
+ */
31
+ export function classifyFailure(error) {
32
+ if (error.code === 'SEARCH_THROTTLED' || error.code === 'SEARCH_TIMEOUT')
33
+ return 'transient';
34
+ if (error.code === 'SEARCH_NETWORK')
35
+ return 'transient';
36
+ if (error.code === 'SEARCH_PARSE' || error.code === 'SEARCH_EMPTY' || error.code === 'SEARCH_UNSUPPORTED') {
37
+ return 'invalid-response';
38
+ }
39
+ if (error.code === 'SEARCH_BLOCKED' || error.code === 'SEARCH_HTTP_STATUS') {
40
+ if (error.status === 401)
41
+ return 'auth';
42
+ if (error.status === 402)
43
+ return 'quota';
44
+ if (error.status === 429)
45
+ return 'transient';
46
+ if (error.status !== undefined && error.status >= 500)
47
+ return 'transient';
48
+ if (/\b(quota|credit|out of budget|limit exceeded)\b/iu.test(error.message))
49
+ return 'quota';
50
+ if (/\b(unauthori[sz]ed|authenticat(?:ion|ed)?|api[ _-]?key|credential)\b/iu.test(error.message))
51
+ return 'auth';
52
+ return 'bot-wall';
53
+ }
54
+ return 'unknown';
55
+ }
17
56
  /**
18
57
  * Read the `Retry-After` header of a response.
19
58
  * @param value - raw header value: delta-seconds, or an HTTP date.