@fffangchenghao/dsh-web-search 0.19.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +160 -0
  2. package/README.md +195 -34
  3. package/README.zh-CN.md +171 -34
  4. package/SECURITY.md +42 -7
  5. package/client.js +131 -9
  6. package/docs/store-evidence.md +10 -5
  7. package/lib/advanced-search.d.ts +107 -0
  8. package/lib/advanced-search.js +255 -0
  9. package/lib/bridge.d.ts +16 -0
  10. package/lib/bridge.js +160 -0
  11. package/lib/cache.d.ts +11 -4
  12. package/lib/cache.js +9 -5
  13. package/lib/config.d.ts +19 -3
  14. package/lib/config.js +24 -0
  15. package/lib/domain-tool.d.ts +5 -0
  16. package/lib/domain-tool.js +45 -6
  17. package/lib/engine-test.d.ts +116 -0
  18. package/lib/engine-test.js +256 -0
  19. package/lib/engines.d.ts +11 -1
  20. package/lib/engines.js +122 -20
  21. package/lib/errors.d.ts +16 -2
  22. package/lib/errors.js +40 -1
  23. package/lib/index.d.ts +24 -2
  24. package/lib/index.js +304 -20
  25. package/lib/inject.d.ts +43 -0
  26. package/lib/inject.js +1 -0
  27. package/lib/markdown.js +18 -1
  28. package/lib/mirrors.d.ts +23 -0
  29. package/lib/mirrors.js +69 -0
  30. package/lib/official-feed.d.ts +114 -0
  31. package/lib/official-feed.js +288 -0
  32. package/lib/official-support.d.ts +74 -0
  33. package/lib/official-support.js +134 -0
  34. package/lib/platform-tool.d.ts +77 -0
  35. package/lib/platform-tool.js +173 -0
  36. package/lib/platforms.d.ts +178 -0
  37. package/lib/platforms.js +1035 -0
  38. package/lib/query-router.d.ts +29 -0
  39. package/lib/query-router.js +58 -0
  40. package/lib/read-page.js +2 -1
  41. package/lib/research-cache.d.ts +115 -0
  42. package/lib/research-cache.js +275 -0
  43. package/lib/research.d.ts +45 -4
  44. package/lib/research.js +505 -22
  45. package/lib/search-many.d.ts +7 -1
  46. package/lib/search-many.js +7 -2
  47. package/lib/system-prompt.d.ts +27 -0
  48. package/lib/system-prompt.js +50 -0
  49. package/lib/tool-suite.d.ts +97 -0
  50. package/lib/tool-suite.js +224 -0
  51. package/lib/transport.d.ts +33 -2
  52. package/lib/transport.js +42 -9
  53. package/lib/types.d.ts +20 -0
  54. package/lib/types.js +9 -1
  55. package/lib/untrusted.d.ts +33 -0
  56. package/lib/untrusted.js +42 -0
  57. package/lib/verticals.d.ts +3 -1
  58. package/lib/verticals.js +26 -7
  59. package/package.json +5 -3
  60. package/skills/dsh-web-search/SKILL.md +21 -6
@@ -0,0 +1,256 @@
1
+ import { wrapUntrustedContent } from "./untrusted.js";
2
+ import { HOSTS } from "./engines.js";
3
+ /** The name this tool prefers. */
4
+ export const ENGINE_TEST_TOOL_NAME = 'engine_test';
5
+ /** The name used when another plugin already registered `engine_test`. */
6
+ export const ENGINE_TEST_FALLBACK_NAME = 'web_engine_test';
7
+ /** Default query; broad enough that every engine has an answer. */
8
+ export const ENGINE_TEST_DEFAULT_QUERY = 'DeepSeek Harness';
9
+ /** How many engines one run may ask at the same time. */
10
+ const ENGINE_TEST_CONCURRENCY = 3;
11
+ /** True for a value usable as a record. */
12
+ function isRecord(value) {
13
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
14
+ }
15
+ /** One short error line for the tool output. */
16
+ function failureReason(outcome) {
17
+ return outcome.failureDetails[0]?.reason ?? outcome.failures[0];
18
+ }
19
+ /** Run `worker` over items with at most `limit` in flight, preserving order. */
20
+ async function mapLimit(items, limit, worker) {
21
+ const results = new Array(items.length);
22
+ let next = 0;
23
+ const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
24
+ for (;;) {
25
+ const index = next;
26
+ next += 1;
27
+ if (index >= items.length)
28
+ return;
29
+ results[index] = await worker(items[index], index);
30
+ }
31
+ });
32
+ await Promise.all(runners);
33
+ return results;
34
+ }
35
+ /** The parameter schema, in the registry's raw JSON-Schema dialect. */
36
+ function parameterSchema() {
37
+ return {
38
+ type: 'object',
39
+ additionalProperties: false,
40
+ properties: {
41
+ engine: {
42
+ type: 'string',
43
+ description: `Optional engine id to test alone, e.g. anysearch, exa, or ddg-html. Omit to test all ${HOSTS.length} engines.`,
44
+ },
45
+ query: {
46
+ type: 'string',
47
+ description: `Optional probe query; defaults to ${JSON.stringify(ENGINE_TEST_DEFAULT_QUERY)}.`,
48
+ },
49
+ },
50
+ };
51
+ }
52
+ /** The output schema, in the same dialect. */
53
+ function outputSchema() {
54
+ return {
55
+ type: 'object',
56
+ additionalProperties: false,
57
+ properties: {
58
+ query: { type: 'string' },
59
+ ok: { type: 'integer' },
60
+ failed: { type: 'integer' },
61
+ results: {
62
+ type: 'array',
63
+ items: {
64
+ type: 'object',
65
+ additionalProperties: false,
66
+ properties: {
67
+ engine: { type: 'string' },
68
+ ok: { type: 'boolean' },
69
+ latencyMs: { type: 'integer' },
70
+ count: { type: 'integer' },
71
+ failureKind: { type: 'string' },
72
+ code: { type: 'string' },
73
+ status: { type: 'integer' },
74
+ reason: { type: 'string' },
75
+ sample: { type: 'string' },
76
+ },
77
+ required: ['engine', 'ok', 'latencyMs', 'count'],
78
+ },
79
+ },
80
+ },
81
+ required: ['query', 'ok', 'failed', 'results'],
82
+ };
83
+ }
84
+ /** Format the model-facing result text. */
85
+ export function formatEngineTestValue(value) {
86
+ const head = `Engine test (${value.query}): ${value.ok} OK, ${value.failed} failed.`;
87
+ const lines = value.results.map((result) => {
88
+ if (result.ok) {
89
+ const sample = result.sample === undefined ? '' : ` — ${result.sample}`;
90
+ return `- ${result.engine}: OK (${result.latencyMs}ms, ${result.count} result(s))${sample}`;
91
+ }
92
+ const kind = result.failureKind === undefined ? '' : ` [${result.failureKind}]`;
93
+ const reason = result.reason === undefined ? 'no result' : result.reason;
94
+ return `- ${result.engine}: FAIL${kind} (${result.latencyMs}ms) ${reason}`;
95
+ });
96
+ return [head, '', ...lines].join('\n');
97
+ }
98
+ /** Parse model arguments into a validated shape, throwing on nonsense. */
99
+ function parseArgs(args) {
100
+ if (!isRecord(args))
101
+ throw new Error('engine_test requires arguments');
102
+ const rawEngine = args['engine'];
103
+ let engine;
104
+ if (rawEngine !== undefined) {
105
+ if (typeof rawEngine !== 'string' || rawEngine.trim().length === 0) {
106
+ throw new Error('engine_test engine must be a non-empty string');
107
+ }
108
+ engine = rawEngine.trim();
109
+ if (!HOSTS.some((host) => host.id === engine)) {
110
+ throw new Error(`engine_test unknown engine ${JSON.stringify(engine)}; expected one of ${HOSTS.map((host) => host.id).join(', ')}`);
111
+ }
112
+ }
113
+ const rawQuery = args['query'];
114
+ let query = ENGINE_TEST_DEFAULT_QUERY;
115
+ if (rawQuery !== undefined) {
116
+ if (typeof rawQuery !== 'string' || rawQuery.trim().length === 0) {
117
+ throw new Error('engine_test query must be a non-empty string');
118
+ }
119
+ query = rawQuery.trim();
120
+ }
121
+ return { engine, query };
122
+ }
123
+ /**
124
+ * Run a list of engines and assemble the tool value.
125
+ *
126
+ * Exported so the Settings bridge and the model tool share one implementation
127
+ * of latency, sample and failure reporting.
128
+ *
129
+ * @param runTest - tests one engine through the normal search path.
130
+ * @param engines - engine ids to test, in display order.
131
+ * @param query - probe query.
132
+ * @param signal - cancellation from the caller.
133
+ * @returns per-engine results plus counts.
134
+ */
135
+ export async function runEngineTestValue(runTest, engines, query, signal) {
136
+ const results = await mapLimit(engines, ENGINE_TEST_CONCURRENCY, async (engine) => {
137
+ const started = Date.now();
138
+ try {
139
+ const outcome = await runTest(engine, query, signal);
140
+ const latencyMs = Date.now() - started;
141
+ if (outcome.sources.length > 0) {
142
+ const first = outcome.sources[0];
143
+ return {
144
+ engine,
145
+ ok: true,
146
+ latencyMs,
147
+ count: outcome.sources.length,
148
+ ...(first.title === undefined || first.title.length === 0 ? {} : { sample: first.title }),
149
+ };
150
+ }
151
+ const detail = outcome.failureDetails[0];
152
+ const reason = failureReason(outcome) ?? 'no result';
153
+ return {
154
+ engine,
155
+ ok: false,
156
+ latencyMs,
157
+ count: 0,
158
+ ...(detail?.kind === undefined ? {} : { failureKind: detail.kind }),
159
+ ...(detail?.code === undefined ? {} : { code: detail.code }),
160
+ ...(detail?.status === undefined ? {} : { status: detail.status }),
161
+ reason,
162
+ };
163
+ }
164
+ catch (error) {
165
+ return {
166
+ engine,
167
+ ok: false,
168
+ latencyMs: Date.now() - started,
169
+ count: 0,
170
+ failureKind: 'unknown',
171
+ reason: error instanceof Error ? error.message : String(error),
172
+ };
173
+ }
174
+ });
175
+ const ok = results.filter((result) => result.ok).length;
176
+ return { query, results, ok, failed: results.length - ok };
177
+ }
178
+ /**
179
+ * Build the tool definition the host registry accepts.
180
+ * @param name - `engine_test`, or the fallback when that name is taken.
181
+ * @param runTest - tests one engine through the normal search path.
182
+ * @returns a registry-ready definition.
183
+ */
184
+ export function createEngineTestDefinition(name, runTest) {
185
+ return {
186
+ name,
187
+ description: 'Test the available search engines with one probe each and report which answered, their latency, and a failure class (quota, auth, bot-wall, transient, invalid-response) for the rest. Use it when search fails or is slow; it never changes the cache, fallback or cooldown state.',
188
+ parameters: parameterSchema(),
189
+ output: {
190
+ schema: outputSchema(),
191
+ render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatEngineTestValue(value)) }],
192
+ },
193
+ timeoutMs: 60_000,
194
+ isConcurrencySafe: () => true,
195
+ async execute(args, exec) {
196
+ const parsed = parseArgs(args);
197
+ const engines = parsed.engine === undefined ? HOSTS.map((host) => host.id) : [parsed.engine];
198
+ return runEngineTestValue(runTest, engines, parsed.query, exec.signal);
199
+ },
200
+ };
201
+ }
202
+ /**
203
+ * Pick the tool name this deployment can accept.
204
+ * @param tools - the live registry.
205
+ * @returns the preferred or fallback name.
206
+ */
207
+ export function pickEngineTestToolName(tools) {
208
+ if (typeof tools.get !== 'function')
209
+ return ENGINE_TEST_TOOL_NAME;
210
+ try {
211
+ return tools.get(ENGINE_TEST_TOOL_NAME) === undefined ? ENGINE_TEST_TOOL_NAME : ENGINE_TEST_FALLBACK_NAME;
212
+ }
213
+ catch {
214
+ return ENGINE_TEST_TOOL_NAME;
215
+ }
216
+ }
217
+ /** Report a registration failure without throwing out of the plugin. */
218
+ export function warnEngineTest(message) {
219
+ try {
220
+ console.warn(`[dsh-web-search-bing] ${message}`);
221
+ }
222
+ catch {
223
+ // Logging must never break plugin loading.
224
+ }
225
+ }
226
+ /**
227
+ * Register the tool and return its undo handle.
228
+ * @param tools - the host tool registry.
229
+ * @param runTest - tests one engine.
230
+ * @param options - diagnostics hook.
231
+ * @returns the registration, or `undefined` when the registry refused it.
232
+ */
233
+ export function registerEngineTestTool(tools, runTest, options = {}) {
234
+ const warn = options.warn ?? warnEngineTest;
235
+ const name = pickEngineTestToolName(tools);
236
+ const definition = createEngineTestDefinition(name, runTest);
237
+ try {
238
+ const handle = tools.register(definition);
239
+ const unregister = typeof handle === 'function' ? handle : undefined;
240
+ return {
241
+ name,
242
+ dispose: () => {
243
+ try {
244
+ unregister?.();
245
+ }
246
+ catch (error) {
247
+ warn(`engine_test unregister failed: ${error instanceof Error ? error.message : String(error)}`);
248
+ }
249
+ },
250
+ };
251
+ }
252
+ catch (error) {
253
+ warn(`engine_test registration skipped: ${error instanceof Error ? error.message : String(error)}`);
254
+ return undefined;
255
+ }
256
+ }
package/lib/engines.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  *
12
12
  * @module @fangchenghao/dsh-web-search/engines
13
13
  */
14
- import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
14
+ import type { EngineAnswer, EngineFailure, FailureKind, HostSpec, RankOptions, Region, SafeSearch, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
15
15
  import type { SearchCache } from './cache.ts';
16
16
  import type { EngineHealth } from './health.ts';
17
17
  /**
@@ -30,6 +30,12 @@ export interface EngineRunOptions extends SearchOptions {
30
30
  anysearchDomain?: string;
31
31
  /** Ranking knobs, falling back to the documented defaults. */
32
32
  rank?: RankOptions;
33
+ /** Failure classes that may trigger the fallback tier; absent means all. */
34
+ fallbackOn?: readonly FailureKind[];
35
+ /** Domains the caller never wants in a result list. */
36
+ blockedDomains?: readonly string[];
37
+ /** Content-filter level forwarded to engines that support it. */
38
+ safeSearch?: SafeSearch;
33
39
  /** Result cache to consult and fill, when one is configured. */
34
40
  cache?: SearchCache;
35
41
  /** Cooldown tracker, when one is configured. */
@@ -72,6 +78,10 @@ export interface SearchTrace {
72
78
  sources: Source[];
73
79
  /** Failure strings accumulated by this run. */
74
80
  failures: string[];
81
+ /** Structured failures, including the class used by `fallbackOn`. */
82
+ failureDetails: EngineFailure[];
83
+ /** Human-readable note when policy changed the answer. */
84
+ note?: string;
75
85
  /** Whether the result came from the cache. */
76
86
  cached: boolean;
77
87
  }
package/lib/engines.js CHANGED
@@ -1,4 +1,5 @@
1
- import { SearchError } from "./errors.js";
1
+ import { FAILURE_KINDS } from "./types.js";
2
+ import { SearchError, classifyFailure } from "./errors.js";
2
3
  import { filterByTimeRange, preferWindow } from "./freshness.js";
3
4
  import { resolveProxy } from "./proxy.js";
4
5
  import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseExaMcp, parseFirecrawl, parseSo } from "./parsers.js";
@@ -78,7 +79,10 @@ export const HOSTS = [
78
79
  tier: 'fallback',
79
80
  region: 'both',
80
81
  weight: 1,
81
- viaProxy: true,
82
+ // Direct first: the tunnel's exit IP is rejected as a VPN by the keyless
83
+ // endpoint (`HTTP 403 … looks like a VPN or proxy`), while a direct request
84
+ // still answers. The transport still retries through the tunnel when the
85
+ // direct attempt fails, so deployments that need the tunnel lose nothing.
82
86
  method: 'POST',
83
87
  skipEcho: true,
84
88
  build: () => 'https://api.firecrawl.dev/v1/search',
@@ -182,6 +186,69 @@ const SOFT_DEADLINE = {
182
186
  function leadsQuery(host, lead) {
183
187
  return host.region === 'both' || host.region === lead;
184
188
  }
189
+ /** Build the structured counterpart of a failure string. */
190
+ function failureDetail(id, reason, error) {
191
+ return {
192
+ id,
193
+ reason,
194
+ ...(error === undefined ? {} : { code: error.code }),
195
+ kind: error === undefined ? 'unknown' : classifyFailure(error),
196
+ ...(error?.status === undefined ? {} : { status: error.status }),
197
+ };
198
+ }
199
+ /**
200
+ * Normalise a configured domain for suffix matching.
201
+ *
202
+ * Accepts a bare domain, a `*.domain` wildcard or a pasted URL, because all
203
+ * three are things a person reasonably types into a blocklist field.
204
+ */
205
+ function normaliseDomain(value) {
206
+ let text = value.trim().toLowerCase();
207
+ if (text.includes('://')) {
208
+ try {
209
+ text = new URL(text).hostname;
210
+ }
211
+ catch {
212
+ // Fall through: the bare-domain cleanup below may still help.
213
+ }
214
+ }
215
+ return text
216
+ .replace(/^\*\./u, '')
217
+ .replace(/^\.+/u, '')
218
+ .replace(/\.+$/u, '')
219
+ .replace(/\/.*$/u, '')
220
+ .trim();
221
+ }
222
+ /** Whether a URL's host is the configured domain or a subdomain of it. */
223
+ function isBlockedHost(hostname, blocked) {
224
+ const host = hostname.toLowerCase().replace(/\.$/u, '');
225
+ return blocked.some((domain) => host === domain || host.endsWith(`.${domain}`));
226
+ }
227
+ /**
228
+ * Remove a caller-blocked domain from a result list.
229
+ *
230
+ * The filter is applied after fusion so it can never change the ranking of the
231
+ * rows that survive; it only removes rows the user explicitly excluded.
232
+ */
233
+ function removeBlocked(sources, blocked) {
234
+ if (blocked.length === 0)
235
+ return [...sources];
236
+ return sources.filter((source) => {
237
+ try {
238
+ return !isBlockedHost(new URL(source.url).hostname, blocked);
239
+ }
240
+ catch {
241
+ return true;
242
+ }
243
+ });
244
+ }
245
+ /** Whether the primary failures permit a fallback under the configured policy. */
246
+ function allowsFallback(details, fallbackOn) {
247
+ if (fallbackOn === undefined)
248
+ return true;
249
+ const allowed = new Set(fallbackOn);
250
+ return details.every((detail) => allowed.has(detail.kind));
251
+ }
185
252
  /**
186
253
  * Ask one engine, rotating to the next key when the failure blames the key.
187
254
  *
@@ -199,9 +266,9 @@ function leadsQuery(host, lead) {
199
266
  * @param domain - AnySearch vertical domain, if configured.
200
267
  * @returns the last attempt's outcome.
201
268
  */
202
- async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain) {
269
+ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain, safeSearch) {
203
270
  if (host.id !== 'anysearch' || pool.size === 0) {
204
- return searchHost(host, query, signal, timeoutMs, proxy, timeRange);
271
+ return searchHost(host, query, signal, timeoutMs, proxy, timeRange, safeSearch);
205
272
  }
206
273
  let key = pool.pick();
207
274
  let last;
@@ -210,7 +277,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
210
277
  anysearchApiKey: key,
211
278
  ...(domain === undefined ? {} : { anysearchDomain: domain }),
212
279
  });
213
- const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange);
280
+ const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange, safeSearch);
214
281
  if (outcome.ok) {
215
282
  pool.reward(key);
216
283
  return outcome;
@@ -239,7 +306,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
239
306
  * @param health - cooldown tracker to update, when one is configured.
240
307
  * @returns one answer per engine that produced sources.
241
308
  */
242
- async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits, health, pool, anysearchDomain) {
309
+ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails, proxy, limits, health, pool, anysearchDomain, safeSearch) {
243
310
  const counters = limits.lead === 'all' || !hosts.some((host) => leadsQuery(host, limits.lead))
244
311
  ? () => true
245
312
  : (host) => leadsQuery(host, limits.lead);
@@ -272,7 +339,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
272
339
  releasedEarly = early;
273
340
  released();
274
341
  };
275
- const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain).then((outcome) => {
342
+ const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain, safeSearch).then((outcome) => {
276
343
  if (outcome.ok) {
277
344
  rawCounts[index] = outcome.sources.length;
278
345
  outcomes[index] = { host, sources: filterRelevant(outcome.sources, query) };
@@ -308,13 +375,18 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
308
375
  for (const [index, outcome] of outcomes.entries()) {
309
376
  const host = hosts[index];
310
377
  if (outcome === undefined) {
311
- failures.push(releasedEarly ? `${host.id}: aborted once enough results had arrived` : `${host.id}: still running at deadline`);
378
+ const reason = releasedEarly ? `${host.id}: aborted once enough results had arrived` : `${host.id}: still running at deadline`;
379
+ failures.push(reason);
380
+ failureDetails.push(failureDetail(host.id, reason));
312
381
  }
313
382
  else if (outcome instanceof SearchError) {
314
383
  failures.push(outcome.message);
384
+ failureDetails.push(failureDetail(host.id, outcome.message, outcome));
315
385
  }
316
386
  else if (outcome.sources.length === 0) {
317
- failures.push(`${host.id}: ${(rawCounts[index] ?? 0) > 0 ? 'no relevant results' : 'no results'}`);
387
+ const reason = `${host.id}: ${(rawCounts[index] ?? 0) > 0 ? 'no relevant results' : 'no results'}`;
388
+ failures.push(reason);
389
+ failureDetails.push(failureDetail(host.id, reason));
318
390
  }
319
391
  else {
320
392
  answers.push(outcome);
@@ -385,16 +457,21 @@ export async function search(query, signal, options) {
385
457
  // a throttled engine is still better than no search.
386
458
  const now = Date.now();
387
459
  const failures = [];
460
+ const failureDetails = [];
388
461
  const rested = hosts.filter((host) => options.health?.isCooling(host.id, now) !== true);
389
462
  if (rested.length > 0 && options.health !== undefined) {
390
463
  for (const host of hosts) {
391
464
  if (rested.includes(host))
392
465
  continue;
393
- const reason = options.health.skipped(host.id, now);
394
- failures.push(`${host.id}: ${reason ?? 'cooling down'}`);
466
+ const reason = `${host.id}: ${options.health.skipped(host.id, now) ?? 'cooling down'}`;
467
+ failures.push(reason);
468
+ failureDetails.push(failureDetail(host.id, reason));
395
469
  }
396
470
  }
397
471
  const usable = rested.length > 0 ? rested : hosts;
472
+ const blocked = (options.blockedDomains ?? []).map(normaliseDomain).filter((domain) => domain.length > 0);
473
+ const safeSearch = options.safeSearch ?? 'off';
474
+ const fallbackOn = options.fallbackOn;
398
475
  const ascii = !CJK.test(query);
399
476
  const lead = ascii ? 'global' : 'cn';
400
477
  const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
@@ -425,9 +502,20 @@ export async function search(query, signal, options) {
425
502
  // the cooldown filter: an engine entering or leaving a cooldown must not
426
503
  // invalidate an entry that is still fresh, or every throttle would turn the
427
504
  // next identical query into a cache miss.
428
- const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
505
+ const key = options.cache === undefined ? undefined : cacheKey({
506
+ query,
507
+ limit,
508
+ wait,
509
+ lead,
510
+ engines: hosts.map((host) => host.id),
511
+ timeRange,
512
+ ...(safeSearch === 'off' ? {} : { safeSearch }),
513
+ ...(fallbackOn === undefined || fallbackOn.length === FAILURE_KINDS.length ? {} : { fallbackOn }),
514
+ ...(blocked.length === 0 ? {} : { blockedDomains: blocked }),
515
+ });
429
516
  const cached = key === undefined ? undefined : options.cache?.get(key);
430
- if (cached !== undefined) {
517
+ const cachedSources = cached === undefined ? undefined : removeBlocked(cached, blocked);
518
+ if (cachedSources !== undefined && cachedSources.length > 0) {
431
519
  options.trace?.({
432
520
  limit,
433
521
  lead,
@@ -436,27 +524,39 @@ export async function search(query, signal, options) {
436
524
  rank,
437
525
  tier: 'cache',
438
526
  answers: [],
439
- sources: cached,
527
+ sources: cachedSources,
440
528
  failures: [...failures],
529
+ failureDetails: [...failureDetails],
441
530
  cached: true,
442
531
  });
443
- return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
532
+ return { sources: cachedSources, failures, failureDetails, ...(proxy === undefined ? {} : { proxy }) };
444
533
  }
445
534
  const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
446
535
  const primary = ordered.filter((host) => host.tier === 'primary');
447
536
  const fallback = ordered.filter((host) => host.tier === 'fallback');
448
537
  let tier = 'none';
449
- let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
538
+ let note;
539
+ let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
450
540
  if (answers.length > 0)
451
541
  tier = 'primary';
452
542
  let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
453
543
  if (sources.length === 0 && fallback.length > 0) {
454
- tier = 'fallback';
455
- answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
456
- sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
544
+ if (allowsFallback(failureDetails, fallbackOn)) {
545
+ tier = 'fallback';
546
+ const kinds = [...new Set(failureDetails.map((detail) => detail.kind))];
547
+ answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
548
+ sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
549
+ if (answers.length > 0 && kinds.length > 0)
550
+ note = `primary engines failed (${kinds.join(', ')}); using fallback.`;
551
+ }
552
+ else {
553
+ const blockedKind = failureDetails.find((detail) => !(fallbackOn ?? []).includes(detail.kind))?.kind ?? 'unknown';
554
+ note = `fallback skipped: ${blockedKind} is not enabled in fallbackOn.`;
555
+ }
457
556
  }
458
557
  if (signal?.aborted === true)
459
558
  throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
559
+ sources = removeBlocked(sources, blocked);
460
560
  if (key !== undefined && sources.length > 0)
461
561
  options.cache?.set(key, sources);
462
562
  options.trace?.({
@@ -469,7 +569,9 @@ export async function search(query, signal, options) {
469
569
  answers,
470
570
  sources,
471
571
  failures: [...failures],
572
+ failureDetails: [...failureDetails],
573
+ ...(note === undefined ? {} : { note }),
472
574
  cached: false,
473
575
  });
474
- return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
576
+ return { sources, failures, failureDetails, ...(note === undefined ? {} : { note }), ...(proxy === undefined ? {} : { proxy }) };
475
577
  }
package/lib/errors.d.ts CHANGED
@@ -8,7 +8,7 @@
8
8
  *
9
9
  * @module @fangchenghao/dsh-web-search/errors
10
10
  */
11
- import type { SearchErrorCode } from './types.ts';
11
+ import type { FailureKind, SearchErrorCode } from './types.ts';
12
12
  /** Failure raised by one engine, carrying its business code. */
13
13
  export declare class SearchError extends Error {
14
14
  /** Business code, used for cooldown decisions and for caller diagnostics. */
@@ -17,8 +17,22 @@ export declare class SearchError extends Error {
17
17
  readonly retryable: boolean;
18
18
  /** The service's own wait hint in milliseconds, when it sent one. */
19
19
  readonly retryAfterMs: number | undefined;
20
- constructor(message: string, code: SearchErrorCode, retryable?: boolean, retryAfterMs?: number);
20
+ /** HTTP status when the failure came from a response. */
21
+ readonly status: number | undefined;
22
+ constructor(message: string, code: SearchErrorCode, retryable?: boolean, retryAfterMs?: number, status?: number);
21
23
  }
24
+ /**
25
+ * Classify one engine failure into an actionable kind.
26
+ *
27
+ * The mapping is deliberately conservative: only a response status that says
28
+ * the credential or quota is the problem becomes `auth`/`quota`; an unexplained
29
+ * 403 is treated as a bot wall, because retrying the same request cannot fix a
30
+ * missing key but a wall often clears on its own.
31
+ *
32
+ * @param error - the classified engine failure.
33
+ * @returns the kind used by `fallbackOn` and diagnostics.
34
+ */
35
+ export declare function classifyFailure(error: SearchError): FailureKind;
22
36
  /**
23
37
  * Read the `Retry-After` header of a response.
24
38
  * @param value - raw header value: delta-seconds, or an HTTP date.
package/lib/errors.js CHANGED
@@ -6,14 +6,53 @@ export class SearchError extends Error {
6
6
  retryable;
7
7
  /** The service's own wait hint in milliseconds, when it sent one. */
8
8
  retryAfterMs;
9
- constructor(message, code, retryable = true, retryAfterMs) {
9
+ /** HTTP status when the failure came from a response. */
10
+ status;
11
+ constructor(message, code, retryable = true, retryAfterMs, status) {
10
12
  super(message);
11
13
  this.name = 'SearchError';
12
14
  this.code = code;
13
15
  this.retryable = retryable;
14
16
  this.retryAfterMs = retryAfterMs;
17
+ this.status = status;
15
18
  }
16
19
  }
20
+ /**
21
+ * Classify one engine failure into an actionable kind.
22
+ *
23
+ * The mapping is deliberately conservative: only a response status that says
24
+ * the credential or quota is the problem becomes `auth`/`quota`; an unexplained
25
+ * 403 is treated as a bot wall, because retrying the same request cannot fix a
26
+ * missing key but a wall often clears on its own.
27
+ *
28
+ * @param error - the classified engine failure.
29
+ * @returns the kind used by `fallbackOn` and diagnostics.
30
+ */
31
+ export function classifyFailure(error) {
32
+ if (error.code === 'SEARCH_THROTTLED' || error.code === 'SEARCH_TIMEOUT')
33
+ return 'transient';
34
+ if (error.code === 'SEARCH_NETWORK')
35
+ return 'transient';
36
+ if (error.code === 'SEARCH_PARSE' || error.code === 'SEARCH_EMPTY' || error.code === 'SEARCH_UNSUPPORTED') {
37
+ return 'invalid-response';
38
+ }
39
+ if (error.code === 'SEARCH_BLOCKED' || error.code === 'SEARCH_HTTP_STATUS') {
40
+ if (error.status === 401)
41
+ return 'auth';
42
+ if (error.status === 402)
43
+ return 'quota';
44
+ if (error.status === 429)
45
+ return 'transient';
46
+ if (error.status !== undefined && error.status >= 500)
47
+ return 'transient';
48
+ if (/\b(quota|credit|out of budget|limit exceeded)\b/iu.test(error.message))
49
+ return 'quota';
50
+ if (/\b(unauthori[sz]ed|authenticat(?:ion|ed)?|api[ _-]?key|credential)\b/iu.test(error.message))
51
+ return 'auth';
52
+ return 'bot-wall';
53
+ }
54
+ return 'unknown';
55
+ }
17
56
  /**
18
57
  * Read the `Retry-After` header of a response.
19
58
  * @param value - raw header value: delta-seconds, or an HTTP date.