@fffangchenghao/dsh-web-search 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +194 -33
- package/README.zh-CN.md +171 -34
- package/SECURITY.md +42 -7
- package/client.js +131 -9
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +16 -0
- package/lib/bridge.js +160 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +118 -19
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +304 -20
- package/lib/inject.d.ts +43 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { wrapUntrustedContent } from "./untrusted.js";
|
|
2
|
+
import { HOSTS } from "./engines.js";
|
|
3
|
+
/** The name this tool prefers. */
|
|
4
|
+
export const ENGINE_TEST_TOOL_NAME = 'engine_test';
|
|
5
|
+
/** The name used when another plugin already registered `engine_test`. */
|
|
6
|
+
export const ENGINE_TEST_FALLBACK_NAME = 'web_engine_test';
|
|
7
|
+
/** Default query; broad enough that every engine has an answer. */
|
|
8
|
+
export const ENGINE_TEST_DEFAULT_QUERY = 'DeepSeek Harness';
|
|
9
|
+
/** How many engines one run may ask at the same time. */
|
|
10
|
+
const ENGINE_TEST_CONCURRENCY = 3;
|
|
11
|
+
/** True for a value usable as a record. */
|
|
12
|
+
function isRecord(value) {
|
|
13
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
14
|
+
}
|
|
15
|
+
/** One short error line for the tool output. */
|
|
16
|
+
function failureReason(outcome) {
|
|
17
|
+
return outcome.failureDetails[0]?.reason ?? outcome.failures[0];
|
|
18
|
+
}
|
|
19
|
+
/** Run `worker` over items with at most `limit` in flight, preserving order. */
|
|
20
|
+
async function mapLimit(items, limit, worker) {
|
|
21
|
+
const results = new Array(items.length);
|
|
22
|
+
let next = 0;
|
|
23
|
+
const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
24
|
+
for (;;) {
|
|
25
|
+
const index = next;
|
|
26
|
+
next += 1;
|
|
27
|
+
if (index >= items.length)
|
|
28
|
+
return;
|
|
29
|
+
results[index] = await worker(items[index], index);
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
await Promise.all(runners);
|
|
33
|
+
return results;
|
|
34
|
+
}
|
|
35
|
+
/** The parameter schema, in the registry's raw JSON-Schema dialect. */
|
|
36
|
+
function parameterSchema() {
|
|
37
|
+
return {
|
|
38
|
+
type: 'object',
|
|
39
|
+
additionalProperties: false,
|
|
40
|
+
properties: {
|
|
41
|
+
engine: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
description: `Optional engine id to test alone, e.g. anysearch, exa, or ddg-html. Omit to test all ${HOSTS.length} engines.`,
|
|
44
|
+
},
|
|
45
|
+
query: {
|
|
46
|
+
type: 'string',
|
|
47
|
+
description: `Optional probe query; defaults to ${JSON.stringify(ENGINE_TEST_DEFAULT_QUERY)}.`,
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/** The output schema, in the same dialect. */
|
|
53
|
+
function outputSchema() {
|
|
54
|
+
return {
|
|
55
|
+
type: 'object',
|
|
56
|
+
additionalProperties: false,
|
|
57
|
+
properties: {
|
|
58
|
+
query: { type: 'string' },
|
|
59
|
+
ok: { type: 'integer' },
|
|
60
|
+
failed: { type: 'integer' },
|
|
61
|
+
results: {
|
|
62
|
+
type: 'array',
|
|
63
|
+
items: {
|
|
64
|
+
type: 'object',
|
|
65
|
+
additionalProperties: false,
|
|
66
|
+
properties: {
|
|
67
|
+
engine: { type: 'string' },
|
|
68
|
+
ok: { type: 'boolean' },
|
|
69
|
+
latencyMs: { type: 'integer' },
|
|
70
|
+
count: { type: 'integer' },
|
|
71
|
+
failureKind: { type: 'string' },
|
|
72
|
+
code: { type: 'string' },
|
|
73
|
+
status: { type: 'integer' },
|
|
74
|
+
reason: { type: 'string' },
|
|
75
|
+
sample: { type: 'string' },
|
|
76
|
+
},
|
|
77
|
+
required: ['engine', 'ok', 'latencyMs', 'count'],
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
required: ['query', 'ok', 'failed', 'results'],
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
/** Format the model-facing result text. */
|
|
85
|
+
export function formatEngineTestValue(value) {
|
|
86
|
+
const head = `Engine test (${value.query}): ${value.ok} OK, ${value.failed} failed.`;
|
|
87
|
+
const lines = value.results.map((result) => {
|
|
88
|
+
if (result.ok) {
|
|
89
|
+
const sample = result.sample === undefined ? '' : ` — ${result.sample}`;
|
|
90
|
+
return `- ${result.engine}: OK (${result.latencyMs}ms, ${result.count} result(s))${sample}`;
|
|
91
|
+
}
|
|
92
|
+
const kind = result.failureKind === undefined ? '' : ` [${result.failureKind}]`;
|
|
93
|
+
const reason = result.reason === undefined ? 'no result' : result.reason;
|
|
94
|
+
return `- ${result.engine}: FAIL${kind} (${result.latencyMs}ms) ${reason}`;
|
|
95
|
+
});
|
|
96
|
+
return [head, '', ...lines].join('\n');
|
|
97
|
+
}
|
|
98
|
+
/** Parse model arguments into a validated shape, throwing on nonsense. */
|
|
99
|
+
function parseArgs(args) {
|
|
100
|
+
if (!isRecord(args))
|
|
101
|
+
throw new Error('engine_test requires arguments');
|
|
102
|
+
const rawEngine = args['engine'];
|
|
103
|
+
let engine;
|
|
104
|
+
if (rawEngine !== undefined) {
|
|
105
|
+
if (typeof rawEngine !== 'string' || rawEngine.trim().length === 0) {
|
|
106
|
+
throw new Error('engine_test engine must be a non-empty string');
|
|
107
|
+
}
|
|
108
|
+
engine = rawEngine.trim();
|
|
109
|
+
if (!HOSTS.some((host) => host.id === engine)) {
|
|
110
|
+
throw new Error(`engine_test unknown engine ${JSON.stringify(engine)}; expected one of ${HOSTS.map((host) => host.id).join(', ')}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const rawQuery = args['query'];
|
|
114
|
+
let query = ENGINE_TEST_DEFAULT_QUERY;
|
|
115
|
+
if (rawQuery !== undefined) {
|
|
116
|
+
if (typeof rawQuery !== 'string' || rawQuery.trim().length === 0) {
|
|
117
|
+
throw new Error('engine_test query must be a non-empty string');
|
|
118
|
+
}
|
|
119
|
+
query = rawQuery.trim();
|
|
120
|
+
}
|
|
121
|
+
return { engine, query };
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Run a list of engines and assemble the tool value.
|
|
125
|
+
*
|
|
126
|
+
* Exported so the Settings bridge and the model tool share one implementation
|
|
127
|
+
* of latency, sample and failure reporting.
|
|
128
|
+
*
|
|
129
|
+
* @param runTest - tests one engine through the normal search path.
|
|
130
|
+
* @param engines - engine ids to test, in display order.
|
|
131
|
+
* @param query - probe query.
|
|
132
|
+
* @param signal - cancellation from the caller.
|
|
133
|
+
* @returns per-engine results plus counts.
|
|
134
|
+
*/
|
|
135
|
+
export async function runEngineTestValue(runTest, engines, query, signal) {
|
|
136
|
+
const results = await mapLimit(engines, ENGINE_TEST_CONCURRENCY, async (engine) => {
|
|
137
|
+
const started = Date.now();
|
|
138
|
+
try {
|
|
139
|
+
const outcome = await runTest(engine, query, signal);
|
|
140
|
+
const latencyMs = Date.now() - started;
|
|
141
|
+
if (outcome.sources.length > 0) {
|
|
142
|
+
const first = outcome.sources[0];
|
|
143
|
+
return {
|
|
144
|
+
engine,
|
|
145
|
+
ok: true,
|
|
146
|
+
latencyMs,
|
|
147
|
+
count: outcome.sources.length,
|
|
148
|
+
...(first.title === undefined || first.title.length === 0 ? {} : { sample: first.title }),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const detail = outcome.failureDetails[0];
|
|
152
|
+
const reason = failureReason(outcome) ?? 'no result';
|
|
153
|
+
return {
|
|
154
|
+
engine,
|
|
155
|
+
ok: false,
|
|
156
|
+
latencyMs,
|
|
157
|
+
count: 0,
|
|
158
|
+
...(detail?.kind === undefined ? {} : { failureKind: detail.kind }),
|
|
159
|
+
...(detail?.code === undefined ? {} : { code: detail.code }),
|
|
160
|
+
...(detail?.status === undefined ? {} : { status: detail.status }),
|
|
161
|
+
reason,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
return {
|
|
166
|
+
engine,
|
|
167
|
+
ok: false,
|
|
168
|
+
latencyMs: Date.now() - started,
|
|
169
|
+
count: 0,
|
|
170
|
+
failureKind: 'unknown',
|
|
171
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
const ok = results.filter((result) => result.ok).length;
|
|
176
|
+
return { query, results, ok, failed: results.length - ok };
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Build the tool definition the host registry accepts.
|
|
180
|
+
* @param name - `engine_test`, or the fallback when that name is taken.
|
|
181
|
+
* @param runTest - tests one engine through the normal search path.
|
|
182
|
+
* @returns a registry-ready definition.
|
|
183
|
+
*/
|
|
184
|
+
export function createEngineTestDefinition(name, runTest) {
|
|
185
|
+
return {
|
|
186
|
+
name,
|
|
187
|
+
description: 'Test the available search engines with one probe each and report which answered, their latency, and a failure class (quota, auth, bot-wall, transient, invalid-response) for the rest. Use it when search fails or is slow; it never changes the cache, fallback or cooldown state.',
|
|
188
|
+
parameters: parameterSchema(),
|
|
189
|
+
output: {
|
|
190
|
+
schema: outputSchema(),
|
|
191
|
+
render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatEngineTestValue(value)) }],
|
|
192
|
+
},
|
|
193
|
+
timeoutMs: 60_000,
|
|
194
|
+
isConcurrencySafe: () => true,
|
|
195
|
+
async execute(args, exec) {
|
|
196
|
+
const parsed = parseArgs(args);
|
|
197
|
+
const engines = parsed.engine === undefined ? HOSTS.map((host) => host.id) : [parsed.engine];
|
|
198
|
+
return runEngineTestValue(runTest, engines, parsed.query, exec.signal);
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Pick the tool name this deployment can accept.
|
|
204
|
+
* @param tools - the live registry.
|
|
205
|
+
* @returns the preferred or fallback name.
|
|
206
|
+
*/
|
|
207
|
+
export function pickEngineTestToolName(tools) {
|
|
208
|
+
if (typeof tools.get !== 'function')
|
|
209
|
+
return ENGINE_TEST_TOOL_NAME;
|
|
210
|
+
try {
|
|
211
|
+
return tools.get(ENGINE_TEST_TOOL_NAME) === undefined ? ENGINE_TEST_TOOL_NAME : ENGINE_TEST_FALLBACK_NAME;
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
return ENGINE_TEST_TOOL_NAME;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
218
|
+
export function warnEngineTest(message) {
|
|
219
|
+
try {
|
|
220
|
+
console.warn(`[dsh-web-search-bing] ${message}`);
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
// Logging must never break plugin loading.
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Register the tool and return its undo handle.
|
|
228
|
+
* @param tools - the host tool registry.
|
|
229
|
+
* @param runTest - tests one engine.
|
|
230
|
+
* @param options - diagnostics hook.
|
|
231
|
+
* @returns the registration, or `undefined` when the registry refused it.
|
|
232
|
+
*/
|
|
233
|
+
export function registerEngineTestTool(tools, runTest, options = {}) {
|
|
234
|
+
const warn = options.warn ?? warnEngineTest;
|
|
235
|
+
const name = pickEngineTestToolName(tools);
|
|
236
|
+
const definition = createEngineTestDefinition(name, runTest);
|
|
237
|
+
try {
|
|
238
|
+
const handle = tools.register(definition);
|
|
239
|
+
const unregister = typeof handle === 'function' ? handle : undefined;
|
|
240
|
+
return {
|
|
241
|
+
name,
|
|
242
|
+
dispose: () => {
|
|
243
|
+
try {
|
|
244
|
+
unregister?.();
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
warn(`engine_test unregister failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
248
|
+
}
|
|
249
|
+
},
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
warn(`engine_test registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
254
|
+
return undefined;
|
|
255
|
+
}
|
|
256
|
+
}
|
package/lib/engines.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module @fangchenghao/dsh-web-search/engines
|
|
13
13
|
*/
|
|
14
|
-
import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
|
|
14
|
+
import type { EngineAnswer, EngineFailure, FailureKind, HostSpec, RankOptions, Region, SafeSearch, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
|
|
15
15
|
import type { SearchCache } from './cache.ts';
|
|
16
16
|
import type { EngineHealth } from './health.ts';
|
|
17
17
|
/**
|
|
@@ -30,6 +30,12 @@ export interface EngineRunOptions extends SearchOptions {
|
|
|
30
30
|
anysearchDomain?: string;
|
|
31
31
|
/** Ranking knobs, falling back to the documented defaults. */
|
|
32
32
|
rank?: RankOptions;
|
|
33
|
+
/** Failure classes that may trigger the fallback tier; absent means all. */
|
|
34
|
+
fallbackOn?: readonly FailureKind[];
|
|
35
|
+
/** Domains the caller never wants in a result list. */
|
|
36
|
+
blockedDomains?: readonly string[];
|
|
37
|
+
/** Content-filter level forwarded to engines that support it. */
|
|
38
|
+
safeSearch?: SafeSearch;
|
|
33
39
|
/** Result cache to consult and fill, when one is configured. */
|
|
34
40
|
cache?: SearchCache;
|
|
35
41
|
/** Cooldown tracker, when one is configured. */
|
|
@@ -72,6 +78,10 @@ export interface SearchTrace {
|
|
|
72
78
|
sources: Source[];
|
|
73
79
|
/** Failure strings accumulated by this run. */
|
|
74
80
|
failures: string[];
|
|
81
|
+
/** Structured failures, including the class used by `fallbackOn`. */
|
|
82
|
+
failureDetails: EngineFailure[];
|
|
83
|
+
/** Human-readable note when policy changed the answer. */
|
|
84
|
+
note?: string;
|
|
75
85
|
/** Whether the result came from the cache. */
|
|
76
86
|
cached: boolean;
|
|
77
87
|
}
|
package/lib/engines.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { FAILURE_KINDS } from "./types.js";
|
|
2
|
+
import { SearchError, classifyFailure } from "./errors.js";
|
|
2
3
|
import { filterByTimeRange, preferWindow } from "./freshness.js";
|
|
3
4
|
import { resolveProxy } from "./proxy.js";
|
|
4
5
|
import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseExaMcp, parseFirecrawl, parseSo } from "./parsers.js";
|
|
@@ -185,6 +186,69 @@ const SOFT_DEADLINE = {
|
|
|
185
186
|
function leadsQuery(host, lead) {
|
|
186
187
|
return host.region === 'both' || host.region === lead;
|
|
187
188
|
}
|
|
189
|
+
/** Build the structured counterpart of a failure string. */
|
|
190
|
+
function failureDetail(id, reason, error) {
|
|
191
|
+
return {
|
|
192
|
+
id,
|
|
193
|
+
reason,
|
|
194
|
+
...(error === undefined ? {} : { code: error.code }),
|
|
195
|
+
kind: error === undefined ? 'unknown' : classifyFailure(error),
|
|
196
|
+
...(error?.status === undefined ? {} : { status: error.status }),
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Normalise a configured domain for suffix matching.
|
|
201
|
+
*
|
|
202
|
+
* Accepts a bare domain, a `*.domain` wildcard or a pasted URL, because all
|
|
203
|
+
* three are things a person reasonably types into a blocklist field.
|
|
204
|
+
*/
|
|
205
|
+
function normaliseDomain(value) {
|
|
206
|
+
let text = value.trim().toLowerCase();
|
|
207
|
+
if (text.includes('://')) {
|
|
208
|
+
try {
|
|
209
|
+
text = new URL(text).hostname;
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
// Fall through: the bare-domain cleanup below may still help.
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return text
|
|
216
|
+
.replace(/^\*\./u, '')
|
|
217
|
+
.replace(/^\.+/u, '')
|
|
218
|
+
.replace(/\.+$/u, '')
|
|
219
|
+
.replace(/\/.*$/u, '')
|
|
220
|
+
.trim();
|
|
221
|
+
}
|
|
222
|
+
/** Whether a URL's host is the configured domain or a subdomain of it. */
|
|
223
|
+
function isBlockedHost(hostname, blocked) {
|
|
224
|
+
const host = hostname.toLowerCase().replace(/\.$/u, '');
|
|
225
|
+
return blocked.some((domain) => host === domain || host.endsWith(`.${domain}`));
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Remove a caller-blocked domain from a result list.
|
|
229
|
+
*
|
|
230
|
+
* The filter is applied after fusion so it can never change the ranking of the
|
|
231
|
+
* rows that survive; it only removes rows the user explicitly excluded.
|
|
232
|
+
*/
|
|
233
|
+
function removeBlocked(sources, blocked) {
|
|
234
|
+
if (blocked.length === 0)
|
|
235
|
+
return [...sources];
|
|
236
|
+
return sources.filter((source) => {
|
|
237
|
+
try {
|
|
238
|
+
return !isBlockedHost(new URL(source.url).hostname, blocked);
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
/** Whether the primary failures permit a fallback under the configured policy. */
|
|
246
|
+
function allowsFallback(details, fallbackOn) {
|
|
247
|
+
if (fallbackOn === undefined)
|
|
248
|
+
return true;
|
|
249
|
+
const allowed = new Set(fallbackOn);
|
|
250
|
+
return details.every((detail) => allowed.has(detail.kind));
|
|
251
|
+
}
|
|
188
252
|
/**
|
|
189
253
|
* Ask one engine, rotating to the next key when the failure blames the key.
|
|
190
254
|
*
|
|
@@ -202,9 +266,9 @@ function leadsQuery(host, lead) {
|
|
|
202
266
|
* @param domain - AnySearch vertical domain, if configured.
|
|
203
267
|
* @returns the last attempt's outcome.
|
|
204
268
|
*/
|
|
205
|
-
async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain) {
|
|
269
|
+
async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain, safeSearch) {
|
|
206
270
|
if (host.id !== 'anysearch' || pool.size === 0) {
|
|
207
|
-
return searchHost(host, query, signal, timeoutMs, proxy, timeRange);
|
|
271
|
+
return searchHost(host, query, signal, timeoutMs, proxy, timeRange, safeSearch);
|
|
208
272
|
}
|
|
209
273
|
let key = pool.pick();
|
|
210
274
|
let last;
|
|
@@ -213,7 +277,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
|
|
|
213
277
|
anysearchApiKey: key,
|
|
214
278
|
...(domain === undefined ? {} : { anysearchDomain: domain }),
|
|
215
279
|
});
|
|
216
|
-
const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange);
|
|
280
|
+
const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange, safeSearch);
|
|
217
281
|
if (outcome.ok) {
|
|
218
282
|
pool.reward(key);
|
|
219
283
|
return outcome;
|
|
@@ -242,7 +306,7 @@ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRan
|
|
|
242
306
|
* @param health - cooldown tracker to update, when one is configured.
|
|
243
307
|
* @returns one answer per engine that produced sources.
|
|
244
308
|
*/
|
|
245
|
-
async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits, health, pool, anysearchDomain) {
|
|
309
|
+
async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails, proxy, limits, health, pool, anysearchDomain, safeSearch) {
|
|
246
310
|
const counters = limits.lead === 'all' || !hosts.some((host) => leadsQuery(host, limits.lead))
|
|
247
311
|
? () => true
|
|
248
312
|
: (host) => leadsQuery(host, limits.lead);
|
|
@@ -275,7 +339,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
|
|
|
275
339
|
releasedEarly = early;
|
|
276
340
|
released();
|
|
277
341
|
};
|
|
278
|
-
const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain).then((outcome) => {
|
|
342
|
+
const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain, safeSearch).then((outcome) => {
|
|
279
343
|
if (outcome.ok) {
|
|
280
344
|
rawCounts[index] = outcome.sources.length;
|
|
281
345
|
outcomes[index] = { host, sources: filterRelevant(outcome.sources, query) };
|
|
@@ -311,13 +375,18 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
|
|
|
311
375
|
for (const [index, outcome] of outcomes.entries()) {
|
|
312
376
|
const host = hosts[index];
|
|
313
377
|
if (outcome === undefined) {
|
|
314
|
-
|
|
378
|
+
const reason = releasedEarly ? `${host.id}: aborted once enough results had arrived` : `${host.id}: still running at deadline`;
|
|
379
|
+
failures.push(reason);
|
|
380
|
+
failureDetails.push(failureDetail(host.id, reason));
|
|
315
381
|
}
|
|
316
382
|
else if (outcome instanceof SearchError) {
|
|
317
383
|
failures.push(outcome.message);
|
|
384
|
+
failureDetails.push(failureDetail(host.id, outcome.message, outcome));
|
|
318
385
|
}
|
|
319
386
|
else if (outcome.sources.length === 0) {
|
|
320
|
-
|
|
387
|
+
const reason = `${host.id}: ${(rawCounts[index] ?? 0) > 0 ? 'no relevant results' : 'no results'}`;
|
|
388
|
+
failures.push(reason);
|
|
389
|
+
failureDetails.push(failureDetail(host.id, reason));
|
|
321
390
|
}
|
|
322
391
|
else {
|
|
323
392
|
answers.push(outcome);
|
|
@@ -388,16 +457,21 @@ export async function search(query, signal, options) {
|
|
|
388
457
|
// a throttled engine is still better than no search.
|
|
389
458
|
const now = Date.now();
|
|
390
459
|
const failures = [];
|
|
460
|
+
const failureDetails = [];
|
|
391
461
|
const rested = hosts.filter((host) => options.health?.isCooling(host.id, now) !== true);
|
|
392
462
|
if (rested.length > 0 && options.health !== undefined) {
|
|
393
463
|
for (const host of hosts) {
|
|
394
464
|
if (rested.includes(host))
|
|
395
465
|
continue;
|
|
396
|
-
const reason = options.health.skipped(host.id, now)
|
|
397
|
-
failures.push(
|
|
466
|
+
const reason = `${host.id}: ${options.health.skipped(host.id, now) ?? 'cooling down'}`;
|
|
467
|
+
failures.push(reason);
|
|
468
|
+
failureDetails.push(failureDetail(host.id, reason));
|
|
398
469
|
}
|
|
399
470
|
}
|
|
400
471
|
const usable = rested.length > 0 ? rested : hosts;
|
|
472
|
+
const blocked = (options.blockedDomains ?? []).map(normaliseDomain).filter((domain) => domain.length > 0);
|
|
473
|
+
const safeSearch = options.safeSearch ?? 'off';
|
|
474
|
+
const fallbackOn = options.fallbackOn;
|
|
401
475
|
const ascii = !CJK.test(query);
|
|
402
476
|
const lead = ascii ? 'global' : 'cn';
|
|
403
477
|
const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
|
|
@@ -428,9 +502,20 @@ export async function search(query, signal, options) {
|
|
|
428
502
|
// the cooldown filter: an engine entering or leaving a cooldown must not
|
|
429
503
|
// invalidate an entry that is still fresh, or every throttle would turn the
|
|
430
504
|
// next identical query into a cache miss.
|
|
431
|
-
const key = options.cache === undefined ? undefined : cacheKey({
|
|
505
|
+
const key = options.cache === undefined ? undefined : cacheKey({
|
|
506
|
+
query,
|
|
507
|
+
limit,
|
|
508
|
+
wait,
|
|
509
|
+
lead,
|
|
510
|
+
engines: hosts.map((host) => host.id),
|
|
511
|
+
timeRange,
|
|
512
|
+
...(safeSearch === 'off' ? {} : { safeSearch }),
|
|
513
|
+
...(fallbackOn === undefined || fallbackOn.length === FAILURE_KINDS.length ? {} : { fallbackOn }),
|
|
514
|
+
...(blocked.length === 0 ? {} : { blockedDomains: blocked }),
|
|
515
|
+
});
|
|
432
516
|
const cached = key === undefined ? undefined : options.cache?.get(key);
|
|
433
|
-
|
|
517
|
+
const cachedSources = cached === undefined ? undefined : removeBlocked(cached, blocked);
|
|
518
|
+
if (cachedSources !== undefined && cachedSources.length > 0) {
|
|
434
519
|
options.trace?.({
|
|
435
520
|
limit,
|
|
436
521
|
lead,
|
|
@@ -439,27 +524,39 @@ export async function search(query, signal, options) {
|
|
|
439
524
|
rank,
|
|
440
525
|
tier: 'cache',
|
|
441
526
|
answers: [],
|
|
442
|
-
sources:
|
|
527
|
+
sources: cachedSources,
|
|
443
528
|
failures: [...failures],
|
|
529
|
+
failureDetails: [...failureDetails],
|
|
444
530
|
cached: true,
|
|
445
531
|
});
|
|
446
|
-
return { sources:
|
|
532
|
+
return { sources: cachedSources, failures, failureDetails, ...(proxy === undefined ? {} : { proxy }) };
|
|
447
533
|
}
|
|
448
534
|
const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
|
|
449
535
|
const primary = ordered.filter((host) => host.tier === 'primary');
|
|
450
536
|
const fallback = ordered.filter((host) => host.tier === 'fallback');
|
|
451
537
|
let tier = 'none';
|
|
452
|
-
let
|
|
538
|
+
let note;
|
|
539
|
+
let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
|
|
453
540
|
if (answers.length > 0)
|
|
454
541
|
tier = 'primary';
|
|
455
542
|
let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
456
543
|
if (sources.length === 0 && fallback.length > 0) {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
544
|
+
if (allowsFallback(failureDetails, fallbackOn)) {
|
|
545
|
+
tier = 'fallback';
|
|
546
|
+
const kinds = [...new Set(failureDetails.map((detail) => detail.kind))];
|
|
547
|
+
answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, failureDetails, proxy, limits, options.health, pool, anysearchDomain, safeSearch));
|
|
548
|
+
sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
549
|
+
if (answers.length > 0 && kinds.length > 0)
|
|
550
|
+
note = `primary engines failed (${kinds.join(', ')}); using fallback.`;
|
|
551
|
+
}
|
|
552
|
+
else {
|
|
553
|
+
const blockedKind = failureDetails.find((detail) => !(fallbackOn ?? []).includes(detail.kind))?.kind ?? 'unknown';
|
|
554
|
+
note = `fallback skipped: ${blockedKind} is not enabled in fallbackOn.`;
|
|
555
|
+
}
|
|
460
556
|
}
|
|
461
557
|
if (signal?.aborted === true)
|
|
462
558
|
throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
|
|
559
|
+
sources = removeBlocked(sources, blocked);
|
|
463
560
|
if (key !== undefined && sources.length > 0)
|
|
464
561
|
options.cache?.set(key, sources);
|
|
465
562
|
options.trace?.({
|
|
@@ -472,7 +569,9 @@ export async function search(query, signal, options) {
|
|
|
472
569
|
answers,
|
|
473
570
|
sources,
|
|
474
571
|
failures: [...failures],
|
|
572
|
+
failureDetails: [...failureDetails],
|
|
573
|
+
...(note === undefined ? {} : { note }),
|
|
475
574
|
cached: false,
|
|
476
575
|
});
|
|
477
|
-
return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
576
|
+
return { sources, failures, failureDetails, ...(note === undefined ? {} : { note }), ...(proxy === undefined ? {} : { proxy }) };
|
|
478
577
|
}
|
package/lib/errors.d.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @module @fangchenghao/dsh-web-search/errors
|
|
10
10
|
*/
|
|
11
|
-
import type { SearchErrorCode } from './types.ts';
|
|
11
|
+
import type { FailureKind, SearchErrorCode } from './types.ts';
|
|
12
12
|
/** Failure raised by one engine, carrying its business code. */
|
|
13
13
|
export declare class SearchError extends Error {
|
|
14
14
|
/** Business code, used for cooldown decisions and for caller diagnostics. */
|
|
@@ -17,8 +17,22 @@ export declare class SearchError extends Error {
|
|
|
17
17
|
readonly retryable: boolean;
|
|
18
18
|
/** The service's own wait hint in milliseconds, when it sent one. */
|
|
19
19
|
readonly retryAfterMs: number | undefined;
|
|
20
|
-
|
|
20
|
+
/** HTTP status when the failure came from a response. */
|
|
21
|
+
readonly status: number | undefined;
|
|
22
|
+
constructor(message: string, code: SearchErrorCode, retryable?: boolean, retryAfterMs?: number, status?: number);
|
|
21
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Classify one engine failure into an actionable kind.
|
|
26
|
+
*
|
|
27
|
+
* The mapping is deliberately conservative: only a response status that says
|
|
28
|
+
* the credential or quota is the problem becomes `auth`/`quota`; an unexplained
|
|
29
|
+
* 403 is treated as a bot wall, because retrying the same request cannot fix a
|
|
30
|
+
* missing key but a wall often clears on its own.
|
|
31
|
+
*
|
|
32
|
+
* @param error - the classified engine failure.
|
|
33
|
+
* @returns the kind used by `fallbackOn` and diagnostics.
|
|
34
|
+
*/
|
|
35
|
+
export declare function classifyFailure(error: SearchError): FailureKind;
|
|
22
36
|
/**
|
|
23
37
|
* Read the `Retry-After` header of a response.
|
|
24
38
|
* @param value - raw header value: delta-seconds, or an HTTP date.
|
package/lib/errors.js
CHANGED
|
@@ -6,14 +6,53 @@ export class SearchError extends Error {
|
|
|
6
6
|
retryable;
|
|
7
7
|
/** The service's own wait hint in milliseconds, when it sent one. */
|
|
8
8
|
retryAfterMs;
|
|
9
|
-
|
|
9
|
+
/** HTTP status when the failure came from a response. */
|
|
10
|
+
status;
|
|
11
|
+
constructor(message, code, retryable = true, retryAfterMs, status) {
|
|
10
12
|
super(message);
|
|
11
13
|
this.name = 'SearchError';
|
|
12
14
|
this.code = code;
|
|
13
15
|
this.retryable = retryable;
|
|
14
16
|
this.retryAfterMs = retryAfterMs;
|
|
17
|
+
this.status = status;
|
|
15
18
|
}
|
|
16
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* Classify one engine failure into an actionable kind.
|
|
22
|
+
*
|
|
23
|
+
* The mapping is deliberately conservative: only a response status that says
|
|
24
|
+
* the credential or quota is the problem becomes `auth`/`quota`; an unexplained
|
|
25
|
+
* 403 is treated as a bot wall, because retrying the same request cannot fix a
|
|
26
|
+
* missing key but a wall often clears on its own.
|
|
27
|
+
*
|
|
28
|
+
* @param error - the classified engine failure.
|
|
29
|
+
* @returns the kind used by `fallbackOn` and diagnostics.
|
|
30
|
+
*/
|
|
31
|
+
export function classifyFailure(error) {
|
|
32
|
+
if (error.code === 'SEARCH_THROTTLED' || error.code === 'SEARCH_TIMEOUT')
|
|
33
|
+
return 'transient';
|
|
34
|
+
if (error.code === 'SEARCH_NETWORK')
|
|
35
|
+
return 'transient';
|
|
36
|
+
if (error.code === 'SEARCH_PARSE' || error.code === 'SEARCH_EMPTY' || error.code === 'SEARCH_UNSUPPORTED') {
|
|
37
|
+
return 'invalid-response';
|
|
38
|
+
}
|
|
39
|
+
if (error.code === 'SEARCH_BLOCKED' || error.code === 'SEARCH_HTTP_STATUS') {
|
|
40
|
+
if (error.status === 401)
|
|
41
|
+
return 'auth';
|
|
42
|
+
if (error.status === 402)
|
|
43
|
+
return 'quota';
|
|
44
|
+
if (error.status === 429)
|
|
45
|
+
return 'transient';
|
|
46
|
+
if (error.status !== undefined && error.status >= 500)
|
|
47
|
+
return 'transient';
|
|
48
|
+
if (/\b(quota|credit|out of budget|limit exceeded)\b/iu.test(error.message))
|
|
49
|
+
return 'quota';
|
|
50
|
+
if (/\b(unauthori[sz]ed|authenticat(?:ion|ed)?|api[ _-]?key|credential)\b/iu.test(error.message))
|
|
51
|
+
return 'auth';
|
|
52
|
+
return 'bot-wall';
|
|
53
|
+
}
|
|
54
|
+
return 'unknown';
|
|
55
|
+
}
|
|
17
56
|
/**
|
|
18
57
|
* Read the `Retry-After` header of a response.
|
|
19
58
|
* @param value - raw header value: delta-seconds, or an HTTP date.
|