@fffangchenghao/dsh-web-search 0.19.3 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +173 -1
- package/README.md +214 -34
- package/README.zh-CN.md +187 -35
- package/SECURITY.md +42 -7
- package/client.js +248 -10
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +34 -0
- package/lib/bridge.js +305 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +118 -19
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +309 -21
- package/lib/inject.d.ts +68 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
package/lib/index.js
CHANGED
|
@@ -18,6 +18,19 @@ import { registerDomainTool } from "./domain-tool.js";
|
|
|
18
18
|
import { registerSearchManyTool } from "./search-many.js";
|
|
19
19
|
import { registerReadPageTool } from "./read-page.js";
|
|
20
20
|
import { registerResearchTool } from "./research.js";
|
|
21
|
+
import { ResearchBodyCache, registerCacheSearchTool } from "./research-cache.js";
|
|
22
|
+
import { alternateOfficialUrl } from "./mirrors.js";
|
|
23
|
+
import { searchPlatforms } from "./platforms.js";
|
|
24
|
+
import { registerPlatformTool } from "./platform-tool.js";
|
|
25
|
+
import { registerEngineTestTool } from "./engine-test.js";
|
|
26
|
+
import { registerEngineTestBridge, registerUpdateBridge } from "./bridge.js";
|
|
27
|
+
import { registerAdvancedSearchTool, undatedWindowNote } from "./advanced-search.js";
|
|
28
|
+
import { searchVerticals } from "./verticals.js";
|
|
29
|
+
import { routePackageQuery } from "./query-router.js";
|
|
30
|
+
import { fetchPostgresSupport, isPostgresVersionQuery } from "./official-support.js";
|
|
31
|
+
import { chromeFeedNote, chromeWindow, fetchChromeReleases, filterChromeChannel, hasNonStableChromePosts, isChromeSecurityQuery, } from "./official-feed.js";
|
|
32
|
+
import { registerSearchPrompt } from "./system-prompt.js";
|
|
33
|
+
import { registerUnifiedTool } from "./tool-suite.js";
|
|
21
34
|
export const name = 'dsh-web-search';
|
|
22
35
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
23
36
|
export const inject = ['web'];
|
|
@@ -109,8 +122,9 @@ export function apply(ctx, config = {}) {
|
|
|
109
122
|
* without a restart. The configured limit is the hard ceiling: a caller may
|
|
110
123
|
* ask for fewer results, never for more.
|
|
111
124
|
*/
|
|
112
|
-
const runQuery = async (query, requested, signal) => {
|
|
125
|
+
const runQuery = async (query, requested, signal, overrides = {}) => {
|
|
113
126
|
const resolved = resolveConfig(section, undefined);
|
|
127
|
+
const timeRange = overrides.timeRange ?? resolved.timeRange;
|
|
114
128
|
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
115
129
|
throw new Error('web search requires a non-empty query');
|
|
116
130
|
}
|
|
@@ -120,10 +134,13 @@ export function apply(ctx, config = {}) {
|
|
|
120
134
|
? requested
|
|
121
135
|
: undefined;
|
|
122
136
|
const limit = Math.min(asked ?? resolved.limit, resolved.limit);
|
|
123
|
-
const
|
|
137
|
+
const outcome = await search(query, signal, {
|
|
124
138
|
limit,
|
|
125
139
|
wait: resolved.wait,
|
|
126
|
-
timeRange
|
|
140
|
+
timeRange,
|
|
141
|
+
blockedDomains: resolved.blockedDomains,
|
|
142
|
+
safeSearch: resolved.safeSearch,
|
|
143
|
+
fallbackOn: resolved.fallbackOn,
|
|
127
144
|
timeoutMs: resolved.timeoutMs,
|
|
128
145
|
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
129
146
|
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
@@ -135,11 +152,93 @@ export function apply(ctx, config = {}) {
|
|
|
135
152
|
...(signal === undefined ? {} : { signal }),
|
|
136
153
|
});
|
|
137
154
|
return {
|
|
138
|
-
sources,
|
|
139
|
-
failures,
|
|
155
|
+
sources: outcome.sources,
|
|
156
|
+
failures: outcome.failures,
|
|
157
|
+
failureDetails: outcome.failureDetails,
|
|
158
|
+
...(outcome.note === undefined ? {} : { note: outcome.note }),
|
|
140
159
|
truncated: asked !== undefined && asked > resolved.limit,
|
|
141
160
|
};
|
|
142
161
|
};
|
|
162
|
+
/**
|
|
163
|
+
* Official Chrome Releases path for a windowed security/release query.
|
|
164
|
+
*
|
|
165
|
+
* General search snippets for this blog often lose the post date; the feed
|
|
166
|
+
* carries it. Returning an undefined value here means "not this path", so the
|
|
167
|
+
* ordinary engine search remains the fallback when the feed is unavailable.
|
|
168
|
+
*/
|
|
169
|
+
const runChromeOfficial = async (query, requested, effective, signal) => {
|
|
170
|
+
if (effective === 'off' || !isChromeSecurityQuery(query))
|
|
171
|
+
return undefined;
|
|
172
|
+
const live = resolveConfig(section, undefined);
|
|
173
|
+
try {
|
|
174
|
+
const posts = await fetchChromeReleases({
|
|
175
|
+
timeoutMs: Math.min(live.domainTimeoutMs, 8_000),
|
|
176
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
177
|
+
...(signal === undefined ? {} : { signal }),
|
|
178
|
+
});
|
|
179
|
+
const windowed = chromeWindow(posts, effective);
|
|
180
|
+
const stableOnly = /\bstable\b|稳定版|稳定渠道/i.test(query);
|
|
181
|
+
const selected = filterChromeChannel(windowed, query);
|
|
182
|
+
// A stable-only feed with no stable post should not mask the ordinary
|
|
183
|
+
// engine path; fall back rather than returning an empty evidence pack.
|
|
184
|
+
if (selected.length === 0)
|
|
185
|
+
return undefined;
|
|
186
|
+
const budget = Math.min(requested ?? live.limit, live.limit);
|
|
187
|
+
const note = chromeFeedNote(selected, effective, query);
|
|
188
|
+
const channelNote = stableOnly && hasNonStableChromePosts(windowed, query)
|
|
189
|
+
? ` Stable-channel filter applied: ${windowed.length - selected.length} Beta/Dev/non-stable post(s) omitted.`
|
|
190
|
+
: '';
|
|
191
|
+
return {
|
|
192
|
+
sources: selected.slice(0, budget),
|
|
193
|
+
failures: [],
|
|
194
|
+
note: `${note}${channelNote}`,
|
|
195
|
+
truncated: selected.length > budget,
|
|
196
|
+
timeRange: effective,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
/**
|
|
204
|
+
* Run one per-call windowed search. `undefined` keeps the deployment setting;
|
|
205
|
+
* the tool's validation only lets the five known windows through.
|
|
206
|
+
*/
|
|
207
|
+
const runAdvancedSearch = async (query, requested, timeRange, signal) => {
|
|
208
|
+
const resolved = resolveConfig(section, undefined);
|
|
209
|
+
const effective = timeRange ?? resolved.timeRange;
|
|
210
|
+
const official = await runChromeOfficial(query, requested, effective, signal);
|
|
211
|
+
if (official !== undefined)
|
|
212
|
+
return official;
|
|
213
|
+
const outcome = await runQuery(query, requested, signal, { timeRange: effective });
|
|
214
|
+
const windowNote = undatedWindowNote(outcome.sources, effective);
|
|
215
|
+
const note = [outcome.note, windowNote]
|
|
216
|
+
.filter((part) => typeof part === 'string' && part.length > 0)
|
|
217
|
+
.join(' ');
|
|
218
|
+
return {
|
|
219
|
+
...outcome,
|
|
220
|
+
...(note.length > 0 ? { note } : {}),
|
|
221
|
+
timeRange: effective,
|
|
222
|
+
};
|
|
223
|
+
};
|
|
224
|
+
/**
|
|
225
|
+
* Run one engine for diagnostics. Deliberately bypasses the cache and the
|
|
226
|
+
* health tracker: a test must report the engine's current behaviour, not the
|
|
227
|
+
* state left by an earlier failure.
|
|
228
|
+
*/
|
|
229
|
+
const runEngineTest = async (engine, query, signal) => {
|
|
230
|
+
const live = resolveConfig(section, undefined);
|
|
231
|
+
return search(query, signal, {
|
|
232
|
+
limit: 1,
|
|
233
|
+
wait: 'all',
|
|
234
|
+
timeoutMs: Math.min(live.timeoutMs, 8_000),
|
|
235
|
+
safeSearch: live.safeSearch,
|
|
236
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
237
|
+
...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
|
|
238
|
+
...(live.anysearchDomain.length === 0 ? {} : { anysearchDomain: live.anysearchDomain }),
|
|
239
|
+
hostIds: [engine],
|
|
240
|
+
});
|
|
241
|
+
};
|
|
143
242
|
ctx.web.registerSearchProvider({
|
|
144
243
|
id: PROVIDER_ID,
|
|
145
244
|
available: () => true,
|
|
@@ -156,10 +255,27 @@ export function apply(ctx, config = {}) {
|
|
|
156
255
|
? request.maxResults
|
|
157
256
|
: undefined;
|
|
158
257
|
const outcome = await runQuery(query, asked, signal);
|
|
159
|
-
|
|
258
|
+
// A version/EOL question about PostgreSQL is better answered by the
|
|
259
|
+
// project's own support table than by whatever an engine ranks first.
|
|
260
|
+
// The lookup is query-scoped and failure-tolerant; it never replaces the
|
|
261
|
+
// ordinary rows when the page is unavailable.
|
|
262
|
+
let sources = outcome.sources;
|
|
263
|
+
if (isPostgresVersionQuery(query)) {
|
|
264
|
+
const live = resolveConfig(section, undefined);
|
|
265
|
+
const official = await fetchPostgresSupport({
|
|
266
|
+
timeoutMs: Math.min(live.timeoutMs, 8_000),
|
|
267
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
268
|
+
...(signal === undefined ? {} : { signal }),
|
|
269
|
+
});
|
|
270
|
+
if (official !== undefined) {
|
|
271
|
+
sources = [official, ...sources.filter((source) => source.url !== official.url)];
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (sources.length === 0) {
|
|
160
275
|
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
161
276
|
}
|
|
162
|
-
|
|
277
|
+
const cap = Math.min(asked ?? resolveConfig(section, undefined).limit, resolveConfig(section, undefined).limit);
|
|
278
|
+
return { sources: sources.slice(0, cap), truncated: outcome.truncated || sources.length > cap };
|
|
163
279
|
},
|
|
164
280
|
});
|
|
165
281
|
// The same tunnel the search side resolves, offered to `web_fetch`: a search
|
|
@@ -174,14 +290,80 @@ export function apply(ctx, config = {}) {
|
|
|
174
290
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
175
291
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
176
292
|
}));
|
|
177
|
-
// Optional half:
|
|
293
|
+
// Optional half: the eight capabilities exposed through `web_tools` (or the
|
|
294
|
+
// legacy names). Fail-safe by
|
|
178
295
|
// construction — a deployment without `ctx.tools` still loads this plugin and
|
|
179
296
|
// keeps `web_search`; the registration path is the host's
|
|
180
297
|
// `ToolRuntime.register(definition)`.
|
|
181
298
|
try {
|
|
182
299
|
const tools = readToolRegistry(ctx);
|
|
183
300
|
if (tools !== undefined) {
|
|
184
|
-
|
|
301
|
+
// In unified mode the eight definitions are built against a capture
|
|
302
|
+
// registry, then exposed through one `web_tools` entry. Legacy mode keeps
|
|
303
|
+
// the old names and their independent lifecycle.
|
|
304
|
+
const captured = [];
|
|
305
|
+
const capture = {
|
|
306
|
+
get: () => undefined,
|
|
307
|
+
register: (definition) => {
|
|
308
|
+
captured.push(definition);
|
|
309
|
+
return () => undefined;
|
|
310
|
+
},
|
|
311
|
+
};
|
|
312
|
+
const toolTarget = mounted.toolSurface === 'legacy'
|
|
313
|
+
? tools
|
|
314
|
+
: capture;
|
|
315
|
+
/**
|
|
316
|
+
* `search_many` is for general web queries, but an explicitly
|
|
317
|
+
* registry-scoped package query is much better answered by the existing
|
|
318
|
+
* domain source. It falls back to the ordinary path when that source
|
|
319
|
+
* returns nothing, so a registry outage cannot empty the batch.
|
|
320
|
+
*/
|
|
321
|
+
const runBatchQuery = async (query, limit, signal) => {
|
|
322
|
+
const route = routePackageQuery(query);
|
|
323
|
+
if (route !== undefined) {
|
|
324
|
+
const live = resolveConfig(section, undefined);
|
|
325
|
+
try {
|
|
326
|
+
const names = route.names.length > 0 ? route.names : [route.query];
|
|
327
|
+
const perName = Math.max(1, Math.ceil(limit / names.length));
|
|
328
|
+
const verticals = await Promise.all(names.map((name) => searchVerticals(name, {
|
|
329
|
+
ids: [route.id],
|
|
330
|
+
limit: perName,
|
|
331
|
+
timeoutMs: live.domainTimeoutMs,
|
|
332
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
333
|
+
...(signal === undefined ? {} : { signal }),
|
|
334
|
+
})));
|
|
335
|
+
const seen = new Set();
|
|
336
|
+
// Keep the exact first hit for every named package ahead of related
|
|
337
|
+
// suggestions: a two-package comparison should show both packages
|
|
338
|
+
// before serde-saphyr or react-router-style neighbours.
|
|
339
|
+
const ordered = [
|
|
340
|
+
...verticals.flatMap((vertical) => vertical.sources.slice(0, 1)),
|
|
341
|
+
...verticals.flatMap((vertical) => vertical.sources.slice(1)),
|
|
342
|
+
];
|
|
343
|
+
const sources = ordered
|
|
344
|
+
.filter((source) => {
|
|
345
|
+
if (seen.has(source.url))
|
|
346
|
+
return false;
|
|
347
|
+
seen.add(source.url);
|
|
348
|
+
return true;
|
|
349
|
+
})
|
|
350
|
+
.slice(0, limit);
|
|
351
|
+
if (sources.length > 0) {
|
|
352
|
+
return {
|
|
353
|
+
sources,
|
|
354
|
+
failures: verticals.flatMap((vertical) => vertical.failures),
|
|
355
|
+
note: `Package query routed to ${route.label} for ${names.join(', ')}; registry rows have stable package URLs and versions.`,
|
|
356
|
+
truncated: sources.length < verticals.reduce((sum, vertical) => sum + vertical.sources.length, 0),
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
catch {
|
|
361
|
+
// The ordinary web path below is the fallback for a registry miss.
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return runQuery(query, limit, signal);
|
|
365
|
+
};
|
|
366
|
+
registerDomainTool(toolTarget, () => {
|
|
185
367
|
const live = resolveConfig(section, undefined);
|
|
186
368
|
const settings = {
|
|
187
369
|
sourceIds: live.domainSourceIds,
|
|
@@ -191,16 +373,23 @@ export function apply(ctx, config = {}) {
|
|
|
191
373
|
};
|
|
192
374
|
return settings;
|
|
193
375
|
}, { warn: warnTool });
|
|
194
|
-
// Batch tool: same optional-registry path, same fail-safe mounting.
|
|
195
|
-
//
|
|
196
|
-
|
|
376
|
+
// Batch tool: same optional-registry path, same fail-safe mounting. Most
|
|
377
|
+
// queries use the `web_search` path; an explicit package-registry query
|
|
378
|
+
// can use the existing domain source instead.
|
|
379
|
+
const batch = registerSearchManyTool(toolTarget, runBatchQuery, { warn: warnTool });
|
|
197
380
|
if (batch !== undefined) {
|
|
198
381
|
onDispose(ctx, batch.dispose);
|
|
199
382
|
}
|
|
383
|
+
// Per-call date window: the only tool that can override the global
|
|
384
|
+
// `timeRange` without changing the user's Settings.
|
|
385
|
+
const advanced = registerAdvancedSearchTool(toolTarget, runAdvancedSearch, { warn: warnTool });
|
|
386
|
+
if (advanced !== undefined) {
|
|
387
|
+
onDispose(ctx, advanced.dispose);
|
|
388
|
+
}
|
|
200
389
|
// Resumable reading: `web_fetch` cannot continue from an offset because
|
|
201
390
|
// the host request carries only `url`, but a plugin-owned tool can. It
|
|
202
391
|
// reuses the same fetch ladder and the same live Settings readers.
|
|
203
|
-
const paged = registerReadPageTool(
|
|
392
|
+
const paged = registerReadPageTool(toolTarget, () => {
|
|
204
393
|
const live = resolveConfig(section, undefined);
|
|
205
394
|
return {
|
|
206
395
|
maxLength: live.fetchMaxLength,
|
|
@@ -223,30 +412,122 @@ export function apply(ctx, config = {}) {
|
|
|
223
412
|
id: 'http-proxy-research',
|
|
224
413
|
timeoutMs: mounted.timeoutMs,
|
|
225
414
|
format: 'markdown',
|
|
226
|
-
|
|
415
|
+
// The evidence pack keeps ~6 K characters, but the second retrieval
|
|
416
|
+
// (passage selection and cache_search) must be able to find a fact
|
|
417
|
+
// deep in the page. 200 K is the provider ceiling, not the model budget.
|
|
418
|
+
maxLength: 200_000,
|
|
227
419
|
renderJs: () => resolveConfig(section, undefined).renderJs,
|
|
228
420
|
renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
|
|
229
421
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
230
422
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
231
423
|
});
|
|
232
424
|
const researchFetch = async (url, signal) => {
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
425
|
+
const read = async (target) => {
|
|
426
|
+
const outcome = await researchProvider.fetch({ url: target }, signal);
|
|
427
|
+
return {
|
|
428
|
+
url: outcome.url,
|
|
429
|
+
status: outcome.statusCode,
|
|
430
|
+
text: outcome.body.kind === 'text' ? outcome.body.content : '',
|
|
431
|
+
truncated: outcome.truncated,
|
|
432
|
+
};
|
|
239
433
|
};
|
|
434
|
+
try {
|
|
435
|
+
const first = await read(url);
|
|
436
|
+
if (first.status < 400 && first.text.trim().length > 0)
|
|
437
|
+
return first;
|
|
438
|
+
const mirror = alternateOfficialUrl(url);
|
|
439
|
+
// The canonical standards page rejected the reader (or returned a
|
|
440
|
+
// shell); try the same organisation's public build once. A failure
|
|
441
|
+
// here falls back to the original outcome so the caller sees the real
|
|
442
|
+
// reason rather than a mirror-specific one.
|
|
443
|
+
if (mirror !== undefined)
|
|
444
|
+
return await read(mirror);
|
|
445
|
+
return first;
|
|
446
|
+
}
|
|
447
|
+
catch (error) {
|
|
448
|
+
const mirror = alternateOfficialUrl(url);
|
|
449
|
+
if (mirror !== undefined)
|
|
450
|
+
return await read(mirror);
|
|
451
|
+
throw error;
|
|
452
|
+
}
|
|
240
453
|
};
|
|
241
|
-
|
|
454
|
+
// Bodies fetched by research are kept in a bounded, expiring in-process
|
|
455
|
+
// cache. `cache_search` is the second retrieval: it searches the full
|
|
456
|
+
// text when the evidence pack only kept the relevant windows.
|
|
457
|
+
const researchCache = new ResearchBodyCache();
|
|
458
|
+
const research = registerResearchTool(toolTarget, (query, limit, signal) => runQuery(query, limit, signal), researchFetch, {
|
|
459
|
+
warn: warnTool,
|
|
460
|
+
onPage: (page) => researchCache.put(page),
|
|
461
|
+
});
|
|
242
462
|
if (research !== undefined) {
|
|
243
463
|
onDispose(ctx, research.dispose);
|
|
244
464
|
}
|
|
465
|
+
const cacheSearch = registerCacheSearchTool(toolTarget, researchCache, { warn: warnTool });
|
|
466
|
+
if (cacheSearch !== undefined) {
|
|
467
|
+
onDispose(ctx, () => researchCache.clear());
|
|
468
|
+
}
|
|
469
|
+
// Community/video search: direct keyless adapters first, the normal
|
|
470
|
+
// `web_search` path as a site-scoped fallback when an endpoint is blocked.
|
|
471
|
+
const platformRunner = async (query, ids, limit, signal) => {
|
|
472
|
+
const live = resolveConfig(section, undefined);
|
|
473
|
+
const outcome = await searchPlatforms(query, {
|
|
474
|
+
ids,
|
|
475
|
+
limit,
|
|
476
|
+
timeoutMs: Math.min(live.timeoutMs, 10_000),
|
|
477
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
478
|
+
...(signal === undefined ? {} : { signal }),
|
|
479
|
+
fallbackSearch: async (fallbackQuery, fallbackLimit, fallbackSignal) => {
|
|
480
|
+
const result = await runQuery(fallbackQuery, fallbackLimit, fallbackSignal);
|
|
481
|
+
return { sources: result.sources, failures: result.failures };
|
|
482
|
+
},
|
|
483
|
+
});
|
|
484
|
+
return {
|
|
485
|
+
sources: outcome.sources,
|
|
486
|
+
failures: outcome.failures,
|
|
487
|
+
via: outcome.via,
|
|
488
|
+
truncated: outcome.truncated,
|
|
489
|
+
};
|
|
490
|
+
};
|
|
491
|
+
const platform = registerPlatformTool(toolTarget, platformRunner, { warn: warnTool });
|
|
492
|
+
// Diagnostics: one tiny query per engine, no cache and no cooldown
|
|
493
|
+
// mutation. It is registered last because it is the least essential tool.
|
|
494
|
+
const engineTest = registerEngineTestTool(toolTarget, runEngineTest, { warn: warnTool });
|
|
495
|
+
if (engineTest !== undefined) {
|
|
496
|
+
onDispose(ctx, engineTest.dispose);
|
|
497
|
+
}
|
|
498
|
+
if (mounted.toolSurface !== 'legacy') {
|
|
499
|
+
// One entry point, eight actions. If the host rejects both candidate
|
|
500
|
+
// names, fall back to the legacy definitions rather than losing every
|
|
501
|
+
// optional capability.
|
|
502
|
+
const unified = registerUnifiedTool(tools, captured, { warn: warnTool });
|
|
503
|
+
if (unified === undefined && captured.length > 0) {
|
|
504
|
+
warnTool('web_tools unavailable; registering legacy tool names as a fallback');
|
|
505
|
+
for (const definition of captured) {
|
|
506
|
+
try {
|
|
507
|
+
tools.register(definition);
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
warnTool(`legacy fallback registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
245
515
|
}
|
|
246
516
|
}
|
|
247
517
|
catch (error) {
|
|
248
518
|
warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
249
519
|
}
|
|
520
|
+
// A short prompt section explains the relationships the tool schemas cannot:
|
|
521
|
+
// when to use advanced_search, and that the boundary tags are data, not
|
|
522
|
+
// instructions. Best-effort: a host without `systemPrompt` simply skips it.
|
|
523
|
+
registerSearchPrompt(ctx);
|
|
524
|
+
// The Settings page cannot call a model tool, so its "Test engines" button
|
|
525
|
+
// gets one narrow loopback endpoint. The model-facing tool above stays the
|
|
526
|
+
// primary path; this bridge is best-effort and silent on hosts without it.
|
|
527
|
+
registerEngineTestBridge(ctx, runEngineTest);
|
|
528
|
+
// The Settings page also gets a one-click update path; it uses the host
|
|
529
|
+
// plugin manager rather than bundling a second updater.
|
|
530
|
+
registerUpdateBridge(ctx);
|
|
250
531
|
}
|
|
251
532
|
export { Config, resolveConfig } from "./config.js";
|
|
252
533
|
/**
|
|
@@ -274,6 +555,13 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
|
|
|
274
555
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
275
556
|
export { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
276
557
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
|
558
|
+
export { ADVANCED_SEARCH_TOOL_NAME, ADVANCED_SEARCH_FALLBACK_NAME, ADVANCED_SEARCH_DEFAULT_LIMIT, ADVANCED_SEARCH_MAX_LIMIT, createAdvancedSearchDefinition, registerAdvancedSearchTool, pickAdvancedSearchToolName, formatAdvancedSearchValue, } from "./advanced-search.js";
|
|
559
|
+
export { SEARCH_PROMPT_SECTION, SEARCH_PROMPT_ORDER, SEARCH_PROMPT_TEXT, registerSearchPrompt, } from "./system-prompt.js";
|
|
560
|
+
export { UNTRUSTED_OPEN, UNTRUSTED_CLOSE, stripUntrustedBoundary, wrapUntrustedContent } from "./untrusted.js";
|
|
277
561
|
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
|
|
278
562
|
export { RESEARCH_TOOL_NAME, RESEARCH_TOOL_FALLBACK_NAME, RESEARCH_DEFAULT_SOURCES, RESEARCH_MAX_SOURCES, RESEARCH_MAX_QUERIES, RESEARCH_EXCERPT_CHARS, RESEARCH_BUDGET_MS, createResearchDefinition, registerResearchTool, pickResearchToolName, formatResearchValue, } from "./research.js";
|
|
563
|
+
export { PLATFORM_DEFAULT_LIMIT, PLATFORM_MAX_LIMIT, PLATFORM_TOOL_FALLBACK_NAME, PLATFORM_TOOL_NAME, createPlatformToolDefinition, parsePlatformArgs, pickPlatformToolName, registerPlatformTool, } from "./platform-tool.js";
|
|
564
|
+
export { DEFAULT_PLATFORM_LIMIT, MAX_PLATFORM_LIMIT, PLATFORM_IDS, PLATFORM_LABELS, isPlatformId, parseBilibili, parseJuejin, parseReddit, parseV2ex, parseYoutube, searchPlatforms, } from "./platforms.js";
|
|
565
|
+
export { CACHE_SEARCH_DEFAULT_LIMIT, CACHE_SEARCH_FALLBACK_NAME, CACHE_SEARCH_MAX_BODIES, CACHE_SEARCH_MAX_CHARS, CACHE_SEARCH_MAX_LIMIT, CACHE_SEARCH_SNIPPET_CHARS, CACHE_SEARCH_TTL_MS, CACHE_SEARCH_TOOL_NAME, ResearchBodyCache, cacheTokens, cacheWindows, createCacheSearchDefinition, formatCacheSearchValue, pickCacheSearchToolName, registerCacheSearchTool, } from "./research-cache.js";
|
|
566
|
+
export { alternateOfficialUrl, canonicalOfficialUrl } from "./mirrors.js";
|
|
279
567
|
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";
|
package/lib/inject.d.ts
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The optional Cordis services this plugin reaches through `ctx.inject`.
|
|
3
|
+
*
|
|
4
|
+
* None of these are hard dependencies: `web` is the only required service, and
|
|
5
|
+
* a deployment without a tool registry, web server or system-prompt service
|
|
6
|
+
* still mounts the search provider. The interfaces live in one module so the
|
|
7
|
+
* bridge and the prompt section agree on the injected shape, and so the plugin
|
|
8
|
+
* context can expose the hook without importing a feature module.
|
|
9
|
+
*
|
|
10
|
+
* @module @fangchenghao/dsh-web-search/inject
|
|
11
|
+
*/
|
|
12
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
13
|
+
/** The slice of the host web server the Settings bridge needs. */
|
|
14
|
+
export interface WebServerLike {
|
|
15
|
+
register(route: {
|
|
16
|
+
kind: 'exact';
|
|
17
|
+
path: string;
|
|
18
|
+
handler: (request: IncomingMessage, response: ServerResponse) => void | Promise<void>;
|
|
19
|
+
}): unknown;
|
|
20
|
+
}
|
|
21
|
+
/** One section the host's system prompt accepts. */
|
|
22
|
+
export interface SystemPromptSectionOptions {
|
|
23
|
+
/** Stable identifier used by the host to replace or remove the section. */
|
|
24
|
+
name: string;
|
|
25
|
+
/** Ordering hint; lower values appear earlier. */
|
|
26
|
+
order?: number;
|
|
27
|
+
/** The text inserted into the model's system prompt. */
|
|
28
|
+
text: string;
|
|
29
|
+
}
|
|
30
|
+
/** The slice of the host system-prompt service this plugin needs. */
|
|
31
|
+
export interface SystemPromptLike {
|
|
32
|
+
section(options: SystemPromptSectionOptions): unknown;
|
|
33
|
+
}
|
|
34
|
+
/** The slice of the host plugin manager the Settings update button needs. */
|
|
35
|
+
export interface PluginManagerLike {
|
|
36
|
+
/** Profile bundles, used to refuse a link:/file: install before replacing it. */
|
|
37
|
+
listBundles(): Promise<Array<{
|
|
38
|
+
name: string;
|
|
39
|
+
installed?: boolean;
|
|
40
|
+
source?: string;
|
|
41
|
+
version?: string;
|
|
42
|
+
}>>;
|
|
43
|
+
/** Install one exact bundle spec; the host persists it in the profile. */
|
|
44
|
+
installBundle(spec: string, options?: {
|
|
45
|
+
saveExact?: boolean;
|
|
46
|
+
}): Promise<{
|
|
47
|
+
changed: boolean;
|
|
48
|
+
application: 'applied' | 'restart-required' | 'overridden' | 'failed' | 'cancelled';
|
|
49
|
+
version?: string;
|
|
50
|
+
error?: {
|
|
51
|
+
code?: string;
|
|
52
|
+
diagnostic?: string;
|
|
53
|
+
};
|
|
54
|
+
}>;
|
|
55
|
+
}
|
|
56
|
+
/** Services that may arrive through one `ctx.inject` callback. */
|
|
57
|
+
export interface ServiceContext {
|
|
58
|
+
webServer?: WebServerLike;
|
|
59
|
+
systemPrompt?: SystemPromptLike;
|
|
60
|
+
pluginManager?: PluginManagerLike;
|
|
61
|
+
effect?: (execute: () => (() => void) | void) => unknown;
|
|
62
|
+
}
|
|
63
|
+
/** A context that may support Cordis's dynamic dependency injection. */
|
|
64
|
+
export interface InjectableHost {
|
|
65
|
+
inject?: (dependencies: string[], callback: (context: ServiceContext) => unknown) => unknown;
|
|
66
|
+
/** Optional service lookup for hosts that expose a manager outside inject. */
|
|
67
|
+
get?: (name: string) => unknown;
|
|
68
|
+
}
|
package/lib/inject.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/lib/markdown.js
CHANGED
|
@@ -403,6 +403,14 @@ function density(node) {
|
|
|
403
403
|
const characters = characterCount(node);
|
|
404
404
|
return characters === 0 ? 0 : characters / (1 + elementCount(node));
|
|
405
405
|
}
|
|
406
|
+
/** True when most of a container's text is one <pre> block (an appendix, not prose). */
|
|
407
|
+
function codeDominated(node) {
|
|
408
|
+
const pre = findElement(node, (child) => child.tag === 'pre');
|
|
409
|
+
if (pre === undefined)
|
|
410
|
+
return false;
|
|
411
|
+
const characters = characterCount(node);
|
|
412
|
+
return characters > 0 && characterCount(pre) >= characters * 0.6;
|
|
413
|
+
}
|
|
406
414
|
/** Highest-density container, favouring the one that carries the most text. */
|
|
407
415
|
function densestContainer(body) {
|
|
408
416
|
let best = body;
|
|
@@ -414,6 +422,11 @@ function densestContainer(body) {
|
|
|
414
422
|
const characters = characterCount(node);
|
|
415
423
|
if (characters < 120)
|
|
416
424
|
return;
|
|
425
|
+
// An appendix that is one huge ABNF/code block has a very high text/markup
|
|
426
|
+
// ratio, but it is not the page's prose. Skipping it lets the body or a
|
|
427
|
+
// prose section win; the body fallback still keeps the code.
|
|
428
|
+
if (codeDominated(node))
|
|
429
|
+
return;
|
|
417
430
|
const score = density(node);
|
|
418
431
|
if (score > bestScore || (score === bestScore && characters > bestCharacters)) {
|
|
419
432
|
best = node;
|
|
@@ -465,7 +478,11 @@ function pickMain(root) {
|
|
|
465
478
|
if (landmark !== undefined && characterCount(landmark) >= floor)
|
|
466
479
|
return landmark;
|
|
467
480
|
}
|
|
468
|
-
|
|
481
|
+
// A single section can be denser than the whole body on long documents with
|
|
482
|
+
// no <article>/<main> (RFC 9110 has dozens of sections). Choosing it makes
|
|
483
|
+
// the rest of the document unreachable; only trust density when it carries
|
|
484
|
+
// at least half the page's text.
|
|
485
|
+
if (characterCount(densest) >= floor && characterCount(densest) * 2 > bodyCharacters)
|
|
469
486
|
return densest;
|
|
470
487
|
// Density alone can crown a short, dense stub — a comment form or a footer —
|
|
471
488
|
// while the prose sits in a container with more markup and fewer characters
|
package/lib/mirrors.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Official mirrors for a few standards pages that block automated readers.
|
|
3
|
+
*
|
|
4
|
+
* This is not a scraper fallback hierarchy: it is a very small, explicit map
|
|
5
|
+
* from a canonical standards URL to the same organisation's public GitHub
|
|
6
|
+
* Pages copy. A mirror is only tried after the canonical URL failed or returned
|
|
7
|
+
* an empty body. The citation keeps the canonical W3C page; the body is the
|
|
8
|
+
* official mirror copy.
|
|
9
|
+
*
|
|
10
|
+
* @module @fangchenghao/dsh-web-search/mirrors
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Return an official mirror for a blocked standards URL, if one is known.
|
|
14
|
+
* @param raw - canonical URL.
|
|
15
|
+
* @returns the mirror URL, or `undefined` when no rule applies.
|
|
16
|
+
*/
|
|
17
|
+
export declare function alternateOfficialUrl(raw: string): string | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Map a mirror URL back to the canonical page that is safe to cite.
|
|
20
|
+
* @param raw - URL actually fetched.
|
|
21
|
+
* @returns the canonical URL, or the input when it is not a known mirror.
|
|
22
|
+
*/
|
|
23
|
+
export declare function canonicalOfficialUrl(raw: string): string;
|
package/lib/mirrors.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Official mirrors for a few standards pages that block automated readers.
|
|
3
|
+
*
|
|
4
|
+
* This is not a scraper fallback hierarchy: it is a very small, explicit map
|
|
5
|
+
* from a canonical standards URL to the same organisation's public GitHub
|
|
6
|
+
* Pages copy. A mirror is only tried after the canonical URL failed or returned
|
|
7
|
+
* an empty body. The citation keeps the canonical W3C page; the body is the
|
|
8
|
+
* official mirror copy.
|
|
9
|
+
*
|
|
10
|
+
* @module @fangchenghao/dsh-web-search/mirrors
|
|
11
|
+
*/
|
|
12
|
+
const RULES = [
|
|
13
|
+
{
|
|
14
|
+
// W3C's WCAG 2.2 Understanding pages are served from a GitHub Pages build
|
|
15
|
+
// of the same W3C repository when the main site rejects non-browser UAs.
|
|
16
|
+
pattern: /^https?:\/\/www\.w3\.org\/WAI\/WCAG22\/Understanding\/([^/?#]+\.html?)$/iu,
|
|
17
|
+
build: (match) => `https://w3c.github.io/wcag/understanding/${match[1]}`,
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
// The WAI overview page is blocked too; use the same build's index.
|
|
21
|
+
pattern: /^https?:\/\/www\.w3\.org\/WAI\/standards-guidelines\/wcag\/?$/iu,
|
|
22
|
+
build: () => 'https://w3c.github.io/wcag/',
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
// The TR landing page maps to the full guidelines build.
|
|
26
|
+
pattern: /^https?:\/\/www\.w3\.org\/TR\/WCAG22\/?$/iu,
|
|
27
|
+
build: () => 'https://w3c.github.io/wcag/guidelines/',
|
|
28
|
+
},
|
|
29
|
+
];
|
|
30
|
+
const CANONICAL_RULES = [
|
|
31
|
+
{
|
|
32
|
+
pattern: /^https?:\/\/w3c\.github\.io\/wcag\/understanding\/([^/?#]+\.html?)$/iu,
|
|
33
|
+
build: (match) => `https://www.w3.org/WAI/WCAG22/Understanding/${match[1]}`,
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
pattern: /^https?:\/\/w3c\.github\.io\/wcag\/guidelines\/?$/iu,
|
|
37
|
+
build: () => 'https://www.w3.org/TR/WCAG22/',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
pattern: /^https?:\/\/w3c\.github\.io\/wcag\/?$/iu,
|
|
41
|
+
build: () => 'https://www.w3.org/WAI/standards-guidelines/wcag/',
|
|
42
|
+
},
|
|
43
|
+
];
|
|
44
|
+
/**
|
|
45
|
+
* Return an official mirror for a blocked standards URL, if one is known.
|
|
46
|
+
* @param raw - canonical URL.
|
|
47
|
+
* @returns the mirror URL, or `undefined` when no rule applies.
|
|
48
|
+
*/
|
|
49
|
+
export function alternateOfficialUrl(raw) {
|
|
50
|
+
for (const rule of RULES) {
|
|
51
|
+
const match = rule.pattern.exec(raw);
|
|
52
|
+
if (match !== null)
|
|
53
|
+
return rule.build(match);
|
|
54
|
+
}
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Map a mirror URL back to the canonical page that is safe to cite.
|
|
59
|
+
* @param raw - URL actually fetched.
|
|
60
|
+
* @returns the canonical URL, or the input when it is not a known mirror.
|
|
61
|
+
*/
|
|
62
|
+
export function canonicalOfficialUrl(raw) {
|
|
63
|
+
for (const rule of CANONICAL_RULES) {
|
|
64
|
+
const match = rule.pattern.exec(raw);
|
|
65
|
+
if (match !== null)
|
|
66
|
+
return rule.build(match);
|
|
67
|
+
}
|
|
68
|
+
return raw;
|
|
69
|
+
}
|