@fffangchenghao/dsh-web-search 0.19.3 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +150 -0
- package/README.md +194 -33
- package/README.zh-CN.md +171 -34
- package/SECURITY.md +42 -7
- package/client.js +131 -9
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +16 -0
- package/lib/bridge.js +160 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +118 -19
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +304 -20
- package/lib/inject.d.ts +43 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
package/lib/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { PluginConfig } from './config.ts';
|
|
2
2
|
import type { FetchProvider } from './fetch.ts';
|
|
3
|
+
import type { InjectableHost } from './inject.ts';
|
|
3
4
|
import type { Source } from './types.ts';
|
|
4
5
|
export declare const name = "dsh-web-search";
|
|
5
6
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
@@ -46,6 +47,8 @@ export interface PluginContext {
|
|
|
46
47
|
*/
|
|
47
48
|
effect?: (execute: () => (() => void) | void) => unknown;
|
|
48
49
|
on?: (event: string, listener: () => void) => unknown;
|
|
50
|
+
/** Optional Cordis injection hook; used for the Settings bridge and prompt. */
|
|
51
|
+
inject?: NonNullable<InjectableHost['inject']>;
|
|
49
52
|
}
|
|
50
53
|
/** Configuration as the loader hands it over, already schema-validated. */
|
|
51
54
|
export type ApplyConfig = PluginConfig;
|
|
@@ -72,7 +75,7 @@ declare const _default: typeof apply & {
|
|
|
72
75
|
Config: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<NoInfer<{
|
|
73
76
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
74
77
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
75
|
-
fetchFormat: import("@deepseek-ai/schemastery").default<"
|
|
78
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"text" | "markdown", "text" | "markdown", "volatile-defined">;
|
|
76
79
|
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
77
80
|
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
78
81
|
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -85,6 +88,9 @@ declare const _default: typeof apply & {
|
|
|
85
88
|
freshnessBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
86
89
|
freshnessDays: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
87
90
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
91
|
+
blockedDomains: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
92
|
+
safeSearch: import("@deepseek-ai/schemastery").default<"off" | "moderate" | "strict", "off" | "moderate" | "strict", "volatile-defined">;
|
|
93
|
+
fallbackOn: import("@deepseek-ai/schemastery").default<NoInfer<("quota" | "auth" | "bot-wall" | "transient" | "invalid-response" | "unknown")[]>, NoInfer<("quota" | "auth" | "bot-wall" | "transient" | "invalid-response" | "unknown")[]>, "volatile-defined">;
|
|
88
94
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
89
95
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
90
96
|
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
@@ -96,10 +102,11 @@ declare const _default: typeof apply & {
|
|
|
96
102
|
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
97
103
|
domainLimit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
98
104
|
domainTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
105
|
+
toolSurface: import("@deepseek-ai/schemastery").default<"unified" | "legacy", "unified" | "legacy", "volatile-defined">;
|
|
99
106
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
100
107
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
101
108
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
102
|
-
fetchFormat: import("@deepseek-ai/schemastery").default<"
|
|
109
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"text" | "markdown", "text" | "markdown", "volatile-defined">;
|
|
103
110
|
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
104
111
|
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
105
112
|
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -112,6 +119,9 @@ declare const _default: typeof apply & {
|
|
|
112
119
|
freshnessBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
113
120
|
freshnessDays: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
114
121
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
122
|
+
blockedDomains: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
123
|
+
safeSearch: import("@deepseek-ai/schemastery").default<"off" | "moderate" | "strict", "off" | "moderate" | "strict", "volatile-defined">;
|
|
124
|
+
fallbackOn: import("@deepseek-ai/schemastery").default<NoInfer<("quota" | "auth" | "bot-wall" | "transient" | "invalid-response" | "unknown")[]>, NoInfer<("quota" | "auth" | "bot-wall" | "transient" | "invalid-response" | "unknown")[]>, "volatile-defined">;
|
|
115
125
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
116
126
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
117
127
|
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
@@ -123,6 +133,7 @@ declare const _default: typeof apply & {
|
|
|
123
133
|
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
124
134
|
domainLimit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
125
135
|
domainTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
136
|
+
toolSurface: import("@deepseek-ai/schemastery").default<"unified" | "legacy", "unified" | "legacy", "volatile-defined">;
|
|
126
137
|
}>>, "plain">;
|
|
127
138
|
inject: string[];
|
|
128
139
|
};
|
|
@@ -142,10 +153,21 @@ export { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, VERTICALS, searchVerticals, vertica
|
|
|
142
153
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
|
|
143
154
|
export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
|
|
144
155
|
export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
|
|
156
|
+
export { ADVANCED_SEARCH_TOOL_NAME, ADVANCED_SEARCH_FALLBACK_NAME, ADVANCED_SEARCH_DEFAULT_LIMIT, ADVANCED_SEARCH_MAX_LIMIT, createAdvancedSearchDefinition, registerAdvancedSearchTool, pickAdvancedSearchToolName, formatAdvancedSearchValue, } from './advanced-search.ts';
|
|
157
|
+
export type { AdvancedSearchDefinition, AdvancedSearchOutcome, AdvancedSearchRegistration, AdvancedSearchRegistryLike, AdvancedSearchRunner, AdvancedSearchValue, } from './advanced-search.ts';
|
|
158
|
+
export { SEARCH_PROMPT_SECTION, SEARCH_PROMPT_ORDER, SEARCH_PROMPT_TEXT, registerSearchPrompt, } from './system-prompt.ts';
|
|
159
|
+
export { UNTRUSTED_OPEN, UNTRUSTED_CLOSE, stripUntrustedBoundary, wrapUntrustedContent } from './untrusted.ts';
|
|
145
160
|
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
|
|
146
161
|
export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
|
|
147
162
|
export { RESEARCH_TOOL_NAME, RESEARCH_TOOL_FALLBACK_NAME, RESEARCH_DEFAULT_SOURCES, RESEARCH_MAX_SOURCES, RESEARCH_MAX_QUERIES, RESEARCH_EXCERPT_CHARS, RESEARCH_BUDGET_MS, createResearchDefinition, registerResearchTool, pickResearchToolName, formatResearchValue, } from './research.ts';
|
|
148
163
|
export type { ResearchDefinition, ResearchFetcher, ResearchFetchedPage, ResearchFetchFailure, ResearchPage, ResearchRegistration, ResearchRegistryLike, ResearchSearcher, ResearchSearchOutcome, ResearchValue, } from './research.ts';
|
|
164
|
+
export { PLATFORM_DEFAULT_LIMIT, PLATFORM_MAX_LIMIT, PLATFORM_TOOL_FALLBACK_NAME, PLATFORM_TOOL_NAME, createPlatformToolDefinition, parsePlatformArgs, pickPlatformToolName, registerPlatformTool, } from './platform-tool.ts';
|
|
165
|
+
export type { PlatformRunner, PlatformToolDefinition, PlatformToolRegistryLike, PlatformToolValue, } from './platform-tool.ts';
|
|
166
|
+
export { DEFAULT_PLATFORM_LIMIT, MAX_PLATFORM_LIMIT, PLATFORM_IDS, PLATFORM_LABELS, isPlatformId, parseBilibili, parseJuejin, parseReddit, parseV2ex, parseYoutube, searchPlatforms, } from './platforms.ts';
|
|
167
|
+
export type { PlatformId, PlatformOutcome, PlatformSearchOptions, } from './platforms.ts';
|
|
168
|
+
export { CACHE_SEARCH_DEFAULT_LIMIT, CACHE_SEARCH_FALLBACK_NAME, CACHE_SEARCH_MAX_BODIES, CACHE_SEARCH_MAX_CHARS, CACHE_SEARCH_MAX_LIMIT, CACHE_SEARCH_SNIPPET_CHARS, CACHE_SEARCH_TTL_MS, CACHE_SEARCH_TOOL_NAME, ResearchBodyCache, cacheTokens, cacheWindows, createCacheSearchDefinition, formatCacheSearchValue, pickCacheSearchToolName, registerCacheSearchTool, } from './research-cache.ts';
|
|
169
|
+
export type { CacheSearchDefinition, CacheSearchRegistryLike, CacheSearchValue, CachedResearchBody, CachedResearchMatch, ResearchBodyCacheOptions, } from './research-cache.ts';
|
|
170
|
+
export { alternateOfficialUrl, canonicalOfficialUrl } from './mirrors.ts';
|
|
149
171
|
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
|
|
150
172
|
export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
|
|
151
173
|
export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
|
package/lib/index.js
CHANGED
|
@@ -18,6 +18,19 @@ import { registerDomainTool } from "./domain-tool.js";
|
|
|
18
18
|
import { registerSearchManyTool } from "./search-many.js";
|
|
19
19
|
import { registerReadPageTool } from "./read-page.js";
|
|
20
20
|
import { registerResearchTool } from "./research.js";
|
|
21
|
+
import { ResearchBodyCache, registerCacheSearchTool } from "./research-cache.js";
|
|
22
|
+
import { alternateOfficialUrl } from "./mirrors.js";
|
|
23
|
+
import { searchPlatforms } from "./platforms.js";
|
|
24
|
+
import { registerPlatformTool } from "./platform-tool.js";
|
|
25
|
+
import { registerEngineTestTool } from "./engine-test.js";
|
|
26
|
+
import { registerEngineTestBridge } from "./bridge.js";
|
|
27
|
+
import { registerAdvancedSearchTool, undatedWindowNote } from "./advanced-search.js";
|
|
28
|
+
import { searchVerticals } from "./verticals.js";
|
|
29
|
+
import { routePackageQuery } from "./query-router.js";
|
|
30
|
+
import { fetchPostgresSupport, isPostgresVersionQuery } from "./official-support.js";
|
|
31
|
+
import { chromeFeedNote, chromeWindow, fetchChromeReleases, filterChromeChannel, hasNonStableChromePosts, isChromeSecurityQuery, } from "./official-feed.js";
|
|
32
|
+
import { registerSearchPrompt } from "./system-prompt.js";
|
|
33
|
+
import { registerUnifiedTool } from "./tool-suite.js";
|
|
21
34
|
export const name = 'dsh-web-search';
|
|
22
35
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
23
36
|
export const inject = ['web'];
|
|
@@ -109,8 +122,9 @@ export function apply(ctx, config = {}) {
|
|
|
109
122
|
* without a restart. The configured limit is the hard ceiling: a caller may
|
|
110
123
|
* ask for fewer results, never for more.
|
|
111
124
|
*/
|
|
112
|
-
const runQuery = async (query, requested, signal) => {
|
|
125
|
+
const runQuery = async (query, requested, signal, overrides = {}) => {
|
|
113
126
|
const resolved = resolveConfig(section, undefined);
|
|
127
|
+
const timeRange = overrides.timeRange ?? resolved.timeRange;
|
|
114
128
|
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
115
129
|
throw new Error('web search requires a non-empty query');
|
|
116
130
|
}
|
|
@@ -120,10 +134,13 @@ export function apply(ctx, config = {}) {
|
|
|
120
134
|
? requested
|
|
121
135
|
: undefined;
|
|
122
136
|
const limit = Math.min(asked ?? resolved.limit, resolved.limit);
|
|
123
|
-
const
|
|
137
|
+
const outcome = await search(query, signal, {
|
|
124
138
|
limit,
|
|
125
139
|
wait: resolved.wait,
|
|
126
|
-
timeRange
|
|
140
|
+
timeRange,
|
|
141
|
+
blockedDomains: resolved.blockedDomains,
|
|
142
|
+
safeSearch: resolved.safeSearch,
|
|
143
|
+
fallbackOn: resolved.fallbackOn,
|
|
127
144
|
timeoutMs: resolved.timeoutMs,
|
|
128
145
|
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
129
146
|
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
@@ -135,11 +152,93 @@ export function apply(ctx, config = {}) {
|
|
|
135
152
|
...(signal === undefined ? {} : { signal }),
|
|
136
153
|
});
|
|
137
154
|
return {
|
|
138
|
-
sources,
|
|
139
|
-
failures,
|
|
155
|
+
sources: outcome.sources,
|
|
156
|
+
failures: outcome.failures,
|
|
157
|
+
failureDetails: outcome.failureDetails,
|
|
158
|
+
...(outcome.note === undefined ? {} : { note: outcome.note }),
|
|
140
159
|
truncated: asked !== undefined && asked > resolved.limit,
|
|
141
160
|
};
|
|
142
161
|
};
|
|
162
|
+
/**
|
|
163
|
+
* Official Chrome Releases path for a windowed security/release query.
|
|
164
|
+
*
|
|
165
|
+
* General search snippets for this blog often lose the post date; the feed
|
|
166
|
+
* carries it. Returning an undefined value here means "not this path", so the
|
|
167
|
+
* ordinary engine search remains the fallback when the feed is unavailable.
|
|
168
|
+
*/
|
|
169
|
+
const runChromeOfficial = async (query, requested, effective, signal) => {
|
|
170
|
+
if (effective === 'off' || !isChromeSecurityQuery(query))
|
|
171
|
+
return undefined;
|
|
172
|
+
const live = resolveConfig(section, undefined);
|
|
173
|
+
try {
|
|
174
|
+
const posts = await fetchChromeReleases({
|
|
175
|
+
timeoutMs: Math.min(live.domainTimeoutMs, 8_000),
|
|
176
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
177
|
+
...(signal === undefined ? {} : { signal }),
|
|
178
|
+
});
|
|
179
|
+
const windowed = chromeWindow(posts, effective);
|
|
180
|
+
const stableOnly = /\bstable\b|稳定版|稳定渠道/i.test(query);
|
|
181
|
+
const selected = filterChromeChannel(windowed, query);
|
|
182
|
+
// A stable-only feed with no stable post should not mask the ordinary
|
|
183
|
+
// engine path; fall back rather than returning an empty evidence pack.
|
|
184
|
+
if (selected.length === 0)
|
|
185
|
+
return undefined;
|
|
186
|
+
const budget = Math.min(requested ?? live.limit, live.limit);
|
|
187
|
+
const note = chromeFeedNote(selected, effective, query);
|
|
188
|
+
const channelNote = stableOnly && hasNonStableChromePosts(windowed, query)
|
|
189
|
+
? ` Stable-channel filter applied: ${windowed.length - selected.length} Beta/Dev/non-stable post(s) omitted.`
|
|
190
|
+
: '';
|
|
191
|
+
return {
|
|
192
|
+
sources: selected.slice(0, budget),
|
|
193
|
+
failures: [],
|
|
194
|
+
note: `${note}${channelNote}`,
|
|
195
|
+
truncated: selected.length > budget,
|
|
196
|
+
timeRange: effective,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
/**
|
|
204
|
+
* Run one per-call windowed search. `undefined` keeps the deployment setting;
|
|
205
|
+
* the tool's validation only lets the five known windows through.
|
|
206
|
+
*/
|
|
207
|
+
const runAdvancedSearch = async (query, requested, timeRange, signal) => {
|
|
208
|
+
const resolved = resolveConfig(section, undefined);
|
|
209
|
+
const effective = timeRange ?? resolved.timeRange;
|
|
210
|
+
const official = await runChromeOfficial(query, requested, effective, signal);
|
|
211
|
+
if (official !== undefined)
|
|
212
|
+
return official;
|
|
213
|
+
const outcome = await runQuery(query, requested, signal, { timeRange: effective });
|
|
214
|
+
const windowNote = undatedWindowNote(outcome.sources, effective);
|
|
215
|
+
const note = [outcome.note, windowNote]
|
|
216
|
+
.filter((part) => typeof part === 'string' && part.length > 0)
|
|
217
|
+
.join(' ');
|
|
218
|
+
return {
|
|
219
|
+
...outcome,
|
|
220
|
+
...(note.length > 0 ? { note } : {}),
|
|
221
|
+
timeRange: effective,
|
|
222
|
+
};
|
|
223
|
+
};
|
|
224
|
+
/**
|
|
225
|
+
* Run one engine for diagnostics. Deliberately bypasses the cache and the
|
|
226
|
+
* health tracker: a test must report the engine's current behaviour, not the
|
|
227
|
+
* state left by an earlier failure.
|
|
228
|
+
*/
|
|
229
|
+
const runEngineTest = async (engine, query, signal) => {
|
|
230
|
+
const live = resolveConfig(section, undefined);
|
|
231
|
+
return search(query, signal, {
|
|
232
|
+
limit: 1,
|
|
233
|
+
wait: 'all',
|
|
234
|
+
timeoutMs: Math.min(live.timeoutMs, 8_000),
|
|
235
|
+
safeSearch: live.safeSearch,
|
|
236
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
237
|
+
...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
|
|
238
|
+
...(live.anysearchDomain.length === 0 ? {} : { anysearchDomain: live.anysearchDomain }),
|
|
239
|
+
hostIds: [engine],
|
|
240
|
+
});
|
|
241
|
+
};
|
|
143
242
|
ctx.web.registerSearchProvider({
|
|
144
243
|
id: PROVIDER_ID,
|
|
145
244
|
available: () => true,
|
|
@@ -156,10 +255,27 @@ export function apply(ctx, config = {}) {
|
|
|
156
255
|
? request.maxResults
|
|
157
256
|
: undefined;
|
|
158
257
|
const outcome = await runQuery(query, asked, signal);
|
|
159
|
-
|
|
258
|
+
// A version/EOL question about PostgreSQL is better answered by the
|
|
259
|
+
// project's own support table than by whatever an engine ranks first.
|
|
260
|
+
// The lookup is query-scoped and failure-tolerant; it never replaces the
|
|
261
|
+
// ordinary rows when the page is unavailable.
|
|
262
|
+
let sources = outcome.sources;
|
|
263
|
+
if (isPostgresVersionQuery(query)) {
|
|
264
|
+
const live = resolveConfig(section, undefined);
|
|
265
|
+
const official = await fetchPostgresSupport({
|
|
266
|
+
timeoutMs: Math.min(live.timeoutMs, 8_000),
|
|
267
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
268
|
+
...(signal === undefined ? {} : { signal }),
|
|
269
|
+
});
|
|
270
|
+
if (official !== undefined) {
|
|
271
|
+
sources = [official, ...sources.filter((source) => source.url !== official.url)];
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (sources.length === 0) {
|
|
160
275
|
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
161
276
|
}
|
|
162
|
-
|
|
277
|
+
const cap = Math.min(asked ?? resolveConfig(section, undefined).limit, resolveConfig(section, undefined).limit);
|
|
278
|
+
return { sources: sources.slice(0, cap), truncated: outcome.truncated || sources.length > cap };
|
|
163
279
|
},
|
|
164
280
|
});
|
|
165
281
|
// The same tunnel the search side resolves, offered to `web_fetch`: a search
|
|
@@ -181,7 +297,72 @@ export function apply(ctx, config = {}) {
|
|
|
181
297
|
try {
|
|
182
298
|
const tools = readToolRegistry(ctx);
|
|
183
299
|
if (tools !== undefined) {
|
|
184
|
-
|
|
300
|
+
// In unified mode the eight definitions are built against a capture
|
|
301
|
+
// registry, then exposed through one `web_tools` entry. Legacy mode keeps
|
|
302
|
+
// the old names and their independent lifecycle.
|
|
303
|
+
const captured = [];
|
|
304
|
+
const capture = {
|
|
305
|
+
get: () => undefined,
|
|
306
|
+
register: (definition) => {
|
|
307
|
+
captured.push(definition);
|
|
308
|
+
return () => undefined;
|
|
309
|
+
},
|
|
310
|
+
};
|
|
311
|
+
const toolTarget = mounted.toolSurface === 'legacy'
|
|
312
|
+
? tools
|
|
313
|
+
: capture;
|
|
314
|
+
/**
|
|
315
|
+
* `search_many` is for general web queries, but an explicitly
|
|
316
|
+
* registry-scoped package query is much better answered by the existing
|
|
317
|
+
* domain source. It falls back to the ordinary path when that source
|
|
318
|
+
* returns nothing, so a registry outage cannot empty the batch.
|
|
319
|
+
*/
|
|
320
|
+
const runBatchQuery = async (query, limit, signal) => {
|
|
321
|
+
const route = routePackageQuery(query);
|
|
322
|
+
if (route !== undefined) {
|
|
323
|
+
const live = resolveConfig(section, undefined);
|
|
324
|
+
try {
|
|
325
|
+
const names = route.names.length > 0 ? route.names : [route.query];
|
|
326
|
+
const perName = Math.max(1, Math.ceil(limit / names.length));
|
|
327
|
+
const verticals = await Promise.all(names.map((name) => searchVerticals(name, {
|
|
328
|
+
ids: [route.id],
|
|
329
|
+
limit: perName,
|
|
330
|
+
timeoutMs: live.domainTimeoutMs,
|
|
331
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
332
|
+
...(signal === undefined ? {} : { signal }),
|
|
333
|
+
})));
|
|
334
|
+
const seen = new Set();
|
|
335
|
+
// Keep the exact first hit for every named package ahead of related
|
|
336
|
+
// suggestions: a two-package comparison should show both packages
|
|
337
|
+
// before serde-saphyr or react-router-style neighbours.
|
|
338
|
+
const ordered = [
|
|
339
|
+
...verticals.flatMap((vertical) => vertical.sources.slice(0, 1)),
|
|
340
|
+
...verticals.flatMap((vertical) => vertical.sources.slice(1)),
|
|
341
|
+
];
|
|
342
|
+
const sources = ordered
|
|
343
|
+
.filter((source) => {
|
|
344
|
+
if (seen.has(source.url))
|
|
345
|
+
return false;
|
|
346
|
+
seen.add(source.url);
|
|
347
|
+
return true;
|
|
348
|
+
})
|
|
349
|
+
.slice(0, limit);
|
|
350
|
+
if (sources.length > 0) {
|
|
351
|
+
return {
|
|
352
|
+
sources,
|
|
353
|
+
failures: verticals.flatMap((vertical) => vertical.failures),
|
|
354
|
+
note: `Package query routed to ${route.label} for ${names.join(', ')}; registry rows have stable package URLs and versions.`,
|
|
355
|
+
truncated: sources.length < verticals.reduce((sum, vertical) => sum + vertical.sources.length, 0),
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
catch {
|
|
360
|
+
// The ordinary web path below is the fallback for a registry miss.
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return runQuery(query, limit, signal);
|
|
364
|
+
};
|
|
365
|
+
registerDomainTool(toolTarget, () => {
|
|
185
366
|
const live = resolveConfig(section, undefined);
|
|
186
367
|
const settings = {
|
|
187
368
|
sourceIds: live.domainSourceIds,
|
|
@@ -191,16 +372,23 @@ export function apply(ctx, config = {}) {
|
|
|
191
372
|
};
|
|
192
373
|
return settings;
|
|
193
374
|
}, { warn: warnTool });
|
|
194
|
-
// Batch tool: same optional-registry path, same fail-safe mounting.
|
|
195
|
-
//
|
|
196
|
-
|
|
375
|
+
// Batch tool: same optional-registry path, same fail-safe mounting. Most
|
|
376
|
+
// queries use the `web_search` path; an explicit package-registry query
|
|
377
|
+
// can use the existing domain source instead.
|
|
378
|
+
const batch = registerSearchManyTool(toolTarget, runBatchQuery, { warn: warnTool });
|
|
197
379
|
if (batch !== undefined) {
|
|
198
380
|
onDispose(ctx, batch.dispose);
|
|
199
381
|
}
|
|
382
|
+
// Per-call date window: the only tool that can override the global
|
|
383
|
+
// `timeRange` without changing the user's Settings.
|
|
384
|
+
const advanced = registerAdvancedSearchTool(toolTarget, runAdvancedSearch, { warn: warnTool });
|
|
385
|
+
if (advanced !== undefined) {
|
|
386
|
+
onDispose(ctx, advanced.dispose);
|
|
387
|
+
}
|
|
200
388
|
// Resumable reading: `web_fetch` cannot continue from an offset because
|
|
201
389
|
// the host request carries only `url`, but a plugin-owned tool can. It
|
|
202
390
|
// reuses the same fetch ladder and the same live Settings readers.
|
|
203
|
-
const paged = registerReadPageTool(
|
|
391
|
+
const paged = registerReadPageTool(toolTarget, () => {
|
|
204
392
|
const live = resolveConfig(section, undefined);
|
|
205
393
|
return {
|
|
206
394
|
maxLength: live.fetchMaxLength,
|
|
@@ -223,30 +411,119 @@ export function apply(ctx, config = {}) {
|
|
|
223
411
|
id: 'http-proxy-research',
|
|
224
412
|
timeoutMs: mounted.timeoutMs,
|
|
225
413
|
format: 'markdown',
|
|
226
|
-
|
|
414
|
+
// The evidence pack keeps ~6 K characters, but the second retrieval
|
|
415
|
+
// (passage selection and cache_search) must be able to find a fact
|
|
416
|
+
// deep in the page. 200 K is the provider ceiling, not the model budget.
|
|
417
|
+
maxLength: 200_000,
|
|
227
418
|
renderJs: () => resolveConfig(section, undefined).renderJs,
|
|
228
419
|
renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
|
|
229
420
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
230
421
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
231
422
|
});
|
|
232
423
|
const researchFetch = async (url, signal) => {
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
424
|
+
const read = async (target) => {
|
|
425
|
+
const outcome = await researchProvider.fetch({ url: target }, signal);
|
|
426
|
+
return {
|
|
427
|
+
url: outcome.url,
|
|
428
|
+
status: outcome.statusCode,
|
|
429
|
+
text: outcome.body.kind === 'text' ? outcome.body.content : '',
|
|
430
|
+
truncated: outcome.truncated,
|
|
431
|
+
};
|
|
239
432
|
};
|
|
433
|
+
try {
|
|
434
|
+
const first = await read(url);
|
|
435
|
+
if (first.status < 400 && first.text.trim().length > 0)
|
|
436
|
+
return first;
|
|
437
|
+
const mirror = alternateOfficialUrl(url);
|
|
438
|
+
// The canonical standards page rejected the reader (or returned a
|
|
439
|
+
// shell); try the same organisation's public build once. A failure
|
|
440
|
+
// here falls back to the original outcome so the caller sees the real
|
|
441
|
+
// reason rather than a mirror-specific one.
|
|
442
|
+
if (mirror !== undefined)
|
|
443
|
+
return await read(mirror);
|
|
444
|
+
return first;
|
|
445
|
+
}
|
|
446
|
+
catch (error) {
|
|
447
|
+
const mirror = alternateOfficialUrl(url);
|
|
448
|
+
if (mirror !== undefined)
|
|
449
|
+
return await read(mirror);
|
|
450
|
+
throw error;
|
|
451
|
+
}
|
|
240
452
|
};
|
|
241
|
-
|
|
453
|
+
// Bodies fetched by research are kept in a bounded, expiring in-process
|
|
454
|
+
// cache. `cache_search` is the second retrieval: it searches the full
|
|
455
|
+
// text when the evidence pack only kept the relevant windows.
|
|
456
|
+
const researchCache = new ResearchBodyCache();
|
|
457
|
+
const research = registerResearchTool(toolTarget, (query, limit, signal) => runQuery(query, limit, signal), researchFetch, {
|
|
458
|
+
warn: warnTool,
|
|
459
|
+
onPage: (page) => researchCache.put(page),
|
|
460
|
+
});
|
|
242
461
|
if (research !== undefined) {
|
|
243
462
|
onDispose(ctx, research.dispose);
|
|
244
463
|
}
|
|
464
|
+
const cacheSearch = registerCacheSearchTool(toolTarget, researchCache, { warn: warnTool });
|
|
465
|
+
if (cacheSearch !== undefined) {
|
|
466
|
+
onDispose(ctx, () => researchCache.clear());
|
|
467
|
+
}
|
|
468
|
+
// Community/video search: direct keyless adapters first, the normal
|
|
469
|
+
// `web_search` path as a site-scoped fallback when an endpoint is blocked.
|
|
470
|
+
const platformRunner = async (query, ids, limit, signal) => {
|
|
471
|
+
const live = resolveConfig(section, undefined);
|
|
472
|
+
const outcome = await searchPlatforms(query, {
|
|
473
|
+
ids,
|
|
474
|
+
limit,
|
|
475
|
+
timeoutMs: Math.min(live.timeoutMs, 10_000),
|
|
476
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
477
|
+
...(signal === undefined ? {} : { signal }),
|
|
478
|
+
fallbackSearch: async (fallbackQuery, fallbackLimit, fallbackSignal) => {
|
|
479
|
+
const result = await runQuery(fallbackQuery, fallbackLimit, fallbackSignal);
|
|
480
|
+
return { sources: result.sources, failures: result.failures };
|
|
481
|
+
},
|
|
482
|
+
});
|
|
483
|
+
return {
|
|
484
|
+
sources: outcome.sources,
|
|
485
|
+
failures: outcome.failures,
|
|
486
|
+
via: outcome.via,
|
|
487
|
+
truncated: outcome.truncated,
|
|
488
|
+
};
|
|
489
|
+
};
|
|
490
|
+
const platform = registerPlatformTool(toolTarget, platformRunner, { warn: warnTool });
|
|
491
|
+
// Diagnostics: one tiny query per engine, no cache and no cooldown
|
|
492
|
+
// mutation. It is registered last because it is the least essential tool.
|
|
493
|
+
const engineTest = registerEngineTestTool(toolTarget, runEngineTest, { warn: warnTool });
|
|
494
|
+
if (engineTest !== undefined) {
|
|
495
|
+
onDispose(ctx, engineTest.dispose);
|
|
496
|
+
}
|
|
497
|
+
if (mounted.toolSurface !== 'legacy') {
|
|
498
|
+
// One entry point, eight actions. If the host rejects both candidate
|
|
499
|
+
// names, fall back to the legacy definitions rather than losing every
|
|
500
|
+
// optional capability.
|
|
501
|
+
const unified = registerUnifiedTool(tools, captured, { warn: warnTool });
|
|
502
|
+
if (unified === undefined && captured.length > 0) {
|
|
503
|
+
warnTool('web_tools unavailable; registering legacy tool names as a fallback');
|
|
504
|
+
for (const definition of captured) {
|
|
505
|
+
try {
|
|
506
|
+
tools.register(definition);
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
warnTool(`legacy fallback registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
245
514
|
}
|
|
246
515
|
}
|
|
247
516
|
catch (error) {
|
|
248
517
|
warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
249
518
|
}
|
|
519
|
+
// A short prompt section explains the relationships the tool schemas cannot:
|
|
520
|
+
// when to use advanced_search, and that the boundary tags are data, not
|
|
521
|
+
// instructions. Best-effort: a host without `systemPrompt` simply skips it.
|
|
522
|
+
registerSearchPrompt(ctx);
|
|
523
|
+
// The Settings page cannot call a model tool, so its "Test engines" button
|
|
524
|
+
// gets one narrow loopback endpoint. The model-facing tool above stays the
|
|
525
|
+
// primary path; this bridge is best-effort and silent on hosts without it.
|
|
526
|
+
registerEngineTestBridge(ctx, runEngineTest);
|
|
250
527
|
}
|
|
251
528
|
export { Config, resolveConfig } from "./config.js";
|
|
252
529
|
/**
|
|
@@ -274,6 +551,13 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
|
|
|
274
551
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
275
552
|
export { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
276
553
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
|
554
|
+
export { ADVANCED_SEARCH_TOOL_NAME, ADVANCED_SEARCH_FALLBACK_NAME, ADVANCED_SEARCH_DEFAULT_LIMIT, ADVANCED_SEARCH_MAX_LIMIT, createAdvancedSearchDefinition, registerAdvancedSearchTool, pickAdvancedSearchToolName, formatAdvancedSearchValue, } from "./advanced-search.js";
|
|
555
|
+
export { SEARCH_PROMPT_SECTION, SEARCH_PROMPT_ORDER, SEARCH_PROMPT_TEXT, registerSearchPrompt, } from "./system-prompt.js";
|
|
556
|
+
export { UNTRUSTED_OPEN, UNTRUSTED_CLOSE, stripUntrustedBoundary, wrapUntrustedContent } from "./untrusted.js";
|
|
277
557
|
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
|
|
278
558
|
export { RESEARCH_TOOL_NAME, RESEARCH_TOOL_FALLBACK_NAME, RESEARCH_DEFAULT_SOURCES, RESEARCH_MAX_SOURCES, RESEARCH_MAX_QUERIES, RESEARCH_EXCERPT_CHARS, RESEARCH_BUDGET_MS, createResearchDefinition, registerResearchTool, pickResearchToolName, formatResearchValue, } from "./research.js";
|
|
559
|
+
export { PLATFORM_DEFAULT_LIMIT, PLATFORM_MAX_LIMIT, PLATFORM_TOOL_FALLBACK_NAME, PLATFORM_TOOL_NAME, createPlatformToolDefinition, parsePlatformArgs, pickPlatformToolName, registerPlatformTool, } from "./platform-tool.js";
|
|
560
|
+
export { DEFAULT_PLATFORM_LIMIT, MAX_PLATFORM_LIMIT, PLATFORM_IDS, PLATFORM_LABELS, isPlatformId, parseBilibili, parseJuejin, parseReddit, parseV2ex, parseYoutube, searchPlatforms, } from "./platforms.js";
|
|
561
|
+
export { CACHE_SEARCH_DEFAULT_LIMIT, CACHE_SEARCH_FALLBACK_NAME, CACHE_SEARCH_MAX_BODIES, CACHE_SEARCH_MAX_CHARS, CACHE_SEARCH_MAX_LIMIT, CACHE_SEARCH_SNIPPET_CHARS, CACHE_SEARCH_TTL_MS, CACHE_SEARCH_TOOL_NAME, ResearchBodyCache, cacheTokens, cacheWindows, createCacheSearchDefinition, formatCacheSearchValue, pickCacheSearchToolName, registerCacheSearchTool, } from "./research-cache.js";
|
|
562
|
+
export { alternateOfficialUrl, canonicalOfficialUrl } from "./mirrors.js";
|
|
279
563
|
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";
|
package/lib/inject.d.ts
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The optional Cordis services this plugin reaches through `ctx.inject`.
|
|
3
|
+
*
|
|
4
|
+
* None of these are hard dependencies: `web` is the only required service, and
|
|
5
|
+
* a deployment without a tool registry, web server or system-prompt service
|
|
6
|
+
* still mounts the search provider. The interfaces live in one module so the
|
|
7
|
+
* bridge and the prompt section agree on the injected shape, and so the plugin
|
|
8
|
+
* context can expose the hook without importing a feature module.
|
|
9
|
+
*
|
|
10
|
+
* @module @fangchenghao/dsh-web-search/inject
|
|
11
|
+
*/
|
|
12
|
+
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
13
|
+
/** The slice of the host web server the Settings bridge needs. */
|
|
14
|
+
export interface WebServerLike {
|
|
15
|
+
register(route: {
|
|
16
|
+
kind: 'exact';
|
|
17
|
+
path: string;
|
|
18
|
+
handler: (request: IncomingMessage, response: ServerResponse) => void | Promise<void>;
|
|
19
|
+
}): unknown;
|
|
20
|
+
}
|
|
21
|
+
/** One section the host's system prompt accepts. */
|
|
22
|
+
export interface SystemPromptSectionOptions {
|
|
23
|
+
/** Stable identifier used by the host to replace or remove the section. */
|
|
24
|
+
name: string;
|
|
25
|
+
/** Ordering hint; lower values appear earlier. */
|
|
26
|
+
order?: number;
|
|
27
|
+
/** The text inserted into the model's system prompt. */
|
|
28
|
+
text: string;
|
|
29
|
+
}
|
|
30
|
+
/** The slice of the host system-prompt service this plugin needs. */
|
|
31
|
+
export interface SystemPromptLike {
|
|
32
|
+
section(options: SystemPromptSectionOptions): unknown;
|
|
33
|
+
}
|
|
34
|
+
/** Services that may arrive through one `ctx.inject` callback. */
|
|
35
|
+
export interface ServiceContext {
|
|
36
|
+
webServer?: WebServerLike;
|
|
37
|
+
systemPrompt?: SystemPromptLike;
|
|
38
|
+
effect?: (execute: () => (() => void) | void) => unknown;
|
|
39
|
+
}
|
|
40
|
+
/** A context that may support Cordis's dynamic dependency injection. */
|
|
41
|
+
export interface InjectableHost {
|
|
42
|
+
inject?: (dependencies: string[], callback: (context: ServiceContext) => unknown) => unknown;
|
|
43
|
+
}
|
package/lib/inject.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/lib/markdown.js
CHANGED
|
@@ -403,6 +403,14 @@ function density(node) {
|
|
|
403
403
|
const characters = characterCount(node);
|
|
404
404
|
return characters === 0 ? 0 : characters / (1 + elementCount(node));
|
|
405
405
|
}
|
|
406
|
+
/** True when most of a container's text is one <pre> block (an appendix, not prose). */
|
|
407
|
+
function codeDominated(node) {
|
|
408
|
+
const pre = findElement(node, (child) => child.tag === 'pre');
|
|
409
|
+
if (pre === undefined)
|
|
410
|
+
return false;
|
|
411
|
+
const characters = characterCount(node);
|
|
412
|
+
return characters > 0 && characterCount(pre) >= characters * 0.6;
|
|
413
|
+
}
|
|
406
414
|
/** Highest-density container, favouring the one that carries the most text. */
|
|
407
415
|
function densestContainer(body) {
|
|
408
416
|
let best = body;
|
|
@@ -414,6 +422,11 @@ function densestContainer(body) {
|
|
|
414
422
|
const characters = characterCount(node);
|
|
415
423
|
if (characters < 120)
|
|
416
424
|
return;
|
|
425
|
+
// An appendix that is one huge ABNF/code block has a very high text/markup
|
|
426
|
+
// ratio, but it is not the page's prose. Skipping it lets the body or a
|
|
427
|
+
// prose section win; the body fallback still keeps the code.
|
|
428
|
+
if (codeDominated(node))
|
|
429
|
+
return;
|
|
417
430
|
const score = density(node);
|
|
418
431
|
if (score > bestScore || (score === bestScore && characters > bestCharacters)) {
|
|
419
432
|
best = node;
|
|
@@ -465,7 +478,11 @@ function pickMain(root) {
|
|
|
465
478
|
if (landmark !== undefined && characterCount(landmark) >= floor)
|
|
466
479
|
return landmark;
|
|
467
480
|
}
|
|
468
|
-
|
|
481
|
+
// A single section can be denser than the whole body on long documents with
|
|
482
|
+
// no <article>/<main> (RFC 9110 has dozens of sections). Choosing it makes
|
|
483
|
+
// the rest of the document unreachable; only trust density when it carries
|
|
484
|
+
// at least half the page's text.
|
|
485
|
+
if (characterCount(densest) >= floor && characterCount(densest) * 2 > bodyCharacters)
|
|
469
486
|
return densest;
|
|
470
487
|
// Density alone can crown a short, dense stub — a comment form or a footer —
|
|
471
488
|
// while the prose sits in a container with more markup and fewer characters
|
package/lib/mirrors.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Official mirrors for a few standards pages that block automated readers.
|
|
3
|
+
*
|
|
4
|
+
* This is not a scraper fallback hierarchy: it is a very small, explicit map
|
|
5
|
+
* from a canonical standards URL to the same organisation's public GitHub
|
|
6
|
+
* Pages copy. A mirror is only tried after the canonical URL failed or returned
|
|
7
|
+
* an empty body. The citation keeps the canonical W3C page; the body is the
|
|
8
|
+
* official mirror copy.
|
|
9
|
+
*
|
|
10
|
+
* @module @fangchenghao/dsh-web-search/mirrors
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Return an official mirror for a blocked standards URL, if one is known.
|
|
14
|
+
* @param raw - canonical URL.
|
|
15
|
+
* @returns the mirror URL, or `undefined` when no rule applies.
|
|
16
|
+
*/
|
|
17
|
+
export declare function alternateOfficialUrl(raw: string): string | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Map a mirror URL back to the canonical page that is safe to cite.
|
|
20
|
+
* @param raw - URL actually fetched.
|
|
21
|
+
* @returns the canonical URL, or the input when it is not a known mirror.
|
|
22
|
+
*/
|
|
23
|
+
export declare function canonicalOfficialUrl(raw: string): string;
|