@fffangchenghao/dsh-web-search 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +132 -34
- package/README.zh-CN.md +132 -34
- package/client.js +28 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +16 -0
- package/lib/config.js +28 -0
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/index.d.ts +20 -0
- package/lib/index.js +91 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/types.d.ts +7 -4
- package/package.json +1 -1
package/lib/fetch.js
CHANGED
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
* @module @fangchenghao/dsh-web-search/fetch
|
|
25
25
|
*/
|
|
26
26
|
import { SearchError } from "./errors.js";
|
|
27
|
+
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
27
28
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
28
29
|
/** Default user agent; kept in step with the search side. */
|
|
29
30
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
@@ -126,6 +127,38 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
126
127
|
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
127
128
|
const userAgent = options.userAgent ?? USER_AGENT;
|
|
128
129
|
const apiKey = (options.anysearchApiKey ?? '').trim();
|
|
130
|
+
/**
|
|
131
|
+
* Resolve the two volatile Settings fields per request. Reading them here
|
|
132
|
+
* rather than at construction is what lets a saved edit apply without a
|
|
133
|
+
* remount; caching them made the Settings page lie.
|
|
134
|
+
*/
|
|
135
|
+
const liveFormat = () => typeof options.format === 'function' ? options.format() : (options.format ?? 'markdown');
|
|
136
|
+
const liveMaxLength = () => {
|
|
137
|
+
const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
|
|
138
|
+
return raw ?? DEFAULT_FETCH_MAX_LENGTH;
|
|
139
|
+
};
|
|
140
|
+
// Every rung funnels through here: an HTML body becomes Markdown (or plain
|
|
141
|
+
// text), a text body keeps its content, and both obey the character budget.
|
|
142
|
+
// The Host's own renderer then sees `kind: 'text'`, so it does not convert the
|
|
143
|
+
// same page a second time.
|
|
144
|
+
const finalize = (outcome) => {
|
|
145
|
+
const format = liveFormat();
|
|
146
|
+
const maxLength = liveMaxLength();
|
|
147
|
+
if (outcome.body.kind === 'html') {
|
|
148
|
+
const rendered = renderFetchText(outcome.body.content, { format, maxLength, baseUrl: outcome.url });
|
|
149
|
+
return {
|
|
150
|
+
...outcome,
|
|
151
|
+
body: { kind: 'text', content: rendered.content },
|
|
152
|
+
truncated: outcome.truncated || rendered.truncated,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
const limited = truncateFetchText(outcome.body.content, maxLength);
|
|
156
|
+
return {
|
|
157
|
+
...outcome,
|
|
158
|
+
body: { kind: 'text', content: limited.content },
|
|
159
|
+
truncated: outcome.truncated || limited.truncated,
|
|
160
|
+
};
|
|
161
|
+
};
|
|
129
162
|
return {
|
|
130
163
|
id: options.id ?? 'http-proxy',
|
|
131
164
|
available: () => true,
|
|
@@ -141,7 +174,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
141
174
|
// shorter deadline than the tunnel: spending the whole budget on a request
|
|
142
175
|
// that cannot work would leave the rung that can work with nothing.
|
|
143
176
|
try {
|
|
144
|
-
return await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
|
|
177
|
+
return finalize(await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes));
|
|
145
178
|
}
|
|
146
179
|
catch (error) {
|
|
147
180
|
if (cancelled())
|
|
@@ -159,7 +192,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
159
192
|
}
|
|
160
193
|
if (proxy !== undefined && proxy.length > 0) {
|
|
161
194
|
try {
|
|
162
|
-
return await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
|
|
195
|
+
return finalize(await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes));
|
|
163
196
|
}
|
|
164
197
|
catch (error) {
|
|
165
198
|
if (cancelled())
|
|
@@ -169,7 +202,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
169
202
|
}
|
|
170
203
|
if (apiKey.length > 0) {
|
|
171
204
|
try {
|
|
172
|
-
return await deps.extract(target, apiKey, timeoutMs, signal);
|
|
205
|
+
return finalize(await deps.extract(target, apiKey, timeoutMs, signal));
|
|
173
206
|
}
|
|
174
207
|
catch (error) {
|
|
175
208
|
if (cancelled())
|
package/lib/index.d.ts
CHANGED
|
@@ -39,6 +39,13 @@ export interface PluginContext {
|
|
|
39
39
|
get?: (name: string) => unknown;
|
|
40
40
|
/** Fallback for tests and hosts that expose the registry as a property. */
|
|
41
41
|
tools?: unknown;
|
|
42
|
+
/**
|
|
43
|
+
* Cordis lifecycle hook. `effect` is the canonical registration; `on` is the
|
|
44
|
+
* older/test fallback. Both let the tool registration and the cache release
|
|
45
|
+
* their resources when the plugin is disposed.
|
|
46
|
+
*/
|
|
47
|
+
effect?: (execute: () => (() => void) | void) => unknown;
|
|
48
|
+
on?: (event: string, listener: () => void) => unknown;
|
|
42
49
|
}
|
|
43
50
|
/** Configuration as the loader hands it over, already schema-validated. */
|
|
44
51
|
export type ApplyConfig = PluginConfig;
|
|
@@ -65,6 +72,8 @@ declare const _default: typeof apply & {
|
|
|
65
72
|
Config: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<NoInfer<{
|
|
66
73
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
67
74
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
75
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
76
|
+
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
68
77
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
69
78
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
70
79
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -76,6 +85,8 @@ declare const _default: typeof apply & {
|
|
|
76
85
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
77
86
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
78
87
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
88
|
+
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
89
|
+
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
79
90
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
80
91
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
81
92
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -86,6 +97,8 @@ declare const _default: typeof apply & {
|
|
|
86
97
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
87
98
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
88
99
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
100
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
101
|
+
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
89
102
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
90
103
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
91
104
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -97,6 +110,8 @@ declare const _default: typeof apply & {
|
|
|
97
110
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
98
111
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
99
112
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
113
|
+
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
114
|
+
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
100
115
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
101
116
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
102
117
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -110,6 +125,9 @@ declare const _default: typeof apply & {
|
|
|
110
125
|
export default _default;
|
|
111
126
|
export { HOSTS, search } from './engines.ts';
|
|
112
127
|
export { SearchCache, cacheKey } from './cache.ts';
|
|
128
|
+
export type { CachePersistOptions, SearchCacheOptions } from './cache.ts';
|
|
129
|
+
export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from './cache-persist.ts';
|
|
130
|
+
export type { CacheFileIo, CacheWriteOptions, PersistedEntry, SerializedCache } from './cache-persist.ts';
|
|
113
131
|
export { EngineHealth } from './health.ts';
|
|
114
132
|
export { SearchError, parseRetryAfter } from './errors.ts';
|
|
115
133
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from './parsers.ts';
|
|
@@ -120,4 +138,6 @@ export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, p
|
|
|
120
138
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
|
|
121
139
|
export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
|
|
122
140
|
export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
|
|
141
|
+
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
|
|
142
|
+
export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
|
|
123
143
|
export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
|
package/lib/index.js
CHANGED
|
@@ -15,6 +15,7 @@ import { SearchCache } from "./cache.js";
|
|
|
15
15
|
import { EngineHealth } from "./health.js";
|
|
16
16
|
import { createFetchProvider } from "./fetch.js";
|
|
17
17
|
import { registerDomainTool } from "./domain-tool.js";
|
|
18
|
+
import { registerSearchManyTool } from "./search-many.js";
|
|
18
19
|
export const name = 'dsh-web-search';
|
|
19
20
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
20
21
|
export const inject = ['web'];
|
|
@@ -53,6 +54,26 @@ function warnTool(message) {
|
|
|
53
54
|
// Logging must never be the reason a plugin fails to mount.
|
|
54
55
|
}
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Register a cleanup with the plugin lifecycle.
|
|
59
|
+
*
|
|
60
|
+
* `ctx.effect(() => cleanup)` is Cordis's ownership primitive; a host that
|
|
61
|
+
* predates it (or a test double) may only expose a `dispose` event, so that is
|
|
62
|
+
* the fallback. Losing the hook must not break mounting.
|
|
63
|
+
*/
|
|
64
|
+
function onDispose(ctx, cleanup) {
|
|
65
|
+
try {
|
|
66
|
+
if (typeof ctx.effect === 'function') {
|
|
67
|
+
ctx.effect(() => cleanup);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (typeof ctx.on === 'function')
|
|
71
|
+
ctx.on('dispose', cleanup);
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
warnTool(`dispose hook skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
56
77
|
/**
|
|
57
78
|
* Register the search provider.
|
|
58
79
|
* @param ctx - plugin context carrying the `web` service.
|
|
@@ -66,15 +87,63 @@ export function apply(ctx, config = {}) {
|
|
|
66
87
|
// One cache and one health tracker per mounted plugin, so a config change
|
|
67
88
|
// through a restart starts from a clean slate instead of inheriting the
|
|
68
89
|
// cooldowns of a previous instance.
|
|
69
|
-
const cache = new SearchCache({
|
|
90
|
+
const cache = new SearchCache({
|
|
91
|
+
// Read through on every use: a Settings edit must take effect without a
|
|
92
|
+
// remount, which is the whole point of marking these fields volatile.
|
|
93
|
+
ttlMs: () => resolveConfig(section, undefined).cacheTtlMs,
|
|
94
|
+
maxEntries: () => resolveConfig(section, undefined).cacheEntries,
|
|
95
|
+
...(mounted.cachePersist
|
|
96
|
+
? { persist: { ...(mounted.cacheFile === undefined ? {} : { file: mounted.cacheFile }) } }
|
|
97
|
+
: {}),
|
|
98
|
+
});
|
|
70
99
|
const health = new EngineHealth({ baseCooldownMs: mounted.baseCooldownMs, maxCooldownMs: mounted.maxCooldownMs });
|
|
100
|
+
// The disk snapshot is flushed at plugin disposal; the cache also flushes on
|
|
101
|
+
// its own exit hook, so an abrupt process end keeps the newest state.
|
|
102
|
+
onDispose(ctx, () => cache.close());
|
|
103
|
+
/**
|
|
104
|
+
* Run one query through the same engine path as `web_search`.
|
|
105
|
+
*
|
|
106
|
+
* Reading `section` on every call is what makes a Settings commit visible
|
|
107
|
+
* without a restart. The configured limit is the hard ceiling: a caller may
|
|
108
|
+
* ask for fewer results, never for more.
|
|
109
|
+
*/
|
|
110
|
+
const runQuery = async (query, requested, signal) => {
|
|
111
|
+
const resolved = resolveConfig(section, undefined);
|
|
112
|
+
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
113
|
+
throw new Error('web search requires a non-empty query');
|
|
114
|
+
}
|
|
115
|
+
if (signal?.aborted === true)
|
|
116
|
+
throw new Error('web search was cancelled');
|
|
117
|
+
const asked = typeof requested === 'number' && Number.isInteger(requested) && requested > 0
|
|
118
|
+
? requested
|
|
119
|
+
: undefined;
|
|
120
|
+
const limit = Math.min(asked ?? resolved.limit, resolved.limit);
|
|
121
|
+
const { sources, failures } = await search(query, signal, {
|
|
122
|
+
limit,
|
|
123
|
+
wait: resolved.wait,
|
|
124
|
+
timeRange: resolved.timeRange,
|
|
125
|
+
timeoutMs: resolved.timeoutMs,
|
|
126
|
+
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
127
|
+
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
128
|
+
rank: resolved.rank,
|
|
129
|
+
...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
|
|
130
|
+
...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
|
|
131
|
+
cache,
|
|
132
|
+
health,
|
|
133
|
+
...(signal === undefined ? {} : { signal }),
|
|
134
|
+
});
|
|
135
|
+
return {
|
|
136
|
+
sources,
|
|
137
|
+
failures,
|
|
138
|
+
truncated: asked !== undefined && asked > resolved.limit,
|
|
139
|
+
};
|
|
140
|
+
};
|
|
71
141
|
ctx.web.registerSearchProvider({
|
|
72
142
|
id: PROVIDER_ID,
|
|
73
143
|
available: () => true,
|
|
74
144
|
async search(request, signal) {
|
|
75
145
|
// A Settings commit only mutates the existing references; reading the
|
|
76
|
-
// section
|
|
77
|
-
const resolved = resolveConfig(section, undefined);
|
|
146
|
+
// section inside `runQuery` is what makes the edit apply without a restart.
|
|
78
147
|
const query = request.query;
|
|
79
148
|
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
80
149
|
throw new Error('web search requires a non-empty query');
|
|
@@ -84,27 +153,11 @@ export function apply(ctx, config = {}) {
|
|
|
84
153
|
const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
|
|
85
154
|
? request.maxResults
|
|
86
155
|
: undefined;
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
wait: resolved.wait,
|
|
91
|
-
timeRange: resolved.timeRange,
|
|
92
|
-
timeoutMs: resolved.timeoutMs,
|
|
93
|
-
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
94
|
-
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
95
|
-
rank: resolved.rank,
|
|
96
|
-
...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
|
|
97
|
-
...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
|
|
98
|
-
cache,
|
|
99
|
-
health,
|
|
100
|
-
...(signal === undefined ? {} : { signal }),
|
|
101
|
-
});
|
|
102
|
-
if (sources.length === 0) {
|
|
103
|
-
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${failures.join('; ')})`);
|
|
156
|
+
const outcome = await runQuery(query, asked, signal);
|
|
157
|
+
if (outcome.sources.length === 0) {
|
|
158
|
+
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
104
159
|
}
|
|
105
|
-
|
|
106
|
-
return { sources, truncated: false };
|
|
107
|
-
return { sources: sources.slice(0, limit), truncated: true };
|
|
160
|
+
return { sources: outcome.sources, truncated: outcome.truncated };
|
|
108
161
|
},
|
|
109
162
|
});
|
|
110
163
|
// The same tunnel the search side resolves, offered to `web_fetch`: a search
|
|
@@ -112,12 +165,15 @@ export function apply(ctx, config = {}) {
|
|
|
112
165
|
ctx.web.registerFetchProvider(createFetchProvider({
|
|
113
166
|
id: FETCH_PROVIDER_ID,
|
|
114
167
|
timeoutMs: mounted.timeoutMs,
|
|
168
|
+
format: () => resolveConfig(section, undefined).fetchFormat,
|
|
169
|
+
maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
|
|
115
170
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
116
171
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
117
172
|
}));
|
|
118
|
-
// Optional half: `domain_search`. Fail-safe by
|
|
119
|
-
// without `ctx.tools` still loads this plugin and
|
|
120
|
-
// registration path is the host's
|
|
173
|
+
// Optional half: `domain_search` and `search_many`. Fail-safe by
|
|
174
|
+
// construction — a deployment without `ctx.tools` still loads this plugin and
|
|
175
|
+
// keeps `web_search`; the registration path is the host's
|
|
176
|
+
// `ToolRuntime.register(definition)`.
|
|
121
177
|
try {
|
|
122
178
|
const tools = readToolRegistry(ctx);
|
|
123
179
|
if (tools !== undefined) {
|
|
@@ -131,10 +187,16 @@ export function apply(ctx, config = {}) {
|
|
|
131
187
|
};
|
|
132
188
|
return settings;
|
|
133
189
|
}, { warn: warnTool });
|
|
190
|
+
// Batch tool: same optional-registry path, same fail-safe mounting. The
|
|
191
|
+
// runner is the `web_search` path, so both tools share lookups and cache.
|
|
192
|
+
const batch = registerSearchManyTool(tools, (query, limit, signal) => runQuery(query, limit, signal), { warn: warnTool });
|
|
193
|
+
if (batch !== undefined) {
|
|
194
|
+
onDispose(ctx, batch.dispose);
|
|
195
|
+
}
|
|
134
196
|
}
|
|
135
197
|
}
|
|
136
198
|
catch (error) {
|
|
137
|
-
warnTool(`
|
|
199
|
+
warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
138
200
|
}
|
|
139
201
|
}
|
|
140
202
|
export { Config, resolveConfig } from "./config.js";
|
|
@@ -154,6 +216,7 @@ export { Config, resolveConfig } from "./config.js";
|
|
|
154
216
|
export default Object.assign(apply, { Config, inject });
|
|
155
217
|
export { HOSTS, search } from "./engines.js";
|
|
156
218
|
export { SearchCache, cacheKey } from "./cache.js";
|
|
219
|
+
export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from "./cache-persist.js";
|
|
157
220
|
export { EngineHealth } from "./health.js";
|
|
158
221
|
export { SearchError, parseRetryAfter } from "./errors.js";
|
|
159
222
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
|
|
@@ -162,3 +225,4 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
|
|
|
162
225
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
163
226
|
export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
164
227
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
|
228
|
+
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dependency-free HTML → readable Markdown for `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* A fetched page is mostly furniture: site chrome, a table of contents, ad
|
|
5
|
+
* slots, scripts. Handing that markup to a model wastes its budget on
|
|
6
|
+
* navigation and makes it parse tags instead of reading prose. This module
|
|
7
|
+
* keeps the parts a reader would keep and renders them as Markdown:
|
|
8
|
+
*
|
|
9
|
+
* 1. a tolerant tokenizer and tree builder, because real pages are not XML;
|
|
10
|
+
* 2. a pruning pass that drops chrome tags, comments and hidden nodes;
|
|
11
|
+
* 3. a body selection pass that prefers `<article>`, then `<main>`/`role=main`,
|
|
12
|
+
* then the densest text container;
|
|
13
|
+
* 4. a renderer for headings, paragraphs, lists, code, quotes, links, images,
|
|
14
|
+
* rules and tables, with entities decoded and URLs made absolute.
|
|
15
|
+
*
|
|
16
|
+
* It deliberately has no runtime dependency: the plugin ships as plain ESM and
|
|
17
|
+
* a parser package would be the first thing to break on a Node upgrade.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/markdown
|
|
20
|
+
*/
|
|
21
|
+
/** Output style selected by `fetchFormat`. */
|
|
22
|
+
export type FetchFormat = 'markdown' | 'text';
|
|
23
|
+
/** Options shared by the Markdown and plain-text renderers. */
|
|
24
|
+
export interface RenderOptions {
|
|
25
|
+
/** `markdown` keeps syntax; `text` keeps only the words. */
|
|
26
|
+
readonly format?: FetchFormat | undefined;
|
|
27
|
+
/** Page URL, used to make relative links absolute. */
|
|
28
|
+
readonly baseUrl?: string | undefined;
|
|
29
|
+
}
|
|
30
|
+
/** Result of applying the configured character budget. */
|
|
31
|
+
export interface TruncatedText {
|
|
32
|
+
readonly content: string;
|
|
33
|
+
readonly truncated: boolean;
|
|
34
|
+
}
|
|
35
|
+
/** A converted page plus the numbers a truncation notice reports. */
|
|
36
|
+
export interface RenderedFetchText {
|
|
37
|
+
/** Converted content, cut to the budget when necessary. */
|
|
38
|
+
readonly content: string;
|
|
39
|
+
/** Length of the converted page before truncation. */
|
|
40
|
+
readonly originalLength: number;
|
|
41
|
+
/** Whether the character budget cut the page. */
|
|
42
|
+
readonly truncated: boolean;
|
|
43
|
+
}
|
|
44
|
+
/** Default `fetchMaxLength`: about 20k characters of prose. */
|
|
45
|
+
export declare const DEFAULT_FETCH_MAX_LENGTH = 20000;
|
|
46
|
+
/**
|
|
47
|
+
* Decode named and numeric HTML entities.
|
|
48
|
+
*
|
|
49
|
+
* Unknown named entities are preserved verbatim: dropping them would silently
|
|
50
|
+
* delete text, and guessing a codepoint would corrupt it.
|
|
51
|
+
*
|
|
52
|
+
* @param value - text or an attribute value.
|
|
53
|
+
* @returns the decoded text.
|
|
54
|
+
*/
|
|
55
|
+
export declare function decodeEntities(value: string): string;
|
|
56
|
+
/**
|
|
57
|
+
* Convert an HTML document or fragment to readable Markdown or plain text.
|
|
58
|
+
*
|
|
59
|
+
* @param html - the raw markup.
|
|
60
|
+
* @param options - output format and the base URL for relative links.
|
|
61
|
+
* @returns the converted text; an empty or contentless document yields `''`.
|
|
62
|
+
*/
|
|
63
|
+
export declare function htmlToMarkdown(html: string, options?: RenderOptions): string;
|
|
64
|
+
/**
|
|
65
|
+
* Apply the output character budget.
|
|
66
|
+
*
|
|
67
|
+
* The notice names both the configured limit and the original converted
|
|
68
|
+
* length, so a reader knows how much is missing and can ask for a narrower URL.
|
|
69
|
+
*
|
|
70
|
+
* @param content - converted content.
|
|
71
|
+
* @param maxLength - configured character budget.
|
|
72
|
+
* @returns the content, cut with a notice when it exceeded the budget.
|
|
73
|
+
*/
|
|
74
|
+
export declare function truncateFetchText(content: string, maxLength: number): TruncatedText;
|
|
75
|
+
/**
|
|
76
|
+
* Convert a fetched HTML page and apply the configured budget in one step.
|
|
77
|
+
*
|
|
78
|
+
* @param html - the page markup.
|
|
79
|
+
* @param options - format, base URL and character budget.
|
|
80
|
+
* @returns converted content plus the pre-truncation length and cut flag.
|
|
81
|
+
*/
|
|
82
|
+
export declare function renderFetchText(html: string, options: RenderOptions & {
|
|
83
|
+
readonly maxLength: number;
|
|
84
|
+
}): RenderedFetchText;
|