@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -14,13 +14,66 @@ import { search } from "./engines.js";
14
14
  import { SearchCache } from "./cache.js";
15
15
  import { EngineHealth } from "./health.js";
16
16
  import { createFetchProvider } from "./fetch.js";
17
- export const name = 'dsh-web-search-bing';
17
+ import { registerDomainTool } from "./domain-tool.js";
18
+ import { registerSearchManyTool } from "./search-many.js";
19
+ export const name = 'dsh-web-search';
18
20
  /** The `ctx.web` service this plugin registers a search provider with. */
19
21
  export const inject = ['web'];
20
22
  /** Provider id referenced by `web.searchProvider`. */
21
23
  export const PROVIDER_ID = 'bing';
22
24
  /** Fetch provider id, named in the `web` service config as `fetchProvider`. */
23
25
  export const FETCH_PROVIDER_ID = 'http-proxy';
26
+ /**
27
+ * Read the optional tool registry without requiring it.
28
+ * @param ctx - the plugin context.
29
+ * @returns the registry, or `undefined` when this deployment has none.
30
+ */
31
+ function readToolRegistry(ctx) {
32
+ let candidate;
33
+ try {
34
+ const found = typeof ctx.get === 'function' ? ctx.get('tools') : undefined;
35
+ // Cordis throws on an un-injected service property, so even the fallback
36
+ // read is guarded: a missing `tools` is a supported deployment, not an error.
37
+ candidate = found ?? ctx.tools;
38
+ }
39
+ catch {
40
+ return undefined;
41
+ }
42
+ if (typeof candidate !== 'object' || candidate === null)
43
+ return undefined;
44
+ return typeof candidate.register === 'function'
45
+ ? candidate
46
+ : undefined;
47
+ }
48
+ /** Report a skipped tool registration without breaking plugin loading. */
49
+ function warnTool(message) {
50
+ try {
51
+ console.warn(`[dsh-web-search-bing] ${message}`);
52
+ }
53
+ catch {
54
+ // Logging must never be the reason a plugin fails to mount.
55
+ }
56
+ }
57
+ /**
58
+ * Register a cleanup with the plugin lifecycle.
59
+ *
60
+ * `ctx.effect(() => cleanup)` is Cordis's ownership primitive; a host that
61
+ * predates it (or a test double) may only expose a `dispose` event, so that is
62
+ * the fallback. Losing the hook must not break mounting.
63
+ */
64
+ function onDispose(ctx, cleanup) {
65
+ try {
66
+ if (typeof ctx.effect === 'function') {
67
+ ctx.effect(() => cleanup);
68
+ return;
69
+ }
70
+ if (typeof ctx.on === 'function')
71
+ ctx.on('dispose', cleanup);
72
+ }
73
+ catch (error) {
74
+ warnTool(`dispose hook skipped: ${error instanceof Error ? error.message : String(error)}`);
75
+ }
76
+ }
24
77
  /**
25
78
  * Register the search provider.
26
79
  * @param ctx - plugin context carrying the `web` service.
@@ -34,15 +87,63 @@ export function apply(ctx, config = {}) {
34
87
  // One cache and one health tracker per mounted plugin, so a config change
35
88
  // through a restart starts from a clean slate instead of inheriting the
36
89
  // cooldowns of a previous instance.
37
- const cache = new SearchCache({ ttlMs: mounted.cacheTtlMs, maxEntries: mounted.cacheEntries });
90
+ const cache = new SearchCache({
91
+ // Read through on every use: a Settings edit must take effect without a
92
+ // remount, which is the whole point of marking these fields volatile.
93
+ ttlMs: () => resolveConfig(section, undefined).cacheTtlMs,
94
+ maxEntries: () => resolveConfig(section, undefined).cacheEntries,
95
+ ...(mounted.cachePersist
96
+ ? { persist: { ...(mounted.cacheFile === undefined ? {} : { file: mounted.cacheFile }) } }
97
+ : {}),
98
+ });
38
99
  const health = new EngineHealth({ baseCooldownMs: mounted.baseCooldownMs, maxCooldownMs: mounted.maxCooldownMs });
100
+ // The disk snapshot is flushed at plugin disposal; the cache also flushes on
101
+ // its own exit hook, so an abrupt process end keeps the newest state.
102
+ onDispose(ctx, () => cache.close());
103
+ /**
104
+ * Run one query through the same engine path as `web_search`.
105
+ *
106
+ * Reading `section` on every call is what makes a Settings commit visible
107
+ * without a restart. The configured limit is the hard ceiling: a caller may
108
+ * ask for fewer results, never for more.
109
+ */
110
+ const runQuery = async (query, requested, signal) => {
111
+ const resolved = resolveConfig(section, undefined);
112
+ if (typeof query !== 'string' || query.trim().length === 0) {
113
+ throw new Error('web search requires a non-empty query');
114
+ }
115
+ if (signal?.aborted === true)
116
+ throw new Error('web search was cancelled');
117
+ const asked = typeof requested === 'number' && Number.isInteger(requested) && requested > 0
118
+ ? requested
119
+ : undefined;
120
+ const limit = Math.min(asked ?? resolved.limit, resolved.limit);
121
+ const { sources, failures } = await search(query, signal, {
122
+ limit,
123
+ wait: resolved.wait,
124
+ timeRange: resolved.timeRange,
125
+ timeoutMs: resolved.timeoutMs,
126
+ ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
127
+ ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
128
+ rank: resolved.rank,
129
+ ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
130
+ ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
131
+ cache,
132
+ health,
133
+ ...(signal === undefined ? {} : { signal }),
134
+ });
135
+ return {
136
+ sources,
137
+ failures,
138
+ truncated: asked !== undefined && asked > resolved.limit,
139
+ };
140
+ };
39
141
  ctx.web.registerSearchProvider({
40
142
  id: PROVIDER_ID,
41
143
  available: () => true,
42
144
  async search(request, signal) {
43
145
  // A Settings commit only mutates the existing references; reading the
44
- // section here is what makes the edit apply without a restart.
45
- const resolved = resolveConfig(section, undefined);
146
+ // section inside `runQuery` is what makes the edit apply without a restart.
46
147
  const query = request.query;
47
148
  if (typeof query !== 'string' || query.trim().length === 0) {
48
149
  throw new Error('web search requires a non-empty query');
@@ -52,26 +153,11 @@ export function apply(ctx, config = {}) {
52
153
  const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
53
154
  ? request.maxResults
54
155
  : undefined;
55
- const limit = Math.min(asked ?? resolved.limit, resolved.limit);
56
- const { sources, failures } = await search(query, signal, {
57
- limit,
58
- wait: resolved.wait,
59
- timeoutMs: resolved.timeoutMs,
60
- ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
61
- ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
62
- rank: resolved.rank,
63
- ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
64
- ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
65
- cache,
66
- health,
67
- ...(signal === undefined ? {} : { signal }),
68
- });
69
- if (sources.length === 0) {
70
- throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${failures.join('; ')})`);
156
+ const outcome = await runQuery(query, asked, signal);
157
+ if (outcome.sources.length === 0) {
158
+ throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
71
159
  }
72
- if (sources.length <= limit)
73
- return { sources, truncated: false };
74
- return { sources: sources.slice(0, limit), truncated: true };
160
+ return { sources: outcome.sources, truncated: outcome.truncated };
75
161
  },
76
162
  });
77
163
  // The same tunnel the search side resolves, offered to `web_fetch`: a search
@@ -79,9 +165,39 @@ export function apply(ctx, config = {}) {
79
165
  ctx.web.registerFetchProvider(createFetchProvider({
80
166
  id: FETCH_PROVIDER_ID,
81
167
  timeoutMs: mounted.timeoutMs,
168
+ format: () => resolveConfig(section, undefined).fetchFormat,
169
+ maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
82
170
  ...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
83
171
  ...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
84
172
  }));
173
+ // Optional half: `domain_search` and `search_many`. Fail-safe by
174
+ // construction — a deployment without `ctx.tools` still loads this plugin and
175
+ // keeps `web_search`; the registration path is the host's
176
+ // `ToolRuntime.register(definition)`.
177
+ try {
178
+ const tools = readToolRegistry(ctx);
179
+ if (tools !== undefined) {
180
+ registerDomainTool(tools, () => {
181
+ const live = resolveConfig(section, undefined);
182
+ const settings = {
183
+ sourceIds: live.domainSourceIds,
184
+ limit: live.domainLimit,
185
+ timeoutMs: live.domainTimeoutMs,
186
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
187
+ };
188
+ return settings;
189
+ }, { warn: warnTool });
190
+ // Batch tool: same optional-registry path, same fail-safe mounting. The
191
+ // runner is the `web_search` path, so both tools share lookups and cache.
192
+ const batch = registerSearchManyTool(tools, (query, limit, signal) => runQuery(query, limit, signal), { warn: warnTool });
193
+ if (batch !== undefined) {
194
+ onDispose(ctx, batch.dispose);
195
+ }
196
+ }
197
+ }
198
+ catch (error) {
199
+ warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
200
+ }
85
201
  }
86
202
  export { Config, resolveConfig } from "./config.js";
87
203
  /**
@@ -100,8 +216,13 @@ export { Config, resolveConfig } from "./config.js";
100
216
  export default Object.assign(apply, { Config, inject });
101
217
  export { HOSTS, search } from "./engines.js";
102
218
  export { SearchCache, cacheKey } from "./cache.js";
219
+ export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from "./cache-persist.js";
103
220
  export { EngineHealth } from "./health.js";
104
221
  export { SearchError, parseRetryAfter } from "./errors.js";
105
222
  export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
106
223
  export { fuse, interleave, dedupe, mergeRegions } from "./rank.js";
224
+ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublishedAt, publishedTimestamp, isWithinTimeRange, filterByTimeRange, } from "./freshness.js";
107
225
  export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
226
+ export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
227
+ export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
228
+ export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Dependency-free HTML → readable Markdown for `web_fetch`.
3
+ *
4
+ * A fetched page is mostly furniture: site chrome, a table of contents, ad
5
+ * slots, scripts. Handing that markup to a model wastes its budget on
6
+ * navigation and makes it parse tags instead of reading prose. This module
7
+ * keeps the parts a reader would keep and renders them as Markdown:
8
+ *
9
+ * 1. a tolerant tokenizer and tree builder, because real pages are not XML;
10
+ * 2. a pruning pass that drops chrome tags, comments and hidden nodes;
11
+ * 3. a body selection pass that prefers `<article>`, then `<main>`/`role=main`,
12
+ * then the densest text container;
13
+ * 4. a renderer for headings, paragraphs, lists, code, quotes, links, images,
14
+ * rules and tables, with entities decoded and URLs made absolute.
15
+ *
16
+ * It deliberately has no runtime dependency: the plugin ships as plain ESM and
17
+ * a parser package would be the first thing to break on a Node upgrade.
18
+ *
19
+ * @module @fangchenghao/dsh-web-search/markdown
20
+ */
21
+ /** Output style selected by `fetchFormat`. */
22
+ export type FetchFormat = 'markdown' | 'text';
23
+ /** Options shared by the Markdown and plain-text renderers. */
24
+ export interface RenderOptions {
25
+ /** `markdown` keeps syntax; `text` keeps only the words. */
26
+ readonly format?: FetchFormat | undefined;
27
+ /** Page URL, used to make relative links absolute. */
28
+ readonly baseUrl?: string | undefined;
29
+ }
30
+ /** Result of applying the configured character budget. */
31
+ export interface TruncatedText {
32
+ readonly content: string;
33
+ readonly truncated: boolean;
34
+ }
35
+ /** A converted page plus the numbers a truncation notice reports. */
36
+ export interface RenderedFetchText {
37
+ /** Converted content, cut to the budget when necessary. */
38
+ readonly content: string;
39
+ /** Length of the converted page before truncation. */
40
+ readonly originalLength: number;
41
+ /** Whether the character budget cut the page. */
42
+ readonly truncated: boolean;
43
+ }
44
+ /** Default `fetchMaxLength`: about 20k characters of prose. */
45
+ export declare const DEFAULT_FETCH_MAX_LENGTH = 20000;
46
+ /**
47
+ * Decode named and numeric HTML entities.
48
+ *
49
+ * Unknown named entities are preserved verbatim: dropping them would silently
50
+ * delete text, and guessing a codepoint would corrupt it.
51
+ *
52
+ * @param value - text or an attribute value.
53
+ * @returns the decoded text.
54
+ */
55
+ export declare function decodeEntities(value: string): string;
56
+ /**
57
+ * Convert an HTML document or fragment to readable Markdown or plain text.
58
+ *
59
+ * @param html - the raw markup.
60
+ * @param options - output format and the base URL for relative links.
61
+ * @returns the converted text; an empty or contentless document yields `''`.
62
+ */
63
+ export declare function htmlToMarkdown(html: string, options?: RenderOptions): string;
64
+ /**
65
+ * Apply the output character budget.
66
+ *
67
+ * The notice names both the configured limit and the original converted
68
+ * length, so a reader knows how much is missing and can ask for a narrower URL.
69
+ *
70
+ * @param content - converted content.
71
+ * @param maxLength - configured character budget.
72
+ * @returns the content, cut with a notice when it exceeded the budget.
73
+ */
74
+ export declare function truncateFetchText(content: string, maxLength: number): TruncatedText;
75
+ /**
76
+ * Convert a fetched HTML page and apply the configured budget in one step.
77
+ *
78
+ * @param html - the page markup.
79
+ * @param options - format, base URL and character budget.
80
+ * @returns converted content plus the pre-truncation length and cut flag.
81
+ */
82
+ export declare function renderFetchText(html: string, options: RenderOptions & {
83
+ readonly maxLength: number;
84
+ }): RenderedFetchText;