@fffangchenghao/dsh-web-search 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/fetch.js CHANGED
@@ -24,6 +24,7 @@
24
24
  * @module @fangchenghao/dsh-web-search/fetch
25
25
  */
26
26
  import { SearchError } from "./errors.js";
27
+ import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
27
28
  import { proxyGet, resolveProxy } from "./proxy.js";
28
29
  /** Default user agent; kept in step with the search side. */
29
30
  const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
@@ -126,6 +127,38 @@ export function createFetchProvider(options = {}, overrides = {}) {
126
127
  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
127
128
  const userAgent = options.userAgent ?? USER_AGENT;
128
129
  const apiKey = (options.anysearchApiKey ?? '').trim();
130
+ /**
131
+ * Resolve the two volatile Settings fields per request. Reading them here
132
+ * rather than at construction is what lets a saved edit apply without a
133
+ * remount; caching them made the Settings page lie.
134
+ */
135
+ const liveFormat = () => typeof options.format === 'function' ? options.format() : (options.format ?? 'markdown');
136
+ const liveMaxLength = () => {
137
+ const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
138
+ return raw ?? DEFAULT_FETCH_MAX_LENGTH;
139
+ };
140
+ // Every rung funnels through here: an HTML body becomes Markdown (or plain
141
+ // text), a text body keeps its content, and both obey the character budget.
142
+ // The Host's own renderer then sees `kind: 'text'`, so it does not convert the
143
+ // same page a second time.
144
+ const finalize = (outcome) => {
145
+ const format = liveFormat();
146
+ const maxLength = liveMaxLength();
147
+ if (outcome.body.kind === 'html') {
148
+ const rendered = renderFetchText(outcome.body.content, { format, maxLength, baseUrl: outcome.url });
149
+ return {
150
+ ...outcome,
151
+ body: { kind: 'text', content: rendered.content },
152
+ truncated: outcome.truncated || rendered.truncated,
153
+ };
154
+ }
155
+ const limited = truncateFetchText(outcome.body.content, maxLength);
156
+ return {
157
+ ...outcome,
158
+ body: { kind: 'text', content: limited.content },
159
+ truncated: outcome.truncated || limited.truncated,
160
+ };
161
+ };
129
162
  return {
130
163
  id: options.id ?? 'http-proxy',
131
164
  available: () => true,
@@ -141,7 +174,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
141
174
  // shorter deadline than the tunnel: spending the whole budget on a request
142
175
  // that cannot work would leave the rung that can work with nothing.
143
176
  try {
144
- return await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
177
+ return finalize(await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes));
145
178
  }
146
179
  catch (error) {
147
180
  if (cancelled())
@@ -159,7 +192,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
159
192
  }
160
193
  if (proxy !== undefined && proxy.length > 0) {
161
194
  try {
162
- return await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
195
+ return finalize(await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes));
163
196
  }
164
197
  catch (error) {
165
198
  if (cancelled())
@@ -169,7 +202,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
169
202
  }
170
203
  if (apiKey.length > 0) {
171
204
  try {
172
- return await deps.extract(target, apiKey, timeoutMs, signal);
205
+ return finalize(await deps.extract(target, apiKey, timeoutMs, signal));
173
206
  }
174
207
  catch (error) {
175
208
  if (cancelled())
package/lib/index.d.ts CHANGED
@@ -39,6 +39,13 @@ export interface PluginContext {
39
39
  get?: (name: string) => unknown;
40
40
  /** Fallback for tests and hosts that expose the registry as a property. */
41
41
  tools?: unknown;
42
+ /**
43
+ * Cordis lifecycle hook. `effect` is the canonical registration; `on` is the
44
+ * older/test fallback. Both let the tool registration and the cache release
45
+ * their resources when the plugin is disposed.
46
+ */
47
+ effect?: (execute: () => (() => void) | void) => unknown;
48
+ on?: (event: string, listener: () => void) => unknown;
42
49
  }
43
50
  /** Configuration as the loader hands it over, already schema-validated. */
44
51
  export type ApplyConfig = PluginConfig;
@@ -65,6 +72,8 @@ declare const _default: typeof apply & {
65
72
  Config: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<NoInfer<{
66
73
  hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
67
74
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
75
+ fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
76
+ fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
68
77
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
69
78
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
70
79
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -76,6 +85,8 @@ declare const _default: typeof apply & {
76
85
  timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
77
86
  cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
78
87
  cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
88
+ cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
89
+ cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
79
90
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
80
91
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
81
92
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -86,6 +97,8 @@ declare const _default: typeof apply & {
86
97
  }>>, Schemastery.ObjectT<NoInfer<{
87
98
  hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
88
99
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
100
+ fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
101
+ fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
89
102
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
90
103
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
91
104
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -97,6 +110,8 @@ declare const _default: typeof apply & {
97
110
  timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
98
111
  cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
99
112
  cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
113
+ cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
114
+ cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
100
115
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
101
116
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
102
117
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -110,6 +125,9 @@ declare const _default: typeof apply & {
110
125
  export default _default;
111
126
  export { HOSTS, search } from './engines.ts';
112
127
  export { SearchCache, cacheKey } from './cache.ts';
128
+ export type { CachePersistOptions, SearchCacheOptions } from './cache.ts';
129
+ export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from './cache-persist.ts';
130
+ export type { CacheFileIo, CacheWriteOptions, PersistedEntry, SerializedCache } from './cache-persist.ts';
113
131
  export { EngineHealth } from './health.ts';
114
132
  export { SearchError, parseRetryAfter } from './errors.ts';
115
133
  export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from './parsers.ts';
@@ -120,4 +138,6 @@ export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, p
120
138
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
121
139
  export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
122
140
  export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
141
+ export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
142
+ export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
123
143
  export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
package/lib/index.js CHANGED
@@ -15,6 +15,7 @@ import { SearchCache } from "./cache.js";
15
15
  import { EngineHealth } from "./health.js";
16
16
  import { createFetchProvider } from "./fetch.js";
17
17
  import { registerDomainTool } from "./domain-tool.js";
18
+ import { registerSearchManyTool } from "./search-many.js";
18
19
  export const name = 'dsh-web-search';
19
20
  /** The `ctx.web` service this plugin registers a search provider with. */
20
21
  export const inject = ['web'];
@@ -53,6 +54,26 @@ function warnTool(message) {
53
54
  // Logging must never be the reason a plugin fails to mount.
54
55
  }
55
56
  }
57
+ /**
58
+ * Register a cleanup with the plugin lifecycle.
59
+ *
60
+ * `ctx.effect(() => cleanup)` is Cordis's ownership primitive; a host that
61
+ * predates it (or a test double) may only expose a `dispose` event, so that is
62
+ * the fallback. Losing the hook must not break mounting.
63
+ */
64
+ function onDispose(ctx, cleanup) {
65
+ try {
66
+ if (typeof ctx.effect === 'function') {
67
+ ctx.effect(() => cleanup);
68
+ return;
69
+ }
70
+ if (typeof ctx.on === 'function')
71
+ ctx.on('dispose', cleanup);
72
+ }
73
+ catch (error) {
74
+ warnTool(`dispose hook skipped: ${error instanceof Error ? error.message : String(error)}`);
75
+ }
76
+ }
56
77
  /**
57
78
  * Register the search provider.
58
79
  * @param ctx - plugin context carrying the `web` service.
@@ -66,15 +87,63 @@ export function apply(ctx, config = {}) {
66
87
  // One cache and one health tracker per mounted plugin, so a config change
67
88
  // through a restart starts from a clean slate instead of inheriting the
68
89
  // cooldowns of a previous instance.
69
- const cache = new SearchCache({ ttlMs: mounted.cacheTtlMs, maxEntries: mounted.cacheEntries });
90
+ const cache = new SearchCache({
91
+ // Read through on every use: a Settings edit must take effect without a
92
+ // remount, which is the whole point of marking these fields volatile.
93
+ ttlMs: () => resolveConfig(section, undefined).cacheTtlMs,
94
+ maxEntries: () => resolveConfig(section, undefined).cacheEntries,
95
+ ...(mounted.cachePersist
96
+ ? { persist: { ...(mounted.cacheFile === undefined ? {} : { file: mounted.cacheFile }) } }
97
+ : {}),
98
+ });
70
99
  const health = new EngineHealth({ baseCooldownMs: mounted.baseCooldownMs, maxCooldownMs: mounted.maxCooldownMs });
100
+ // The disk snapshot is flushed at plugin disposal; the cache also flushes on
101
+ // its own exit hook, so an abrupt process end keeps the newest state.
102
+ onDispose(ctx, () => cache.close());
103
+ /**
104
+ * Run one query through the same engine path as `web_search`.
105
+ *
106
+ * Reading `section` on every call is what makes a Settings commit visible
107
+ * without a restart. The configured limit is the hard ceiling: a caller may
108
+ * ask for fewer results, never for more.
109
+ */
110
+ const runQuery = async (query, requested, signal) => {
111
+ const resolved = resolveConfig(section, undefined);
112
+ if (typeof query !== 'string' || query.trim().length === 0) {
113
+ throw new Error('web search requires a non-empty query');
114
+ }
115
+ if (signal?.aborted === true)
116
+ throw new Error('web search was cancelled');
117
+ const asked = typeof requested === 'number' && Number.isInteger(requested) && requested > 0
118
+ ? requested
119
+ : undefined;
120
+ const limit = Math.min(asked ?? resolved.limit, resolved.limit);
121
+ const { sources, failures } = await search(query, signal, {
122
+ limit,
123
+ wait: resolved.wait,
124
+ timeRange: resolved.timeRange,
125
+ timeoutMs: resolved.timeoutMs,
126
+ ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
127
+ ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
128
+ rank: resolved.rank,
129
+ ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
130
+ ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
131
+ cache,
132
+ health,
133
+ ...(signal === undefined ? {} : { signal }),
134
+ });
135
+ return {
136
+ sources,
137
+ failures,
138
+ truncated: asked !== undefined && asked > resolved.limit,
139
+ };
140
+ };
71
141
  ctx.web.registerSearchProvider({
72
142
  id: PROVIDER_ID,
73
143
  available: () => true,
74
144
  async search(request, signal) {
75
145
  // A Settings commit only mutates the existing references; reading the
76
- // section here is what makes the edit apply without a restart.
77
- const resolved = resolveConfig(section, undefined);
146
+ // section inside `runQuery` is what makes the edit apply without a restart.
78
147
  const query = request.query;
79
148
  if (typeof query !== 'string' || query.trim().length === 0) {
80
149
  throw new Error('web search requires a non-empty query');
@@ -84,27 +153,11 @@ export function apply(ctx, config = {}) {
84
153
  const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
85
154
  ? request.maxResults
86
155
  : undefined;
87
- const limit = Math.min(asked ?? resolved.limit, resolved.limit);
88
- const { sources, failures } = await search(query, signal, {
89
- limit,
90
- wait: resolved.wait,
91
- timeRange: resolved.timeRange,
92
- timeoutMs: resolved.timeoutMs,
93
- ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
94
- ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
95
- rank: resolved.rank,
96
- ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
97
- ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
98
- cache,
99
- health,
100
- ...(signal === undefined ? {} : { signal }),
101
- });
102
- if (sources.length === 0) {
103
- throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${failures.join('; ')})`);
156
+ const outcome = await runQuery(query, asked, signal);
157
+ if (outcome.sources.length === 0) {
158
+ throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
104
159
  }
105
- if (sources.length <= limit)
106
- return { sources, truncated: false };
107
- return { sources: sources.slice(0, limit), truncated: true };
160
+ return { sources: outcome.sources, truncated: outcome.truncated };
108
161
  },
109
162
  });
110
163
  // The same tunnel the search side resolves, offered to `web_fetch`: a search
@@ -112,12 +165,15 @@ export function apply(ctx, config = {}) {
112
165
  ctx.web.registerFetchProvider(createFetchProvider({
113
166
  id: FETCH_PROVIDER_ID,
114
167
  timeoutMs: mounted.timeoutMs,
168
+ format: () => resolveConfig(section, undefined).fetchFormat,
169
+ maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
115
170
  ...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
116
171
  ...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
117
172
  }));
118
- // Optional half: `domain_search`. Fail-safe by construction — a deployment
119
- // without `ctx.tools` still loads this plugin and keeps `web_search`; the
120
- // registration path is the host's `ToolRuntime.register(definition)`.
173
+ // Optional half: `domain_search` and `search_many`. Fail-safe by
174
+ // construction — a deployment without `ctx.tools` still loads this plugin and
175
+ // keeps `web_search`; the registration path is the host's
176
+ // `ToolRuntime.register(definition)`.
121
177
  try {
122
178
  const tools = readToolRegistry(ctx);
123
179
  if (tools !== undefined) {
@@ -131,10 +187,16 @@ export function apply(ctx, config = {}) {
131
187
  };
132
188
  return settings;
133
189
  }, { warn: warnTool });
190
+ // Batch tool: same optional-registry path, same fail-safe mounting. The
191
+ // runner is the `web_search` path, so both tools share lookups and cache.
192
+ const batch = registerSearchManyTool(tools, (query, limit, signal) => runQuery(query, limit, signal), { warn: warnTool });
193
+ if (batch !== undefined) {
194
+ onDispose(ctx, batch.dispose);
195
+ }
134
196
  }
135
197
  }
136
198
  catch (error) {
137
- warnTool(`domain_search skipped: ${error instanceof Error ? error.message : String(error)}`);
199
+ warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
138
200
  }
139
201
  }
140
202
  export { Config, resolveConfig } from "./config.js";
@@ -154,6 +216,7 @@ export { Config, resolveConfig } from "./config.js";
154
216
  export default Object.assign(apply, { Config, inject });
155
217
  export { HOSTS, search } from "./engines.js";
156
218
  export { SearchCache, cacheKey } from "./cache.js";
219
+ export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from "./cache-persist.js";
157
220
  export { EngineHealth } from "./health.js";
158
221
  export { SearchError, parseRetryAfter } from "./errors.js";
159
222
  export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
@@ -162,3 +225,4 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
162
225
  export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
163
226
  export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
164
227
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
228
+ export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Dependency-free HTML → readable Markdown for `web_fetch`.
3
+ *
4
+ * A fetched page is mostly furniture: site chrome, a table of contents, ad
5
+ * slots, scripts. Handing that markup to a model wastes its budget on
6
+ * navigation and makes it parse tags instead of reading prose. This module
7
+ * keeps the parts a reader would keep and renders them as Markdown:
8
+ *
9
+ * 1. a tolerant tokenizer and tree builder, because real pages are not XML;
10
+ * 2. a pruning pass that drops chrome tags, comments and hidden nodes;
11
+ * 3. a body selection pass that prefers `<article>`, then `<main>`/`role=main`,
12
+ * then the densest text container;
13
+ * 4. a renderer for headings, paragraphs, lists, code, quotes, links, images,
14
+ * rules and tables, with entities decoded and URLs made absolute.
15
+ *
16
+ * It deliberately has no runtime dependency: the plugin ships as plain ESM and
17
+ * a parser package would be the first thing to break on a Node upgrade.
18
+ *
19
+ * @module @fangchenghao/dsh-web-search/markdown
20
+ */
21
+ /** Output style selected by `fetchFormat`. */
22
+ export type FetchFormat = 'markdown' | 'text';
23
+ /** Options shared by the Markdown and plain-text renderers. */
24
+ export interface RenderOptions {
25
+ /** `markdown` keeps syntax; `text` keeps only the words. */
26
+ readonly format?: FetchFormat | undefined;
27
+ /** Page URL, used to make relative links absolute. */
28
+ readonly baseUrl?: string | undefined;
29
+ }
30
+ /** Result of applying the configured character budget. */
31
+ export interface TruncatedText {
32
+ readonly content: string;
33
+ readonly truncated: boolean;
34
+ }
35
+ /** A converted page plus the numbers a truncation notice reports. */
36
+ export interface RenderedFetchText {
37
+ /** Converted content, cut to the budget when necessary. */
38
+ readonly content: string;
39
+ /** Length of the converted page before truncation. */
40
+ readonly originalLength: number;
41
+ /** Whether the character budget cut the page. */
42
+ readonly truncated: boolean;
43
+ }
44
+ /** Default `fetchMaxLength`: about 20k characters of prose. */
45
+ export declare const DEFAULT_FETCH_MAX_LENGTH = 20000;
46
+ /**
47
+ * Decode named and numeric HTML entities.
48
+ *
49
+ * Unknown named entities are preserved verbatim: dropping them would silently
50
+ * delete text, and guessing a codepoint would corrupt it.
51
+ *
52
+ * @param value - text or an attribute value.
53
+ * @returns the decoded text.
54
+ */
55
+ export declare function decodeEntities(value: string): string;
56
+ /**
57
+ * Convert an HTML document or fragment to readable Markdown or plain text.
58
+ *
59
+ * @param html - the raw markup.
60
+ * @param options - output format and the base URL for relative links.
61
+ * @returns the converted text; an empty or contentless document yields `''`.
62
+ */
63
+ export declare function htmlToMarkdown(html: string, options?: RenderOptions): string;
64
+ /**
65
+ * Apply the output character budget.
66
+ *
67
+ * The notice names both the configured limit and the original converted
68
+ * length, so a reader knows how much is missing and can ask for a narrower URL.
69
+ *
70
+ * @param content - converted content.
71
+ * @param maxLength - configured character budget.
72
+ * @returns the content, cut with a notice when it exceeded the budget.
73
+ */
74
+ export declare function truncateFetchText(content: string, maxLength: number): TruncatedText;
75
+ /**
76
+ * Convert a fetched HTML page and apply the configured budget in one step.
77
+ *
78
+ * @param html - the page markup.
79
+ * @param options - format, base URL and character budget.
80
+ * @returns converted content plus the pre-truncation length and cut flag.
81
+ */
82
+ export declare function renderFetchText(html: string, options: RenderOptions & {
83
+ readonly maxLength: number;
84
+ }): RenderedFetchText;