@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +307 -46
- package/README.zh-CN.md +307 -46
- package/client.js +44 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +25 -0
- package/lib/config.js +42 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +33 -0
- package/lib/fetch.js +218 -6
- package/lib/index.d.ts +26 -0
- package/lib/index.js +113 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +13 -4
- package/package.json +1 -1
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dependency-free HTML → readable Markdown for `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* A fetched page is mostly furniture: site chrome, a table of contents, ad
|
|
5
|
+
* slots, scripts. Handing that markup to a model wastes its budget on
|
|
6
|
+
* navigation and makes it parse tags instead of reading prose. This module
|
|
7
|
+
* keeps the parts a reader would keep and renders them as Markdown:
|
|
8
|
+
*
|
|
9
|
+
* 1. a tolerant tokenizer and tree builder, because real pages are not XML;
|
|
10
|
+
* 2. a pruning pass that drops chrome tags, comments and hidden nodes;
|
|
11
|
+
* 3. a body selection pass that prefers `<article>`, then `<main>`/`role=main`,
|
|
12
|
+
* then the densest text container;
|
|
13
|
+
* 4. a renderer for headings, paragraphs, lists, code, quotes, links, images,
|
|
14
|
+
* rules and tables, with entities decoded and URLs made absolute.
|
|
15
|
+
*
|
|
16
|
+
* It deliberately has no runtime dependency: the plugin ships as plain ESM and
|
|
17
|
+
* a parser package would be the first thing to break on a Node upgrade.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/markdown
|
|
20
|
+
*/
|
|
21
|
+
/** Output style selected by `fetchFormat`. */
|
|
22
|
+
export type FetchFormat = 'markdown' | 'text';
|
|
23
|
+
/** Options shared by the Markdown and plain-text renderers. */
|
|
24
|
+
export interface RenderOptions {
|
|
25
|
+
/** `markdown` keeps syntax; `text` keeps only the words. */
|
|
26
|
+
readonly format?: FetchFormat | undefined;
|
|
27
|
+
/** Page URL, used to make relative links absolute. */
|
|
28
|
+
readonly baseUrl?: string | undefined;
|
|
29
|
+
}
|
|
30
|
+
/** Result of applying the configured character budget. */
|
|
31
|
+
export interface TruncatedText {
|
|
32
|
+
readonly content: string;
|
|
33
|
+
readonly truncated: boolean;
|
|
34
|
+
}
|
|
35
|
+
/** A converted page plus the numbers a truncation notice reports. */
|
|
36
|
+
export interface RenderedFetchText {
|
|
37
|
+
/** Converted content, cut to the budget when necessary. */
|
|
38
|
+
readonly content: string;
|
|
39
|
+
/** Length of the converted page before truncation. */
|
|
40
|
+
readonly originalLength: number;
|
|
41
|
+
/** Whether the character budget cut the page. */
|
|
42
|
+
readonly truncated: boolean;
|
|
43
|
+
}
|
|
44
|
+
/** Default `fetchMaxLength`: about 20k characters of prose. */
|
|
45
|
+
export declare const DEFAULT_FETCH_MAX_LENGTH = 20000;
|
|
46
|
+
/**
|
|
47
|
+
* Decode named and numeric HTML entities.
|
|
48
|
+
*
|
|
49
|
+
* Unknown named entities are preserved verbatim: dropping them would silently
|
|
50
|
+
* delete text, and guessing a codepoint would corrupt it.
|
|
51
|
+
*
|
|
52
|
+
* @param value - text or an attribute value.
|
|
53
|
+
* @returns the decoded text.
|
|
54
|
+
*/
|
|
55
|
+
export declare function decodeEntities(value: string): string;
|
|
56
|
+
/**
|
|
57
|
+
* Convert an HTML document or fragment to readable Markdown or plain text.
|
|
58
|
+
*
|
|
59
|
+
* @param html - the raw markup.
|
|
60
|
+
* @param options - output format and the base URL for relative links.
|
|
61
|
+
* @returns the converted text; an empty or contentless document yields `''`.
|
|
62
|
+
*/
|
|
63
|
+
export declare function htmlToMarkdown(html: string, options?: RenderOptions): string;
|
|
64
|
+
/**
|
|
65
|
+
* Apply the output character budget.
|
|
66
|
+
*
|
|
67
|
+
* The notice names both the configured limit and the original converted
|
|
68
|
+
* length, so a reader knows how much is missing and can ask for a narrower URL.
|
|
69
|
+
*
|
|
70
|
+
* @param content - converted content.
|
|
71
|
+
* @param maxLength - configured character budget.
|
|
72
|
+
* @returns the content, cut with a notice when it exceeded the budget.
|
|
73
|
+
*/
|
|
74
|
+
export declare function truncateFetchText(content: string, maxLength: number): TruncatedText;
|
|
75
|
+
/**
|
|
76
|
+
* Convert a fetched HTML page and apply the configured budget in one step.
|
|
77
|
+
*
|
|
78
|
+
* @param html - the page markup.
|
|
79
|
+
* @param options - format, base URL and character budget.
|
|
80
|
+
* @returns converted content plus the pre-truncation length and cut flag.
|
|
81
|
+
*/
|
|
82
|
+
export declare function renderFetchText(html: string, options: RenderOptions & {
|
|
83
|
+
readonly maxLength: number;
|
|
84
|
+
}): RenderedFetchText;
|