@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +247 -41
- package/README.zh-CN.md +247 -41
- package/client.js +66 -9
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +56 -8
- package/lib/cache.js +141 -13
- package/lib/config.d.ts +33 -1
- package/lib/config.js +54 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +44 -2
- package/lib/index.js +144 -23
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +22 -6
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
package/lib/types.d.ts
CHANGED
|
@@ -16,6 +16,11 @@ export interface Source {
|
|
|
16
16
|
}
|
|
17
17
|
/** Which language side an engine leads. `both` leads either side. */
|
|
18
18
|
export type Region = 'global' | 'cn' | 'both';
|
|
19
|
+
/**
|
|
20
|
+
* How old a result may be: `off` disables time filtering, the rest are rolling
|
|
21
|
+
* windows measured back from the moment of the search.
|
|
22
|
+
*/
|
|
23
|
+
export type TimeRange = 'off' | 'day' | 'week' | 'month' | 'year';
|
|
19
24
|
/** Engine priority tier: fallback engines run only when no primary answered. */
|
|
20
25
|
export type Tier = 'primary' | 'fallback';
|
|
21
26
|
/** How long a tier waits for its engines to answer. */
|
|
@@ -41,8 +46,8 @@ export interface HostSpec {
|
|
|
41
46
|
skipEcho?: boolean;
|
|
42
47
|
/** Confirm the payload really belongs to the query (scraped pages only). */
|
|
43
48
|
echoesQuery?: (payload: string, query: string) => boolean;
|
|
44
|
-
/** Absolute request URL for a query. */
|
|
45
|
-
build: (query: string) => string;
|
|
49
|
+
/** Absolute request URL for a query. `timeRange` may be pushed down when the engine supports it. */
|
|
50
|
+
build: (query: string, timeRange?: TimeRange) => string;
|
|
46
51
|
/** Request body for a JSON POST. */
|
|
47
52
|
body?: (query: string) => string;
|
|
48
53
|
/** Extra request headers, for a provider that authenticates. */
|
|
@@ -80,6 +85,8 @@ export interface SearchOptions {
|
|
|
80
85
|
proxy?: string;
|
|
81
86
|
/** Restrict the run to these engine ids. */
|
|
82
87
|
hostIds?: string[];
|
|
88
|
+
/** Rolling window results must fall in; `off` disables filtering. */
|
|
89
|
+
timeRange?: TimeRange;
|
|
83
90
|
/** Cancellation from the caller. */
|
|
84
91
|
signal?: AbortSignal;
|
|
85
92
|
}
|
|
@@ -102,15 +109,24 @@ export interface RankOptions {
|
|
|
102
109
|
freshnessBoost?: number;
|
|
103
110
|
/** Age in days after which the freshness boost fades to zero. */
|
|
104
111
|
freshnessDays?: number;
|
|
112
|
+
/**
|
|
113
|
+
* Active time window. Ranking never drops rows (the engine layer does), but
|
|
114
|
+
* when a window is active a row without a usable date is scored lower so a
|
|
115
|
+
* similarly relevant dated row wins.
|
|
116
|
+
*/
|
|
117
|
+
timeRange?: TimeRange;
|
|
105
118
|
/** Per-engine weight in the pre-fusion interleave. */
|
|
106
119
|
weights?: number[];
|
|
107
120
|
}
|
|
108
121
|
/** Cache knobs. */
|
|
109
122
|
export interface CacheOptions {
|
|
110
|
-
/**
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
123
|
+
/**
|
|
124
|
+
* Entry lifetime in milliseconds. `0` disables the cache. A reader keeps
|
|
125
|
+
* Settings edits live without a remount.
|
|
126
|
+
*/
|
|
127
|
+
ttlMs?: number | (() => number);
|
|
128
|
+
/** Maximum retained entries, or a reader for it; the least recently used goes. */
|
|
129
|
+
maxEntries?: number | (() => number);
|
|
114
130
|
}
|
|
115
131
|
/** Health knobs for the per-engine cooldown. */
|
|
116
132
|
export interface HealthOptions {
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Key-free vertical (domain-specific) search sources.
|
|
3
|
+
*
|
|
4
|
+
* `web_search` answers "what is on the open web". This module answers the
|
|
5
|
+
* narrower question a coding agent asks far more often: "which package, which
|
|
6
|
+
* paper, which repository, which Q&A thread". Every endpoint here is a public,
|
|
7
|
+
* documented, credential-free API, so the feature costs no key and no account.
|
|
8
|
+
*
|
|
9
|
+
* A source is a build/parse pair: {@link VerticalSpec.build} turns the query
|
|
10
|
+
* into an absolute URL, and the parser turns the served body into the same
|
|
11
|
+
* {@link Source} shape `web_search` returns. Parsers never throw on markup or
|
|
12
|
+
* payload drift — a source that changes shape yields an empty list so the other
|
|
13
|
+
* sources still answer. Timeouts, status classification and failure isolation
|
|
14
|
+
* live in {@link searchVerticals}, not in the parsers.
|
|
15
|
+
*
|
|
16
|
+
* @module @fangchenghao/dsh-web-search/verticals
|
|
17
|
+
*/
|
|
18
|
+
import type { Source } from './types.ts';
|
|
19
|
+
/** Identifiers accepted in `sources` and in the `domainSources` config field. */
|
|
20
|
+
export type VerticalId = 'github' | 'npm' | 'pypi' | 'arxiv' | 'wikipedia' | 'hn' | 'stackexchange' | 'crates';
|
|
21
|
+
/** Every supported source, in the order results are interleaved. */
|
|
22
|
+
export declare const VERTICAL_IDS: readonly VerticalId[];
|
|
23
|
+
/** Default and maximum result budget for one source. */
|
|
24
|
+
export declare const DEFAULT_VERTICAL_LIMIT = 5;
|
|
25
|
+
/** Upper bound the tool and the adapters both respect. */
|
|
26
|
+
export declare const MAX_VERTICAL_LIMIT = 20;
|
|
27
|
+
/** One vertical source definition. */
|
|
28
|
+
export interface VerticalSpec {
|
|
29
|
+
/** Stable id, also what configuration and tool arguments name. */
|
|
30
|
+
id: VerticalId;
|
|
31
|
+
/** Human label used in failure messages and smoke output. */
|
|
32
|
+
label: string;
|
|
33
|
+
/** `Accept` header this endpoint serves. */
|
|
34
|
+
accept: string;
|
|
35
|
+
/**
|
|
36
|
+
* Treat HTTP 404 as "no such record" rather than a failure. Only PyPI needs
|
|
37
|
+
* this: its endpoint is an exact-package lookup, and a missing package is a
|
|
38
|
+
* legitimate empty answer, not a broken source.
|
|
39
|
+
*/
|
|
40
|
+
notFoundIsEmpty?: boolean;
|
|
41
|
+
/** Absolute request URL for a query and result budget. */
|
|
42
|
+
build: (query: string, limit: number) => string;
|
|
43
|
+
/** Extra request headers, merged over the defaults. */
|
|
44
|
+
headers?: Record<string, string>;
|
|
45
|
+
/** Turn the served body into sources; must never throw on drift. */
|
|
46
|
+
parse: (payload: string, finalUrl: string, limit: number) => Source[];
|
|
47
|
+
}
|
|
48
|
+
/** `fetch`-compatible function, injectable so tests never touch the network. */
|
|
49
|
+
export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
|
|
50
|
+
/** Options for {@link searchVerticals}. */
|
|
51
|
+
export interface VerticalSearchOptions {
|
|
52
|
+
/** Source ids to query; empty or absent means every source. */
|
|
53
|
+
ids?: readonly string[];
|
|
54
|
+
/** Results wanted per source; clamped to 1…20. */
|
|
55
|
+
limit?: number;
|
|
56
|
+
/** Per-source deadline in milliseconds. */
|
|
57
|
+
timeoutMs?: number;
|
|
58
|
+
/** Caller cancellation, propagated to every source. */
|
|
59
|
+
signal?: AbortSignal;
|
|
60
|
+
/** Tunnel URL from the plugin's `proxy` setting, when configured. */
|
|
61
|
+
proxy?: string;
|
|
62
|
+
/** Test seam; defaults to the platform `fetch`. */
|
|
63
|
+
fetch?: FetchLike;
|
|
64
|
+
}
|
|
65
|
+
/** What one vertical run reports. */
|
|
66
|
+
export interface VerticalOutcome {
|
|
67
|
+
/** Merged, de-duplicated sources, capped at the caller's budget. */
|
|
68
|
+
sources: Source[];
|
|
69
|
+
/** One `id: reason` line per source that failed. */
|
|
70
|
+
failures: string[];
|
|
71
|
+
/** Parsed row count per source, before the global cap. */
|
|
72
|
+
counts: Record<string, number>;
|
|
73
|
+
/** Whether rows were dropped because of the global cap. */
|
|
74
|
+
truncated: boolean;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Parse GitHub's repository search envelope.
|
|
78
|
+
* @param payload - `api.github.com/search/repositories` JSON.
|
|
79
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
80
|
+
* @param limit - rows to keep.
|
|
81
|
+
* @returns repositories as sources, in API order.
|
|
82
|
+
*/
|
|
83
|
+
export declare function parseGithub(payload: string, _url: string, limit?: number): Source[];
|
|
84
|
+
/**
|
|
85
|
+
* Parse npm's registry search envelope.
|
|
86
|
+
* @param payload - `registry.npmjs.org/-/v1/search` JSON.
|
|
87
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
88
|
+
* @param limit - rows to keep.
|
|
89
|
+
* @returns packages as sources.
|
|
90
|
+
*/
|
|
91
|
+
export declare function parseNpm(payload: string, _url: string, limit?: number): Source[];
|
|
92
|
+
/**
|
|
93
|
+
* Parse PyPI's exact-package envelope.
|
|
94
|
+
*
|
|
95
|
+
* PyPI no longer publishes a credential-free search API, so the adapter asks
|
|
96
|
+
* for the query as a package name. A 404 is handled by the orchestrator
|
|
97
|
+
* (`notFoundIsEmpty`) and never reaches this parser; a body without `info`
|
|
98
|
+
* still parses to `[]`.
|
|
99
|
+
* @param payload - `pypi.org/pypi/<name>/json`.
|
|
100
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
101
|
+
* @param limit - rows to keep (PyPI answers one).
|
|
102
|
+
* @returns the package as one source, or `[]`.
|
|
103
|
+
*/
|
|
104
|
+
export declare function parsePypi(payload: string, _url: string, limit?: number): Source[];
|
|
105
|
+
/**
|
|
106
|
+
* Parse the arXiv Atom feed.
|
|
107
|
+
* @param payload - `export.arxiv.org/api/query` XML.
|
|
108
|
+
* @param _url - request URL, unused by this XML shape.
|
|
109
|
+
* @param limit - entries to keep.
|
|
110
|
+
* @returns papers as sources, in feed order.
|
|
111
|
+
*/
|
|
112
|
+
export declare function parseArxiv(payload: string, _url: string, limit?: number): Source[];
|
|
113
|
+
/**
|
|
114
|
+
* Parse Wikipedia's REST search envelope (zh or en host).
|
|
115
|
+
* @param payload - `*.wikipedia.org/w/rest.php/v1/search/page` JSON.
|
|
116
|
+
* @param finalUrl - the request URL, whose host decides the wiki language.
|
|
117
|
+
* @param limit - pages to keep.
|
|
118
|
+
* @returns wiki pages as sources.
|
|
119
|
+
*/
|
|
120
|
+
export declare function parseWikipedia(payload: string, finalUrl: string, limit?: number): Source[];
|
|
121
|
+
/**
|
|
122
|
+
* Parse Hacker News's Algolia search envelope.
|
|
123
|
+
* @param payload - `hn.algolia.com/api/v1/search` JSON.
|
|
124
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
125
|
+
* @param limit - hits to keep.
|
|
126
|
+
* @returns stories or comments as sources.
|
|
127
|
+
*/
|
|
128
|
+
export declare function parseHn(payload: string, _url: string, limit?: number): Source[];
|
|
129
|
+
/**
|
|
130
|
+
* Parse Stack Exchange's advanced-search envelope.
|
|
131
|
+
* @param payload - `api.stackexchange.com/2.3/search/advanced` JSON.
|
|
132
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
133
|
+
* @param limit - questions to keep.
|
|
134
|
+
* @returns questions as sources.
|
|
135
|
+
*/
|
|
136
|
+
export declare function parseStackExchange(payload: string, _url: string, limit?: number): Source[];
|
|
137
|
+
/**
|
|
138
|
+
* Parse crates.io's crate search envelope.
|
|
139
|
+
* @param payload - `crates.io/api/v1/crates` JSON.
|
|
140
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
141
|
+
* @param limit - crates to keep.
|
|
142
|
+
* @returns crates as sources.
|
|
143
|
+
*/
|
|
144
|
+
export declare function parseCrates(payload: string, _url: string, limit?: number): Source[];
|
|
145
|
+
/** A source id that this module actually knows. */
|
|
146
|
+
export declare function isVerticalId(value: unknown): value is VerticalId;
|
|
147
|
+
/** Every adapter, keyed by id. */
|
|
148
|
+
export declare const VERTICALS: readonly VerticalSpec[];
|
|
149
|
+
/** One adapter by id, or `undefined` for a typo. */
|
|
150
|
+
export declare function verticalById(id: string): VerticalSpec | undefined;
|
|
151
|
+
/**
|
|
152
|
+
* Query every requested vertical concurrently and merge the answers.
|
|
153
|
+
*
|
|
154
|
+
* One source failing never fails the run: its reason is recorded in
|
|
155
|
+
* `failures`, its count stays `0`, and the other sources still answer. Answers
|
|
156
|
+
* are interleaved by rank (one row from each source, then the second row from
|
|
157
|
+
* each) so a single prolific source cannot crowd the others out.
|
|
158
|
+
*
|
|
159
|
+
* @param query - what to search for; must be non-empty.
|
|
160
|
+
* @param options - sources, budget, deadline, cancellation, tunnel, fetch seam.
|
|
161
|
+
* @returns merged sources plus per-source counts and failures.
|
|
162
|
+
* @throws SearchError `SEARCH_ABORTED` when the caller cancelled the run.
|
|
163
|
+
*/
|
|
164
|
+
export declare function searchVerticals(query: string, options?: VerticalSearchOptions): Promise<VerticalOutcome>;
|