@fffangchenghao/dsh-web-search 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/rank.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { publishedTimestamp } from "./freshness.js";
1
2
  /**
2
3
  * Reciprocal-rank-fusion constant over the interleaved slot order; a larger
3
4
  * value flattens the curve so authority and freshness reorder more freely.
@@ -33,6 +34,15 @@ export const DEFAULT_AUTHORITY_BOOST = 0.25;
33
34
  export const DEFAULT_FRESHNESS_BOOST = 0.2;
34
35
  /** Age in days after which the freshness boost has faded to zero. */
35
36
  export const DEFAULT_FRESHNESS_DAYS = 30;
37
+ /**
38
+ * Score factor for a result with no usable date while a time window is active.
39
+ *
40
+ * Ten percent is roughly six fusion positions at `rrfConstant = 60`: enough for
41
+ * a dated result to win a close comparison, small enough that an undated result
42
+ * still outranks a dated one that is clearly less relevant. With every result
43
+ * undated the factor is uniform, so the order is unchanged.
44
+ */
45
+ export const UNDATED_WHEN_FILTERING_FACTOR = 0.9;
36
46
  /**
37
47
  * Scores closer together than this count as equal, so the deterministic
38
48
  * tie-break decides instead of floating-point summation order.
@@ -183,6 +193,7 @@ function resolveOptions(options) {
183
193
  authorityBoost: nonNegativeOr(options?.authorityBoost, DEFAULT_AUTHORITY_BOOST),
184
194
  freshnessBoost: nonNegativeOr(options?.freshnessBoost, DEFAULT_FRESHNESS_BOOST),
185
195
  freshnessDays: positiveOr(options?.freshnessDays, DEFAULT_FRESHNESS_DAYS),
196
+ timeRange: options?.timeRange,
186
197
  weights: Array.isArray(options?.weights) ? options.weights : undefined,
187
198
  };
188
199
  }
@@ -373,12 +384,19 @@ function fuseEntries(entries, limit, options) {
373
384
  if (candidates.size === 0)
374
385
  return [];
375
386
  const ranked = [...candidates.values()];
387
+ const filtering = config.timeRange !== undefined && config.timeRange !== 'off';
376
388
  for (const candidate of ranked) {
377
389
  const voteCount = votes.get(candidate.key) ?? 1;
378
390
  let score = voteCount / (config.rrfConstant + candidate.position);
379
391
  if (isAuthoritative(candidate.source.url, config.authorityDomains))
380
392
  score *= 1 + config.authorityBoost;
381
393
  score *= freshnessFactor(candidate.source.publishedAt, config.freshnessDays, config.freshnessBoost, now);
394
+ // A missing (or unparsable) date is not evidence of staleness, so the row
395
+ // is kept. While a window is active it is discounted instead of being
396
+ // dropped, which keeps the long tail of undated documentation visible.
397
+ if (filtering && publishedTimestamp(candidate.source.publishedAt) === undefined) {
398
+ score *= UNDATED_WHEN_FILTERING_FACTOR;
399
+ }
382
400
  candidate.score = score;
383
401
  }
384
402
  ranked.sort(compareRanked);
@@ -8,7 +8,7 @@
8
8
  *
9
9
  * @module @fangchenghao/dsh-web-search/transport
10
10
  */
11
- import type { HostSpec, Source } from './types.ts';
11
+ import type { HostSpec, Source, TimeRange } from './types.ts';
12
12
  import { SearchError } from './errors.ts';
13
13
  /** Outcome of asking one engine once. */
14
14
  export type AttemptOutcome = {
@@ -31,6 +31,8 @@ export type AttemptOutcome = {
31
31
  * @param signal - cancellation.
32
32
  * @param timeoutMs - per-attempt deadline.
33
33
  * @param proxy - tunnel URL, or `undefined` when none is available.
34
+ * @param timeRange - active publication window, forwarded to engines that can
35
+ * push it into their query (currently DuckDuckGo's `df`).
34
36
  * @returns the outcome of the route that got furthest; cancellation is rethrown.
35
37
  */
36
- export declare function searchHost(host: HostSpec, query: string, signal: AbortSignal | undefined, timeoutMs?: number, proxy?: string): Promise<AttemptOutcome>;
38
+ export declare function searchHost(host: HostSpec, query: string, signal: AbortSignal | undefined, timeoutMs?: number, proxy?: string, timeRange?: TimeRange): Promise<AttemptOutcome>;
package/lib/transport.js CHANGED
@@ -123,10 +123,12 @@ async function attempt(host, url, signal, timeoutMs, proxy, query) {
123
123
  * @param signal - cancellation.
124
124
  * @param timeoutMs - per-attempt deadline.
125
125
  * @param proxy - tunnel URL, or `undefined` when none is available.
126
+ * @param timeRange - active publication window, forwarded to engines that can
127
+ * push it into their query (currently DuckDuckGo's `df`).
126
128
  * @returns the outcome of the route that got furthest; cancellation is rethrown.
127
129
  */
128
- export async function searchHost(host, query, signal, timeoutMs = 15000, proxy) {
129
- const url = host.build(query);
130
+ export async function searchHost(host, query, signal, timeoutMs = 15000, proxy, timeRange) {
131
+ const url = host.build(query, timeRange);
130
132
  const route = host.viaProxy === true ? proxy : undefined;
131
133
  const first = await attempt(host, url, signal, timeoutMs, route, query);
132
134
  if (first.ok || route !== undefined || proxy === undefined || signal?.aborted === true)
package/lib/types.d.ts CHANGED
@@ -16,6 +16,11 @@ export interface Source {
16
16
  }
17
17
  /** Which language side an engine leads. `both` leads either side. */
18
18
  export type Region = 'global' | 'cn' | 'both';
19
+ /**
20
+ * How old a result may be: `off` disables time filtering, the rest are rolling
21
+ * windows measured back from the moment of the search.
22
+ */
23
+ export type TimeRange = 'off' | 'day' | 'week' | 'month' | 'year';
19
24
  /** Engine priority tier: fallback engines run only when no primary answered. */
20
25
  export type Tier = 'primary' | 'fallback';
21
26
  /** How long a tier waits for its engines to answer. */
@@ -41,8 +46,8 @@ export interface HostSpec {
41
46
  skipEcho?: boolean;
42
47
  /** Confirm the payload really belongs to the query (scraped pages only). */
43
48
  echoesQuery?: (payload: string, query: string) => boolean;
44
- /** Absolute request URL for a query. */
45
- build: (query: string) => string;
49
+ /** Absolute request URL for a query. `timeRange` may be pushed down when the engine supports it. */
50
+ build: (query: string, timeRange?: TimeRange) => string;
46
51
  /** Request body for a JSON POST. */
47
52
  body?: (query: string) => string;
48
53
  /** Extra request headers, for a provider that authenticates. */
@@ -80,6 +85,8 @@ export interface SearchOptions {
80
85
  proxy?: string;
81
86
  /** Restrict the run to these engine ids. */
82
87
  hostIds?: string[];
88
+ /** Rolling window results must fall in; `off` disables filtering. */
89
+ timeRange?: TimeRange;
83
90
  /** Cancellation from the caller. */
84
91
  signal?: AbortSignal;
85
92
  }
@@ -102,6 +109,12 @@ export interface RankOptions {
102
109
  freshnessBoost?: number;
103
110
  /** Age in days after which the freshness boost fades to zero. */
104
111
  freshnessDays?: number;
112
+ /**
113
+ * Active time window. Ranking never drops rows (the engine layer does), but
114
+ * when a window is active a row without a usable date is scored lower so a
115
+ * similarly relevant dated row wins.
116
+ */
117
+ timeRange?: TimeRange;
105
118
  /** Per-engine weight in the pre-fusion interleave. */
106
119
  weights?: number[];
107
120
  }
@@ -0,0 +1,164 @@
1
+ /**
2
+ * Key-free vertical (domain-specific) search sources.
3
+ *
4
+ * `web_search` answers "what is on the open web". This module answers the
5
+ * narrower question a coding agent asks far more often: "which package, which
6
+ * paper, which repository, which Q&A thread". Every endpoint here is a public,
7
+ * documented, credential-free API, so the feature costs no key and no account.
8
+ *
9
+ * A source is a build/parse pair: {@link VerticalSpec.build} turns the query
10
+ * into an absolute URL, and the parser turns the served body into the same
11
+ * {@link Source} shape `web_search` returns. Parsers never throw on markup or
12
+ * payload drift — a source that changes shape yields an empty list so the other
13
+ * sources still answer. Timeouts, status classification and failure isolation
14
+ * live in {@link searchVerticals}, not in the parsers.
15
+ *
16
+ * @module @fangchenghao/dsh-web-search/verticals
17
+ */
18
+ import type { Source } from './types.ts';
19
+ /** Identifiers accepted in `sources` and in the `domainSources` config field. */
20
+ export type VerticalId = 'github' | 'npm' | 'pypi' | 'arxiv' | 'wikipedia' | 'hn' | 'stackexchange' | 'crates';
21
+ /** Every supported source, in the order results are interleaved. */
22
+ export declare const VERTICAL_IDS: readonly VerticalId[];
23
+ /** Default and maximum result budget for one source. */
24
+ export declare const DEFAULT_VERTICAL_LIMIT = 5;
25
+ /** Upper bound the tool and the adapters both respect. */
26
+ export declare const MAX_VERTICAL_LIMIT = 20;
27
+ /** One vertical source definition. */
28
+ export interface VerticalSpec {
29
+ /** Stable id, also what configuration and tool arguments name. */
30
+ id: VerticalId;
31
+ /** Human label used in failure messages and smoke output. */
32
+ label: string;
33
+ /** `Accept` header this endpoint serves. */
34
+ accept: string;
35
+ /**
36
+ * Treat HTTP 404 as "no such record" rather than a failure. Only PyPI needs
37
+ * this: its endpoint is an exact-package lookup, and a missing package is a
38
+ * legitimate empty answer, not a broken source.
39
+ */
40
+ notFoundIsEmpty?: boolean;
41
+ /** Absolute request URL for a query and result budget. */
42
+ build: (query: string, limit: number) => string;
43
+ /** Extra request headers, merged over the defaults. */
44
+ headers?: Record<string, string>;
45
+ /** Turn the served body into sources; must never throw on drift. */
46
+ parse: (payload: string, finalUrl: string, limit: number) => Source[];
47
+ }
48
+ /** `fetch`-compatible function, injectable so tests never touch the network. */
49
+ export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
50
+ /** Options for {@link searchVerticals}. */
51
+ export interface VerticalSearchOptions {
52
+ /** Source ids to query; empty or absent means every source. */
53
+ ids?: readonly string[];
54
+ /** Results wanted per source; clamped to 1…20. */
55
+ limit?: number;
56
+ /** Per-source deadline in milliseconds. */
57
+ timeoutMs?: number;
58
+ /** Caller cancellation, propagated to every source. */
59
+ signal?: AbortSignal;
60
+ /** Tunnel URL from the plugin's `proxy` setting, when configured. */
61
+ proxy?: string;
62
+ /** Test seam; defaults to the platform `fetch`. */
63
+ fetch?: FetchLike;
64
+ }
65
+ /** What one vertical run reports. */
66
+ export interface VerticalOutcome {
67
+ /** Merged, de-duplicated sources, capped at the caller's budget. */
68
+ sources: Source[];
69
+ /** One `id: reason` line per source that failed. */
70
+ failures: string[];
71
+ /** Parsed row count per source, before the global cap. */
72
+ counts: Record<string, number>;
73
+ /** Whether rows were dropped because of the global cap. */
74
+ truncated: boolean;
75
+ }
76
+ /**
77
+ * Parse GitHub's repository search envelope.
78
+ * @param payload - `api.github.com/search/repositories` JSON.
79
+ * @param _url - request URL, unused by this JSON shape.
80
+ * @param limit - rows to keep.
81
+ * @returns repositories as sources, in API order.
82
+ */
83
+ export declare function parseGithub(payload: string, _url: string, limit?: number): Source[];
84
+ /**
85
+ * Parse npm's registry search envelope.
86
+ * @param payload - `registry.npmjs.org/-/v1/search` JSON.
87
+ * @param _url - request URL, unused by this JSON shape.
88
+ * @param limit - rows to keep.
89
+ * @returns packages as sources.
90
+ */
91
+ export declare function parseNpm(payload: string, _url: string, limit?: number): Source[];
92
+ /**
93
+ * Parse PyPI's exact-package envelope.
94
+ *
95
+ * PyPI no longer publishes a credential-free search API, so the adapter asks
96
+ * for the query as a package name. A 404 is handled by the orchestrator
97
+ * (`notFoundIsEmpty`) and never reaches this parser; a body without `info`
98
+ * still parses to `[]`.
99
+ * @param payload - `pypi.org/pypi/<name>/json`.
100
+ * @param _url - request URL, unused by this JSON shape.
101
+ * @param limit - rows to keep (PyPI answers one).
102
+ * @returns the package as one source, or `[]`.
103
+ */
104
+ export declare function parsePypi(payload: string, _url: string, limit?: number): Source[];
105
+ /**
106
+ * Parse the arXiv Atom feed.
107
+ * @param payload - `export.arxiv.org/api/query` XML.
108
+ * @param _url - request URL, unused by this XML shape.
109
+ * @param limit - entries to keep.
110
+ * @returns papers as sources, in feed order.
111
+ */
112
+ export declare function parseArxiv(payload: string, _url: string, limit?: number): Source[];
113
+ /**
114
+ * Parse Wikipedia's REST search envelope (zh or en host).
115
+ * @param payload - `*.wikipedia.org/w/rest.php/v1/search/page` JSON.
116
+ * @param finalUrl - the request URL, whose host decides the wiki language.
117
+ * @param limit - pages to keep.
118
+ * @returns wiki pages as sources.
119
+ */
120
+ export declare function parseWikipedia(payload: string, finalUrl: string, limit?: number): Source[];
121
+ /**
122
+ * Parse Hacker News's Algolia search envelope.
123
+ * @param payload - `hn.algolia.com/api/v1/search` JSON.
124
+ * @param _url - request URL, unused by this JSON shape.
125
+ * @param limit - hits to keep.
126
+ * @returns stories or comments as sources.
127
+ */
128
+ export declare function parseHn(payload: string, _url: string, limit?: number): Source[];
129
+ /**
130
+ * Parse Stack Exchange's advanced-search envelope.
131
+ * @param payload - `api.stackexchange.com/2.3/search/advanced` JSON.
132
+ * @param _url - request URL, unused by this JSON shape.
133
+ * @param limit - questions to keep.
134
+ * @returns questions as sources.
135
+ */
136
+ export declare function parseStackExchange(payload: string, _url: string, limit?: number): Source[];
137
+ /**
138
+ * Parse crates.io's crate search envelope.
139
+ * @param payload - `crates.io/api/v1/crates` JSON.
140
+ * @param _url - request URL, unused by this JSON shape.
141
+ * @param limit - crates to keep.
142
+ * @returns crates as sources.
143
+ */
144
+ export declare function parseCrates(payload: string, _url: string, limit?: number): Source[];
145
+ /** A source id that this module actually knows. */
146
+ export declare function isVerticalId(value: unknown): value is VerticalId;
147
+ /** Every adapter, keyed by id. */
148
+ export declare const VERTICALS: readonly VerticalSpec[];
149
+ /** One adapter by id, or `undefined` for a typo. */
150
+ export declare function verticalById(id: string): VerticalSpec | undefined;
151
+ /**
152
+ * Query every requested vertical concurrently and merge the answers.
153
+ *
154
+ * One source failing never fails the run: its reason is recorded in
155
+ * `failures`, its count stays `0`, and the other sources still answer. Answers
156
+ * are interleaved by rank (one row from each source, then the second row from
157
+ * each) so a single prolific source cannot crowd the others out.
158
+ *
159
+ * @param query - what to search for; must be non-empty.
160
+ * @param options - sources, budget, deadline, cancellation, tunnel, fetch seam.
161
+ * @returns merged sources plus per-source counts and failures.
162
+ * @throws SearchError `SEARCH_ABORTED` when the caller cancelled the run.
163
+ */
164
+ export declare function searchVerticals(query: string, options?: VerticalSearchOptions): Promise<VerticalOutcome>;