@fffangchenghao/dsh-web-search 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +131 -23
- package/README.zh-CN.md +131 -23
- package/client.js +39 -7
- package/lib/cache.d.ts +8 -4
- package/lib/cache.js +7 -4
- package/lib/config.d.ts +17 -1
- package/lib/config.js +26 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +24 -2
- package/lib/index.js +58 -1
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +15 -2
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
package/lib/freshness.js
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
/** Milliseconds in one day, shared with the ranking module. */
|
|
2
|
+
export const MS_PER_DAY = 86_400_000;
|
|
3
|
+
/** Rolling window length per range. `off` is handled separately. */
|
|
4
|
+
export const TIME_RANGE_MS = {
|
|
5
|
+
day: MS_PER_DAY,
|
|
6
|
+
week: 7 * MS_PER_DAY,
|
|
7
|
+
month: 30 * MS_PER_DAY,
|
|
8
|
+
year: 365 * MS_PER_DAY,
|
|
9
|
+
};
|
|
10
|
+
/** Every accepted `timeRange` value, in settings order. */
|
|
11
|
+
export const TIME_RANGES = ['off', 'day', 'week', 'month', 'year'];
|
|
12
|
+
/** Longest relative age accepted, so a stray huge number cannot overflow. */
|
|
13
|
+
const MAX_RELATIVE = 10_000;
|
|
14
|
+
/** Plausible absolute years; anything outside is a number, not a date. */
|
|
15
|
+
const MIN_YEAR = 1900;
|
|
16
|
+
const MAX_YEAR = 2200;
|
|
17
|
+
/** Month names as the scraped engines spell them. */
|
|
18
|
+
const MONTHS = {
|
|
19
|
+
jan: 1,
|
|
20
|
+
feb: 2,
|
|
21
|
+
mar: 3,
|
|
22
|
+
apr: 4,
|
|
23
|
+
may: 5,
|
|
24
|
+
jun: 6,
|
|
25
|
+
jul: 7,
|
|
26
|
+
aug: 8,
|
|
27
|
+
sep: 9,
|
|
28
|
+
sept: 9,
|
|
29
|
+
oct: 10,
|
|
30
|
+
nov: 11,
|
|
31
|
+
dec: 12,
|
|
32
|
+
};
|
|
33
|
+
/** Boundaries that make a relative phrase a whole token, not prose. */
|
|
34
|
+
const BEFORE = '(?:^|[\\s,,。;;::|((【\\[])';
|
|
35
|
+
const AFTER = '(?=$|[\\s,,。;;::|))】\\]])';
|
|
36
|
+
/** Chinese relative stamps: `3 天前`, `10 小时之前`, `昨天`, `刚刚`. */
|
|
37
|
+
const CN_RELATIVE = new RegExp(`${BEFORE}(\\d{1,5})\\s*(分钟|小时|天|周|个月|年)\\s*(?:前|之前)${AFTER}`, 'u');
|
|
38
|
+
/** Chinese named days, which carry no number. */
|
|
39
|
+
const CN_NAMED = new RegExp(`${BEFORE}(今天|昨天|前天|刚刚)${AFTER}`, 'u');
|
|
40
|
+
/** English relative stamps: `3 days ago`, `22 hours ago`, `yesterday`. */
|
|
41
|
+
const EN_RELATIVE = new RegExp(`${BEFORE}(\\d{1,5})\\s+(minute|hour|day|week|month|year)s?\\s+ago${AFTER}`, 'i');
|
|
42
|
+
/** English named days. */
|
|
43
|
+
const EN_NAMED = new RegExp(`${BEFORE}(today|yesterday)${AFTER}`, 'i');
|
|
44
|
+
/** `2024年5月1日` and `2024年5月1号`. */
|
|
45
|
+
const CN_ABSOLUTE = /(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*[日号]/u;
|
|
46
|
+
/** `2024-05-01`, `2024/5/1`, and the date part of a full ISO instant. */
|
|
47
|
+
const ISO_ABSOLUTE = /(\d{4})[-/](\d{1,2})[-/](\d{1,2})(?!\d)/u;
|
|
48
|
+
/** `May 1, 2024`, `Sept 25 2025`, `Oct 3rd, 2025`. */
|
|
49
|
+
const EN_MONTH_FIRST = /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\b/i;
|
|
50
|
+
/** `1 May 2024`, `3rd Oct 2025`. */
|
|
51
|
+
const EN_DAY_FIRST = /\b(\d{1,2})(?:st|nd|rd|th)?\s+(jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?,?\s+(\d{4})\b/i;
|
|
52
|
+
/** The marker AnySearch appends to `content`, e.g. `date: 22 hours ago`. */
|
|
53
|
+
const DATE_MARKER = /(?:^|[\s|])date\s*[::]\s*([^|\n]+?)\s*$/i;
|
|
54
|
+
/**
|
|
55
|
+
* Build a UTC instant for a calendar date.
|
|
56
|
+
*
|
|
57
|
+
* The time of day is fixed at noon UTC on purpose: a date without a time
|
|
58
|
+
* belongs to no particular hour, and noon keeps it inside its own calendar day
|
|
59
|
+
* for every timezone from UTC-11 to UTC+12. `Date.UTC` also means the machine's
|
|
60
|
+
* `TZ` is never consulted.
|
|
61
|
+
*
|
|
62
|
+
* @param year - four-digit year.
|
|
63
|
+
* @param month - 1-based month.
|
|
64
|
+
* @param day - day of month.
|
|
65
|
+
* @returns epoch milliseconds, or `undefined` when the components are not a
|
|
66
|
+
* real calendar date (for example 31 February).
|
|
67
|
+
*/
|
|
68
|
+
function utcDate(year, month, day) {
|
|
69
|
+
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day))
|
|
70
|
+
return undefined;
|
|
71
|
+
if (year < MIN_YEAR || year > MAX_YEAR)
|
|
72
|
+
return undefined;
|
|
73
|
+
if (month < 1 || month > 12 || day < 1 || day > 31)
|
|
74
|
+
return undefined;
|
|
75
|
+
const at = Date.UTC(year, month - 1, day, 12, 0, 0, 0);
|
|
76
|
+
const check = new Date(at);
|
|
77
|
+
if (check.getUTCFullYear() !== year || check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
return at;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Convert epoch milliseconds to the canonical ISO string.
|
|
84
|
+
* @param at - epoch milliseconds.
|
|
85
|
+
* @returns ISO 8601 in UTC, the only shape `Source.publishedAt` carries.
|
|
86
|
+
*/
|
|
87
|
+
function iso(at) {
|
|
88
|
+
return new Date(at).toISOString();
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Read the unit multiplier for a Chinese relative unit.
|
|
92
|
+
* @param unit - `分钟`, `小时`, `天`, `周`, `个月` or `年`.
|
|
93
|
+
* @returns milliseconds per unit, or `undefined` for an unknown unit.
|
|
94
|
+
*/
|
|
95
|
+
function cnUnitMs(unit) {
|
|
96
|
+
switch (unit) {
|
|
97
|
+
case '分钟':
|
|
98
|
+
return 60_000;
|
|
99
|
+
case '小时':
|
|
100
|
+
return 3_600_000;
|
|
101
|
+
case '天':
|
|
102
|
+
return MS_PER_DAY;
|
|
103
|
+
case '周':
|
|
104
|
+
return 7 * MS_PER_DAY;
|
|
105
|
+
case '个月':
|
|
106
|
+
return 30 * MS_PER_DAY;
|
|
107
|
+
case '年':
|
|
108
|
+
return 365 * MS_PER_DAY;
|
|
109
|
+
default:
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Read the unit multiplier for an English relative unit.
|
|
115
|
+
* @param unit - `minute`, `hour`, `day`, `week`, `month` or `year`.
|
|
116
|
+
* @returns milliseconds per unit, or `undefined` for an unknown unit.
|
|
117
|
+
*/
|
|
118
|
+
function enUnitMs(unit) {
|
|
119
|
+
switch (unit.toLowerCase()) {
|
|
120
|
+
case 'minute':
|
|
121
|
+
return 60_000;
|
|
122
|
+
case 'hour':
|
|
123
|
+
return 3_600_000;
|
|
124
|
+
case 'day':
|
|
125
|
+
return MS_PER_DAY;
|
|
126
|
+
case 'week':
|
|
127
|
+
return 7 * MS_PER_DAY;
|
|
128
|
+
case 'month':
|
|
129
|
+
return 30 * MS_PER_DAY;
|
|
130
|
+
case 'year':
|
|
131
|
+
return 365 * MS_PER_DAY;
|
|
132
|
+
default:
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Parse the first relative phrase in a fragment.
|
|
138
|
+
*
|
|
139
|
+
* Relative stamps are rolling: `3 days ago` means `now - 3 * 24h`, not three
|
|
140
|
+
* calendar days. That avoids the timezone and daylight-saving questions a
|
|
141
|
+
* calendar subtraction would raise, and matches how the engines mean them.
|
|
142
|
+
*
|
|
143
|
+
* @param value - normalised text.
|
|
144
|
+
* @param now - evaluation instant in epoch milliseconds.
|
|
145
|
+
* @returns epoch milliseconds, or `undefined` when there is no relative phrase.
|
|
146
|
+
*/
|
|
147
|
+
function parseRelative(value, now) {
|
|
148
|
+
const cnNumbered = CN_RELATIVE.exec(value);
|
|
149
|
+
if (cnNumbered !== null) {
|
|
150
|
+
const amount = Number(cnNumbered[1]);
|
|
151
|
+
const unit = cnUnitMs(cnNumbered[2] ?? '');
|
|
152
|
+
if (Number.isFinite(amount) && amount >= 0 && amount <= MAX_RELATIVE && unit !== undefined) {
|
|
153
|
+
return now - amount * unit;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const enNumbered = EN_RELATIVE.exec(value);
|
|
157
|
+
if (enNumbered !== null) {
|
|
158
|
+
const amount = Number(enNumbered[1]);
|
|
159
|
+
const unit = enUnitMs(enNumbered[2] ?? '');
|
|
160
|
+
if (Number.isFinite(amount) && amount >= 0 && amount <= MAX_RELATIVE && unit !== undefined) {
|
|
161
|
+
return now - amount * unit;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
const cnNamed = CN_NAMED.exec(value);
|
|
165
|
+
if (cnNamed !== null) {
|
|
166
|
+
switch (cnNamed[1]) {
|
|
167
|
+
case '昨天':
|
|
168
|
+
return now - MS_PER_DAY;
|
|
169
|
+
case '前天':
|
|
170
|
+
return now - 2 * MS_PER_DAY;
|
|
171
|
+
case '今天':
|
|
172
|
+
case '刚刚':
|
|
173
|
+
default:
|
|
174
|
+
return now;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const enNamed = EN_NAMED.exec(value);
|
|
178
|
+
if (enNamed !== null)
|
|
179
|
+
return enNamed[1]?.toLowerCase() === 'yesterday' ? now - MS_PER_DAY : now;
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Parse the first absolute date in a fragment.
|
|
184
|
+
*
|
|
185
|
+
* Checked before relative stamps only when declared explicitly; the caller
|
|
186
|
+
* decides the order. All arithmetic is UTC.
|
|
187
|
+
*
|
|
188
|
+
* @param value - normalised text.
|
|
189
|
+
* @returns epoch milliseconds, or `undefined` when no valid date is present.
|
|
190
|
+
*/
|
|
191
|
+
function parseAbsolute(value) {
|
|
192
|
+
const cn = CN_ABSOLUTE.exec(value);
|
|
193
|
+
if (cn !== null) {
|
|
194
|
+
const at = utcDate(Number(cn[1]), Number(cn[2]), Number(cn[3]));
|
|
195
|
+
if (at !== undefined)
|
|
196
|
+
return at;
|
|
197
|
+
}
|
|
198
|
+
const isoDate = ISO_ABSOLUTE.exec(value);
|
|
199
|
+
if (isoDate !== null) {
|
|
200
|
+
const at = utcDate(Number(isoDate[1]), Number(isoDate[2]), Number(isoDate[3]));
|
|
201
|
+
if (at !== undefined)
|
|
202
|
+
return at;
|
|
203
|
+
}
|
|
204
|
+
const monthFirst = EN_MONTH_FIRST.exec(value);
|
|
205
|
+
if (monthFirst !== null) {
|
|
206
|
+
const month = MONTHS[(monthFirst[1] ?? '').toLowerCase()];
|
|
207
|
+
if (month !== undefined) {
|
|
208
|
+
const at = utcDate(Number(monthFirst[3]), month, Number(monthFirst[2]));
|
|
209
|
+
if (at !== undefined)
|
|
210
|
+
return at;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
const dayFirst = EN_DAY_FIRST.exec(value);
|
|
214
|
+
if (dayFirst !== null) {
|
|
215
|
+
const month = MONTHS[(dayFirst[2] ?? '').toLowerCase()];
|
|
216
|
+
if (month !== undefined) {
|
|
217
|
+
const at = utcDate(Number(dayFirst[3]), month, Number(dayFirst[1]));
|
|
218
|
+
if (at !== undefined)
|
|
219
|
+
return at;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return undefined;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Normalise whitespace so the boundary patterns see one predictable shape.
|
|
226
|
+
* @param value - raw fragment.
|
|
227
|
+
* @returns single-spaced trimmed text.
|
|
228
|
+
*/
|
|
229
|
+
function normalise(value) {
|
|
230
|
+
return value.replace(/[\u00a0\u2000-\u200a\u202f\u205f\u3000]/gu, ' ').replace(/\s+/gu, ' ').trim();
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Extract a publication date from one fragment of result text.
|
|
234
|
+
*
|
|
235
|
+
* The search order is deliberate: an explicit `date:` marker wins (AnySearch
|
|
236
|
+
* puts the engine's stamp there), then a relative phrase, then an absolute
|
|
237
|
+
* date. `今天` inside a sentence such as `但是我们今天讲的逻辑复制` is not a
|
|
238
|
+
* publication stamp, so relative words only match as whole tokens; a bare date
|
|
239
|
+
* such as `2024-05-01` is unambiguous enough to match anywhere.
|
|
240
|
+
*
|
|
241
|
+
* @param fragment - summary, stamp or result text.
|
|
242
|
+
* @param now - evaluation instant, injected so tests are deterministic.
|
|
243
|
+
* @returns ISO 8601 UTC, or `undefined` when the fragment carries no date.
|
|
244
|
+
*/
|
|
245
|
+
export function parsePublishedAt(fragment, now = Date.now()) {
|
|
246
|
+
if (typeof fragment !== 'string' || fragment.trim().length === 0)
|
|
247
|
+
return undefined;
|
|
248
|
+
const value = normalise(fragment);
|
|
249
|
+
const marker = DATE_MARKER.exec(value);
|
|
250
|
+
if (marker !== null) {
|
|
251
|
+
const marked = normalise(marker[1] ?? '');
|
|
252
|
+
const at = parseRelative(marked, now) ?? parseAbsolute(marked);
|
|
253
|
+
if (at !== undefined)
|
|
254
|
+
return iso(at);
|
|
255
|
+
}
|
|
256
|
+
const relative = parseRelative(value, now);
|
|
257
|
+
if (relative !== undefined)
|
|
258
|
+
return iso(relative);
|
|
259
|
+
const absolute = parseAbsolute(value);
|
|
260
|
+
if (absolute !== undefined)
|
|
261
|
+
return iso(absolute);
|
|
262
|
+
return undefined;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Parse the first fragment that carries a usable date.
|
|
266
|
+
*
|
|
267
|
+
* Parsers pass the most authoritative fragment first (a dedicated timestamp
|
|
268
|
+
* element, then the snippet), so a date mentioned in prose cannot override the
|
|
269
|
+
* stamp the engine printed.
|
|
270
|
+
*
|
|
271
|
+
* @param fragments - candidate texts, most authoritative first.
|
|
272
|
+
* @param now - evaluation instant, injected for tests.
|
|
273
|
+
* @returns the first parsed ISO 8601 UTC value, or `undefined`.
|
|
274
|
+
*/
|
|
275
|
+
export function extractPublishedAt(fragments, now = Date.now()) {
|
|
276
|
+
for (const fragment of fragments) {
|
|
277
|
+
if (typeof fragment !== 'string')
|
|
278
|
+
continue;
|
|
279
|
+
const parsed = parsePublishedAt(fragment, now);
|
|
280
|
+
if (parsed !== undefined)
|
|
281
|
+
return parsed;
|
|
282
|
+
}
|
|
283
|
+
return undefined;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Read a publication date as epoch milliseconds.
|
|
287
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
288
|
+
* @returns epoch milliseconds, or `undefined` when absent or unparsable.
|
|
289
|
+
*/
|
|
290
|
+
export function publishedTimestamp(publishedAt) {
|
|
291
|
+
if (typeof publishedAt !== 'string' || publishedAt.length === 0)
|
|
292
|
+
return undefined;
|
|
293
|
+
const at = Date.parse(publishedAt);
|
|
294
|
+
return Number.isNaN(at) ? undefined : at;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Whether a result belongs to the rolling window.
|
|
298
|
+
*
|
|
299
|
+
* `off` accepts everything. A missing or unparsable date is accepted on
|
|
300
|
+
* purpose: filtering exists to remove results known to be stale, not to punish
|
|
301
|
+
* pages that publish no metadata. A date in the future is accepted too — it is
|
|
302
|
+
* still inside the window, and ranking clamps its age to zero.
|
|
303
|
+
*
|
|
304
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
305
|
+
* @param range - active window.
|
|
306
|
+
* @param now - evaluation instant.
|
|
307
|
+
* @returns `true` when the result may stay.
|
|
308
|
+
*/
|
|
309
|
+
export function isWithinTimeRange(publishedAt, range, now = Date.now()) {
|
|
310
|
+
if (range === 'off')
|
|
311
|
+
return true;
|
|
312
|
+
const at = publishedTimestamp(publishedAt);
|
|
313
|
+
if (at === undefined)
|
|
314
|
+
return true;
|
|
315
|
+
return at >= now - TIME_RANGE_MS[range];
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Keep only results inside the window.
|
|
319
|
+
*
|
|
320
|
+
* A result whose date could not be parsed is not evidence of an old page, so it
|
|
321
|
+
* is held back rather than dropped: when at least one dated result falls inside
|
|
322
|
+
* the window, only those are returned; when none does, the undated results are
|
|
323
|
+
* returned as a fallback so the caller is not left empty-handed. That trades a
|
|
324
|
+
* little precision for never answering "last 24 hours" with nothing at all.
|
|
325
|
+
* @param sources - parsed sources, in engine order.
|
|
326
|
+
* @param range - active window.
|
|
327
|
+
* @param now - evaluation instant.
|
|
328
|
+
* @returns a new list; the input array and its objects are untouched.
|
|
329
|
+
*/
|
|
330
|
+
export function filterByTimeRange(sources, range, now = Date.now()) {
|
|
331
|
+
if (range === 'off')
|
|
332
|
+
return [...sources];
|
|
333
|
+
// "Undated" means the timestamp could not be parsed, not merely absent: a
|
|
334
|
+
// junk value is no more evidence of age than a missing one.
|
|
335
|
+
const dated = sources.filter((source) => publishedTimestamp(source.publishedAt) !== undefined);
|
|
336
|
+
const inside = dated.filter((source) => isWithinTimeRange(source.publishedAt, range, now));
|
|
337
|
+
if (inside.length > 0)
|
|
338
|
+
return inside;
|
|
339
|
+
const undated = sources.filter((source) => publishedTimestamp(source.publishedAt) === undefined);
|
|
340
|
+
return undated.length > 0 ? undated : [];
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Apply a window to an already merged list.
|
|
344
|
+
*
|
|
345
|
+
* Per-engine filtering can only see one engine's dates, so an engine that dates
|
|
346
|
+
* nothing keeps all of its rows; this pass runs after the merge, where a dated
|
|
347
|
+
* row from any engine can speak for the whole list. When nothing is dated inside
|
|
348
|
+
* the window the merged list is returned unchanged rather than emptied.
|
|
349
|
+
* @param sources - merged sources.
|
|
350
|
+
* @param range - active window.
|
|
351
|
+
* @param now - evaluation instant.
|
|
352
|
+
* @returns the rows to keep.
|
|
353
|
+
*/
|
|
354
|
+
export function preferWindow(sources, range, now = Date.now()) {
|
|
355
|
+
if (range === 'off')
|
|
356
|
+
return [...sources];
|
|
357
|
+
const inside = sources.filter((source) => publishedTimestamp(source.publishedAt) !== undefined && isWithinTimeRange(source.publishedAt, range, now));
|
|
358
|
+
return inside.length > 0 ? inside : [...sources];
|
|
359
|
+
}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { PluginConfig } from './config.ts';
|
|
2
2
|
import type { FetchProvider } from './fetch.ts';
|
|
3
3
|
import type { Source } from './types.ts';
|
|
4
|
-
export declare const name = "dsh-web-search
|
|
4
|
+
export declare const name = "dsh-web-search";
|
|
5
5
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
6
6
|
export declare const inject: string[];
|
|
7
7
|
/** Provider id referenced by `web.searchProvider`. */
|
|
@@ -30,6 +30,15 @@ export interface PluginContext {
|
|
|
30
30
|
registerSearchProvider: (provider: SearchProvider) => void;
|
|
31
31
|
registerFetchProvider: (provider: FetchProvider) => void;
|
|
32
32
|
};
|
|
33
|
+
/**
|
|
34
|
+
* Cordis's inject-free service lookup. `tools` is read through this rather
|
|
35
|
+
* than added to {@link inject} on purpose: a hard `tools` dependency would
|
|
36
|
+
* stop the whole plugin (including `web_search`) from mounting in a
|
|
37
|
+
* deployment that has no tool registry.
|
|
38
|
+
*/
|
|
39
|
+
get?: (name: string) => unknown;
|
|
40
|
+
/** Fallback for tests and hosts that expose the registry as a property. */
|
|
41
|
+
tools?: unknown;
|
|
33
42
|
}
|
|
34
43
|
/** Configuration as the loader hands it over, already schema-validated. */
|
|
35
44
|
export type ApplyConfig = PluginConfig;
|
|
@@ -64,12 +73,16 @@ declare const _default: typeof apply & {
|
|
|
64
73
|
authorityBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
65
74
|
freshnessBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
66
75
|
freshnessDays: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
76
|
+
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
67
77
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
68
78
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
69
79
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
70
80
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
71
81
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
72
82
|
maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
83
|
+
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
84
|
+
domainLimit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
85
|
+
domainTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
73
86
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
74
87
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
75
88
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -81,12 +94,16 @@ declare const _default: typeof apply & {
|
|
|
81
94
|
authorityBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
82
95
|
freshnessBoost: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
83
96
|
freshnessDays: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
97
|
+
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
84
98
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
85
99
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
86
100
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
87
101
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
88
102
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
89
103
|
maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
104
|
+
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
105
|
+
domainLimit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
106
|
+
domainTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
90
107
|
}>>, "plain">;
|
|
91
108
|
inject: string[];
|
|
92
109
|
};
|
|
@@ -97,5 +114,10 @@ export { EngineHealth } from './health.ts';
|
|
|
97
114
|
export { SearchError, parseRetryAfter } from './errors.ts';
|
|
98
115
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from './parsers.ts';
|
|
99
116
|
export { fuse, interleave, dedupe, mergeRegions } from './rank.ts';
|
|
117
|
+
export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublishedAt, publishedTimestamp, isWithinTimeRange, filterByTimeRange, } from './freshness.ts';
|
|
100
118
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from './proxy.ts';
|
|
101
|
-
export
|
|
119
|
+
export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from './verticals.ts';
|
|
120
|
+
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
|
|
121
|
+
export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
|
|
122
|
+
export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
|
|
123
|
+
export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
|
package/lib/index.js
CHANGED
|
@@ -14,13 +14,45 @@ import { search } from "./engines.js";
|
|
|
14
14
|
import { SearchCache } from "./cache.js";
|
|
15
15
|
import { EngineHealth } from "./health.js";
|
|
16
16
|
import { createFetchProvider } from "./fetch.js";
|
|
17
|
-
|
|
17
|
+
import { registerDomainTool } from "./domain-tool.js";
|
|
18
|
+
export const name = 'dsh-web-search';
|
|
18
19
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
19
20
|
export const inject = ['web'];
|
|
20
21
|
/** Provider id referenced by `web.searchProvider`. */
|
|
21
22
|
export const PROVIDER_ID = 'bing';
|
|
22
23
|
/** Fetch provider id, named in the `web` service config as `fetchProvider`. */
|
|
23
24
|
export const FETCH_PROVIDER_ID = 'http-proxy';
|
|
25
|
+
/**
|
|
26
|
+
* Read the optional tool registry without requiring it.
|
|
27
|
+
* @param ctx - the plugin context.
|
|
28
|
+
* @returns the registry, or `undefined` when this deployment has none.
|
|
29
|
+
*/
|
|
30
|
+
function readToolRegistry(ctx) {
|
|
31
|
+
let candidate;
|
|
32
|
+
try {
|
|
33
|
+
const found = typeof ctx.get === 'function' ? ctx.get('tools') : undefined;
|
|
34
|
+
// Cordis throws on an un-injected service property, so even the fallback
|
|
35
|
+
// read is guarded: a missing `tools` is a supported deployment, not an error.
|
|
36
|
+
candidate = found ?? ctx.tools;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
if (typeof candidate !== 'object' || candidate === null)
|
|
42
|
+
return undefined;
|
|
43
|
+
return typeof candidate.register === 'function'
|
|
44
|
+
? candidate
|
|
45
|
+
: undefined;
|
|
46
|
+
}
|
|
47
|
+
/** Report a skipped tool registration without breaking plugin loading. */
|
|
48
|
+
function warnTool(message) {
|
|
49
|
+
try {
|
|
50
|
+
console.warn(`[dsh-web-search-bing] ${message}`);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// Logging must never be the reason a plugin fails to mount.
|
|
54
|
+
}
|
|
55
|
+
}
|
|
24
56
|
/**
|
|
25
57
|
* Register the search provider.
|
|
26
58
|
* @param ctx - plugin context carrying the `web` service.
|
|
@@ -56,6 +88,7 @@ export function apply(ctx, config = {}) {
|
|
|
56
88
|
const { sources, failures } = await search(query, signal, {
|
|
57
89
|
limit,
|
|
58
90
|
wait: resolved.wait,
|
|
91
|
+
timeRange: resolved.timeRange,
|
|
59
92
|
timeoutMs: resolved.timeoutMs,
|
|
60
93
|
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
61
94
|
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
@@ -82,6 +115,27 @@ export function apply(ctx, config = {}) {
|
|
|
82
115
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
83
116
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
84
117
|
}));
|
|
118
|
+
// Optional half: `domain_search`. Fail-safe by construction — a deployment
|
|
119
|
+
// without `ctx.tools` still loads this plugin and keeps `web_search`; the
|
|
120
|
+
// registration path is the host's `ToolRuntime.register(definition)`.
|
|
121
|
+
try {
|
|
122
|
+
const tools = readToolRegistry(ctx);
|
|
123
|
+
if (tools !== undefined) {
|
|
124
|
+
registerDomainTool(tools, () => {
|
|
125
|
+
const live = resolveConfig(section, undefined);
|
|
126
|
+
const settings = {
|
|
127
|
+
sourceIds: live.domainSourceIds,
|
|
128
|
+
limit: live.domainLimit,
|
|
129
|
+
timeoutMs: live.domainTimeoutMs,
|
|
130
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
131
|
+
};
|
|
132
|
+
return settings;
|
|
133
|
+
}, { warn: warnTool });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
warnTool(`domain_search skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
138
|
+
}
|
|
85
139
|
}
|
|
86
140
|
export { Config, resolveConfig } from "./config.js";
|
|
87
141
|
/**
|
|
@@ -104,4 +158,7 @@ export { EngineHealth } from "./health.js";
|
|
|
104
158
|
export { SearchError, parseRetryAfter } from "./errors.js";
|
|
105
159
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
|
|
106
160
|
export { fuse, interleave, dedupe, mergeRegions } from "./rank.js";
|
|
161
|
+
export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublishedAt, publishedTimestamp, isWithinTimeRange, filterByTimeRange, } from "./freshness.js";
|
|
107
162
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
163
|
+
export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
164
|
+
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
package/lib/parsers.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { extractPublishedAt, parsePublishedAt } from "./freshness.js";
|
|
1
2
|
/** Named HTML entities that actually show up in result markup. */
|
|
2
3
|
const ENTITIES = {
|
|
3
4
|
amp: '&',
|
|
@@ -146,7 +147,17 @@ export function parseSo(payload, baseUrl) {
|
|
|
146
147
|
const title = text(anchor[2]);
|
|
147
148
|
const summary = /<span[^>]*class="res-list-summary[^"]*"[^>]*>([\s\S]*?)<\/span>/.exec(item);
|
|
148
149
|
const snippet = summary ? text(summary[1]) : '';
|
|
149
|
-
|
|
150
|
+
// 360 prints the stamp in a separate span (`发贴时间:2026年1月22日`), not in
|
|
151
|
+
// the summary. Reading only that span avoids treating prose such as
|
|
152
|
+
// `但是我们今天讲的逻辑复制` as a publication date.
|
|
153
|
+
const stamp = /(?:发[贴布]|发布)时间[::]\s*([^<]{0,60})/.exec(item);
|
|
154
|
+
const publishedAt = stamp ? parsePublishedAt(text(stamp[1])) : undefined;
|
|
155
|
+
sources.push({
|
|
156
|
+
url,
|
|
157
|
+
...(title ? { title } : {}),
|
|
158
|
+
...(snippet ? { snippet } : {}),
|
|
159
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
160
|
+
});
|
|
150
161
|
}
|
|
151
162
|
return sources;
|
|
152
163
|
}
|
|
@@ -171,7 +182,16 @@ export function parseBing(payload, baseUrl) {
|
|
|
171
182
|
const title = text(anchor[2]);
|
|
172
183
|
const paragraph = /<p[^>]*>([\s\S]*?)<\/p>/.exec(item);
|
|
173
184
|
const snippet = paragraph ? text(paragraph[1]) : '';
|
|
174
|
-
|
|
185
|
+
// Bing puts the date at the head of the caption (`2025年9月25日 …`,
|
|
186
|
+
// `2 天之前 …`) or in a `news_dt` span; some captions end with an ISO date.
|
|
187
|
+
const newsDate = /<span[^>]*class="news_dt"[^>]*>([\s\S]*?)<\/span>/.exec(item);
|
|
188
|
+
const publishedAt = extractPublishedAt([newsDate ? text(newsDate[1]) : undefined, snippet]);
|
|
189
|
+
sources.push({
|
|
190
|
+
url,
|
|
191
|
+
...(title ? { title } : {}),
|
|
192
|
+
...(snippet ? { snippet } : {}),
|
|
193
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
194
|
+
});
|
|
175
195
|
}
|
|
176
196
|
return sources;
|
|
177
197
|
}
|
|
@@ -200,7 +220,17 @@ export function parseDdgLite(payload, baseUrl) {
|
|
|
200
220
|
continue;
|
|
201
221
|
const title = text(match[1]);
|
|
202
222
|
const snippet = text(/<td[^>]*class=['"]result-snippet['"][^>]*>([\s\S]*?)<\/td>/i.exec(payload.slice(start, end))?.[1] ?? '');
|
|
203
|
-
|
|
223
|
+
// The lite page has no timestamp in today's markup; if one ever appears it
|
|
224
|
+
// is the authoritative stamp, otherwise a date mentioned in the snippet is
|
|
225
|
+
// the only evidence available.
|
|
226
|
+
const stamp = /<[a-z]+[^>]*class=['"]result-timestamp['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end));
|
|
227
|
+
const publishedAt = extractPublishedAt([stamp?.[1], snippet]);
|
|
228
|
+
sources.push({
|
|
229
|
+
url,
|
|
230
|
+
...(title ? { title } : {}),
|
|
231
|
+
...(snippet ? { snippet } : {}),
|
|
232
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
233
|
+
});
|
|
204
234
|
}
|
|
205
235
|
return sources;
|
|
206
236
|
}
|
|
@@ -230,7 +260,16 @@ export function parseDdgHtml(payload, baseUrl) {
|
|
|
230
260
|
continue;
|
|
231
261
|
const title = text(match[1]);
|
|
232
262
|
const snippet = text(/<[a-z]+[^>]*class=['"]result__snippet['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end))?.[1] ?? '');
|
|
233
|
-
|
|
263
|
+
// DuckDuckGo's HTML endpoint usually omits the date; when a timestamp span
|
|
264
|
+
// is present it is preferred over a date that only appears in the snippet.
|
|
265
|
+
const stamp = /<[a-z]+[^>]*class=['"]result__timestamp['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end));
|
|
266
|
+
const publishedAt = extractPublishedAt([stamp?.[1], snippet]);
|
|
267
|
+
sources.push({
|
|
268
|
+
url,
|
|
269
|
+
...(title ? { title } : {}),
|
|
270
|
+
...(snippet ? { snippet } : {}),
|
|
271
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
272
|
+
});
|
|
234
273
|
}
|
|
235
274
|
return sources;
|
|
236
275
|
}
|
|
@@ -275,7 +314,22 @@ export function parseAnySearch(payload, baseUrl) {
|
|
|
275
314
|
continue;
|
|
276
315
|
const title = typeof row.title === 'string' ? row.title.trim() : '';
|
|
277
316
|
const snippet = typeof row.snippet === 'string' ? row.snippet.trim() : '';
|
|
278
|
-
|
|
317
|
+
// Measured against the live endpoint: rows carry title/url/snippet/content,
|
|
318
|
+
// and the publication stamp is appended to `content` as `date: 22 hours
|
|
319
|
+
// ago` rather than exposed as a top-level date field. A top-level `date`
|
|
320
|
+
// is accepted too, so a future API change does not need a parser rewrite.
|
|
321
|
+
// Only the marked value is parsed: the rest of `content` is article prose
|
|
322
|
+
// and may mention unrelated dates (`Retrieved June 2, 2024`).
|
|
323
|
+
const content = typeof row.content === 'string' ? row.content : '';
|
|
324
|
+
const marker = /(?:^|[\s|])date\s*[::]\s*([^|\n]+?)\s*$/i.exec(content);
|
|
325
|
+
const topLevel = typeof row.date === 'string' ? row.date : undefined;
|
|
326
|
+
const publishedAt = extractPublishedAt([topLevel, marker?.[1]]);
|
|
327
|
+
sources.push({
|
|
328
|
+
url,
|
|
329
|
+
...(title ? { title } : {}),
|
|
330
|
+
...(snippet ? { snippet } : {}),
|
|
331
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
332
|
+
});
|
|
279
333
|
}
|
|
280
334
|
return sources;
|
|
281
335
|
}
|
package/lib/rank.d.ts
CHANGED
|
@@ -31,7 +31,11 @@
|
|
|
31
31
|
* happens to sit at the same rank;
|
|
32
32
|
* - freshness: a result published a few days ago is more useful than the same
|
|
33
33
|
* result published years ago, with the boost fading linearly to zero at
|
|
34
|
-
* `freshnessDays
|
|
34
|
+
* `freshnessDays`;
|
|
35
|
+
* - an active time filter: a dated in-window result beats a result with no
|
|
36
|
+
* date at the same rank. The penalty is small and applied only while
|
|
37
|
+
* filtering is on, so an undated page can still win on relevance and the
|
|
38
|
+
* default (`timeRange: off`) is byte-for-byte the old behaviour.
|
|
35
39
|
*
|
|
36
40
|
* The bilingual region policy is kept intact: engines whose region matches the
|
|
37
41
|
* query language's lead (or are marked `both`) are fused first and fill the
|
|
@@ -55,6 +59,15 @@ export declare const DEFAULT_AUTHORITY_BOOST = 0.25;
|
|
|
55
59
|
export declare const DEFAULT_FRESHNESS_BOOST = 0.2;
|
|
56
60
|
/** Age in days after which the freshness boost has faded to zero. */
|
|
57
61
|
export declare const DEFAULT_FRESHNESS_DAYS = 30;
|
|
62
|
+
/**
|
|
63
|
+
* Score factor for a result with no usable date while a time window is active.
|
|
64
|
+
*
|
|
65
|
+
* Ten percent is roughly six fusion positions at `rrfConstant = 60`: enough for
|
|
66
|
+
* a dated result to win a close comparison, small enough that an undated result
|
|
67
|
+
* still outranks a dated one that is clearly less relevant. With every result
|
|
68
|
+
* undated the factor is uniform, so the order is unchanged.
|
|
69
|
+
*/
|
|
70
|
+
export declare const UNDATED_WHEN_FILTERING_FACTOR = 0.9;
|
|
58
71
|
/**
|
|
59
72
|
* Deduplicate by URL, preserving first-seen order.
|
|
60
73
|
*
|