@fffangchenghao/dsh-web-search 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +206 -43
- package/README.zh-CN.md +206 -43
- package/client.js +16 -0
- package/lib/config.d.ts +9 -0
- package/lib/config.js +14 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +26 -0
- package/lib/fetch.js +187 -8
- package/lib/index.d.ts +6 -0
- package/lib/index.js +23 -1
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +6 -0
- package/package.json +1 -1
package/lib/engines.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module @fangchenghao/dsh-web-search/engines
|
|
13
13
|
*/
|
|
14
|
-
import type { HostSpec, RankOptions, SearchOptions, SearchOutcome } from './types.ts';
|
|
14
|
+
import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
|
|
15
15
|
import type { SearchCache } from './cache.ts';
|
|
16
16
|
import type { EngineHealth } from './health.ts';
|
|
17
17
|
/**
|
|
@@ -34,6 +34,46 @@ export interface EngineRunOptions extends SearchOptions {
|
|
|
34
34
|
cache?: SearchCache;
|
|
35
35
|
/** Cooldown tracker, when one is configured. */
|
|
36
36
|
health?: EngineHealth;
|
|
37
|
+
/**
|
|
38
|
+
* Read-only observer of the fusion inputs.
|
|
39
|
+
*
|
|
40
|
+
* Benchmark recorders use it to persist each engine's own list and rerun the
|
|
41
|
+
* current ranking offline. It never changes the run: the snapshot is a copy
|
|
42
|
+
* handed over after the fused list is final, and an absent hook costs one
|
|
43
|
+
* `undefined` check per run.
|
|
44
|
+
*/
|
|
45
|
+
trace?: (snapshot: SearchTrace) => void;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* What one search run handed to the ranking, captured for offline replay.
|
|
49
|
+
*
|
|
50
|
+
* `answers` are the exact per-engine lists that produced `sources`, in engine
|
|
51
|
+
* priority order and already narrowed by an active time window. Only the tier
|
|
52
|
+
* that actually supplied the fused list is present: when the primary tier
|
|
53
|
+
* answered, the fallback engines never ran. A cache hit has no fresh answers,
|
|
54
|
+
* so `answers` is empty and `cached` is `true`.
|
|
55
|
+
*/
|
|
56
|
+
export interface SearchTrace {
|
|
57
|
+
/** Caller budget, after the engine layer's own clamp. */
|
|
58
|
+
limit: number;
|
|
59
|
+
/** Region that led this query. */
|
|
60
|
+
lead: Region;
|
|
61
|
+
/** Release policy used for the tier run. */
|
|
62
|
+
wait: WaitPolicy;
|
|
63
|
+
/** Active publication window. */
|
|
64
|
+
timeRange: TimeRange;
|
|
65
|
+
/** Ranking knobs, with the window folded in. */
|
|
66
|
+
rank: RankOptions;
|
|
67
|
+
/** Which path produced `sources`. */
|
|
68
|
+
tier: 'primary' | 'fallback' | 'none' | 'cache';
|
|
69
|
+
/** Per-engine lists that were fused, in engine priority order. */
|
|
70
|
+
answers: EngineAnswer[];
|
|
71
|
+
/** The fused list `search` returned. */
|
|
72
|
+
sources: Source[];
|
|
73
|
+
/** Failure strings accumulated by this run. */
|
|
74
|
+
failures: string[];
|
|
75
|
+
/** Whether the result came from the cache. */
|
|
76
|
+
cached: boolean;
|
|
37
77
|
}
|
|
38
78
|
/**
|
|
39
79
|
* Search the web with every configured engine and fuse the answers.
|
package/lib/engines.js
CHANGED
|
@@ -66,7 +66,9 @@ export const HOSTS = [
|
|
|
66
66
|
},
|
|
67
67
|
{
|
|
68
68
|
id: 'so',
|
|
69
|
-
|
|
69
|
+
// 360 answers an anti-bot wall from this network and from the tunnel, so it
|
|
70
|
+
// is demoted: tried only when every primary engine came back empty.
|
|
71
|
+
tier: 'fallback',
|
|
70
72
|
region: 'cn',
|
|
71
73
|
build: (query) => `https://www.so.com/s?q=${encodeURIComponent(query)}`,
|
|
72
74
|
parse: parseSo,
|
|
@@ -317,14 +319,31 @@ export async function search(query, signal, options) {
|
|
|
317
319
|
// next identical query into a cache miss.
|
|
318
320
|
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
|
|
319
321
|
const cached = key === undefined ? undefined : options.cache?.get(key);
|
|
320
|
-
if (cached !== undefined)
|
|
322
|
+
if (cached !== undefined) {
|
|
323
|
+
options.trace?.({
|
|
324
|
+
limit,
|
|
325
|
+
lead,
|
|
326
|
+
wait,
|
|
327
|
+
timeRange,
|
|
328
|
+
rank,
|
|
329
|
+
tier: 'cache',
|
|
330
|
+
answers: [],
|
|
331
|
+
sources: cached,
|
|
332
|
+
failures: [...failures],
|
|
333
|
+
cached: true,
|
|
334
|
+
});
|
|
321
335
|
return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
336
|
+
}
|
|
322
337
|
const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
|
|
323
338
|
const primary = ordered.filter((host) => host.tier === 'primary');
|
|
324
339
|
const fallback = ordered.filter((host) => host.tier === 'fallback');
|
|
340
|
+
let tier = 'none';
|
|
325
341
|
let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
342
|
+
if (answers.length > 0)
|
|
343
|
+
tier = 'primary';
|
|
326
344
|
let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
327
345
|
if (sources.length === 0 && fallback.length > 0) {
|
|
346
|
+
tier = 'fallback';
|
|
328
347
|
answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
329
348
|
sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
330
349
|
}
|
|
@@ -332,5 +351,17 @@ export async function search(query, signal, options) {
|
|
|
332
351
|
throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
|
|
333
352
|
if (key !== undefined && sources.length > 0)
|
|
334
353
|
options.cache?.set(key, sources);
|
|
354
|
+
options.trace?.({
|
|
355
|
+
limit,
|
|
356
|
+
lead,
|
|
357
|
+
wait,
|
|
358
|
+
timeRange,
|
|
359
|
+
rank,
|
|
360
|
+
tier,
|
|
361
|
+
answers,
|
|
362
|
+
sources,
|
|
363
|
+
failures: [...failures],
|
|
364
|
+
cached: false,
|
|
365
|
+
});
|
|
335
366
|
return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
336
367
|
}
|
package/lib/fetch.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { FetchFormat } from './markdown.ts';
|
|
2
|
+
import type { RenderMode, RenderReport, RenderedPage, RendererLoad, RendererModule } from './render.ts';
|
|
2
3
|
/** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
|
|
3
4
|
export interface FetchOutcome {
|
|
4
5
|
readonly url: string;
|
|
@@ -8,6 +9,13 @@ export interface FetchOutcome {
|
|
|
8
9
|
readonly content: string;
|
|
9
10
|
};
|
|
10
11
|
readonly truncated: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* How the optional render rung decided, when this provider produced the
|
|
14
|
+
* result. The Host ignores the extra field; it is here so a caller (and a
|
|
15
|
+
* test) can see why a shell page was or was not rendered, including that
|
|
16
|
+
* Playwright is absent.
|
|
17
|
+
*/
|
|
18
|
+
readonly render?: RenderReport;
|
|
11
19
|
}
|
|
12
20
|
/** What `ctx.web.registerFetchProvider` expects. */
|
|
13
21
|
export interface FetchProvider {
|
|
@@ -37,6 +45,14 @@ export interface FetchProviderOptions {
|
|
|
37
45
|
/** Character budget for the returned text. */
|
|
38
46
|
/** Character budget, or a reader for it. */
|
|
39
47
|
maxLength?: number | (() => number);
|
|
48
|
+
/**
|
|
49
|
+
* Optional browser rendering: `auto` renders only a suspected shell page,
|
|
50
|
+
* `always` renders every fetch, `off` never does. A reader keeps a Settings
|
|
51
|
+
* edit live. Defaults to `auto`; without Playwright it silently does nothing.
|
|
52
|
+
*/
|
|
53
|
+
renderJs?: RenderMode | (() => RenderMode);
|
|
54
|
+
/** Browser-render deadline, or a reader for it. Defaults to 15000 ms. */
|
|
55
|
+
renderTimeoutMs?: number | (() => number);
|
|
40
56
|
/** User agent sent on every rung. */
|
|
41
57
|
userAgent?: string;
|
|
42
58
|
}
|
|
@@ -50,6 +66,16 @@ export interface FetchDependencies {
|
|
|
50
66
|
extract: (url: string, apiKey: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<FetchOutcome>;
|
|
51
67
|
/** Proxy discovery. */
|
|
52
68
|
detectProxy: (explicit: string | undefined) => Promise<string | undefined>;
|
|
69
|
+
/**
|
|
70
|
+
* Load the optional renderer. The default dynamically imports Playwright and
|
|
71
|
+
* reports absence instead of throwing; tests inject a fake here.
|
|
72
|
+
*/
|
|
73
|
+
loadRenderer: () => Promise<RendererLoad>;
|
|
74
|
+
/**
|
|
75
|
+
* Render one page with a loaded renderer. The default is the real
|
|
76
|
+
* headless-Chromium implementation; tests inject a fake browser.
|
|
77
|
+
*/
|
|
78
|
+
render: (renderer: RendererModule, url: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<RenderedPage>;
|
|
53
79
|
}
|
|
54
80
|
/**
|
|
55
81
|
* Build the fetch provider.
|
package/lib/fetch.js
CHANGED
|
@@ -8,24 +8,34 @@
|
|
|
8
8
|
* answers through the local tunnel in one to three seconds. Searching therefore
|
|
9
9
|
* found pages the agent could never read.
|
|
10
10
|
*
|
|
11
|
-
* This provider closes that loop with a
|
|
11
|
+
* This provider closes that loop with a four-step ladder:
|
|
12
12
|
*
|
|
13
13
|
* 1. direct, which is what the shipped provider already does and stays first so
|
|
14
14
|
* an open network pays no tunnel round trip;
|
|
15
15
|
* 2. the tunnel, using the same proxy resolution the search side uses (explicit
|
|
16
16
|
* option, then the environment, then the macOS system proxy);
|
|
17
|
-
* 3.
|
|
17
|
+
* 3. an optional headless-browser render, for a page whose bytes are a shell
|
|
18
|
+
* and whose text only appears after its scripts run. It runs before the
|
|
19
|
+
* extraction endpoint because it is local (no credential, no third-party
|
|
20
|
+
* request) and because `renderJs: 'always'` must actually mean a browser;
|
|
21
|
+
* 4. AnySearch's extraction endpoint, when a key is configured — it renders the
|
|
18
22
|
* page server-side, so it also rescues a page that answers 403 to a bare
|
|
19
|
-
* request.
|
|
23
|
+
* request and remains the last resort when no browser is installed.
|
|
24
|
+
*
|
|
25
|
+
* Playwright is not a dependency and is never imported at module load: the
|
|
26
|
+
* renderer is discovered by a guarded dynamic `import()` and its absence is
|
|
27
|
+
* recorded in the result, never thrown.
|
|
20
28
|
*
|
|
21
29
|
* A non-2xx response is a result, not a failure, and a failure on one rung is
|
|
22
30
|
* not fatal: the ladder only throws when every rung is unavailable or failed.
|
|
31
|
+
* A render failure specifically keeps the earlier rung's page.
|
|
23
32
|
*
|
|
24
33
|
* @module @fangchenghao/dsh-web-search/fetch
|
|
25
34
|
*/
|
|
26
35
|
import { SearchError } from "./errors.js";
|
|
27
36
|
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
28
37
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
38
|
+
import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
|
|
29
39
|
/** Default user agent; kept in step with the search side. */
|
|
30
40
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
31
41
|
/** One mebibyte: enough for a documentation page, cheap enough to hold. */
|
|
@@ -64,6 +74,16 @@ function deadline(timeoutMs, signal) {
|
|
|
64
74
|
const timer = AbortSignal.timeout(timeoutMs);
|
|
65
75
|
return signal === undefined ? timer : AbortSignal.any([signal, timer]);
|
|
66
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Turn a cancelled request into a rejection instead of a partial result.
|
|
79
|
+
* @param signal - the caller signal that is already aborted.
|
|
80
|
+
*/
|
|
81
|
+
function abortFetch(signal) {
|
|
82
|
+
const reason = signal.reason;
|
|
83
|
+
if (reason instanceof Error)
|
|
84
|
+
throw reason;
|
|
85
|
+
throw new DOMException('The fetch was aborted', 'AbortError');
|
|
86
|
+
}
|
|
67
87
|
/** Direct request used by the first rung. */
|
|
68
88
|
async function directFetch(url, timeoutMs, signal, userAgent, maxBytes) {
|
|
69
89
|
const response = await fetch(url, {
|
|
@@ -121,6 +141,8 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
121
141
|
tunnel: tunnelFetch,
|
|
122
142
|
extract: extractFetch,
|
|
123
143
|
detectProxy: (explicit) => resolveProxy(explicit),
|
|
144
|
+
loadRenderer: loadPlaywrightRenderer,
|
|
145
|
+
render: renderWithPlaywright,
|
|
124
146
|
...overrides,
|
|
125
147
|
};
|
|
126
148
|
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
@@ -137,11 +159,41 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
137
159
|
const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
|
|
138
160
|
return raw ?? DEFAULT_FETCH_MAX_LENGTH;
|
|
139
161
|
};
|
|
162
|
+
/** `renderJs` is volatile too: read it per request, never at construction. */
|
|
163
|
+
const liveRenderMode = () => {
|
|
164
|
+
const raw = typeof options.renderJs === 'function' ? options.renderJs() : options.renderJs;
|
|
165
|
+
return raw === 'always' || raw === 'off' ? raw : 'auto';
|
|
166
|
+
};
|
|
167
|
+
const liveRenderTimeout = () => {
|
|
168
|
+
const raw = typeof options.renderTimeoutMs === 'function' ? options.renderTimeoutMs() : options.renderTimeoutMs;
|
|
169
|
+
return typeof raw === 'number' && Number.isFinite(raw) && raw >= 1
|
|
170
|
+
? Math.floor(raw)
|
|
171
|
+
: DEFAULT_RENDER_TIMEOUT_MS;
|
|
172
|
+
};
|
|
173
|
+
/**
|
|
174
|
+
* Load the renderer at most once per provider. A missing package is a
|
|
175
|
+
* supported deployment, and caching the verdict keeps a shell page from
|
|
176
|
+
* paying a failed module resolution on every fetch; installing Playwright
|
|
177
|
+
* later needs a plugin restart, which is the same rule the other imports
|
|
178
|
+
* follow.
|
|
179
|
+
*/
|
|
180
|
+
let rendererLoad;
|
|
181
|
+
const loadRendererOnce = () => {
|
|
182
|
+
if (rendererLoad === undefined) {
|
|
183
|
+
rendererLoad = Promise.resolve()
|
|
184
|
+
.then(() => deps.loadRenderer())
|
|
185
|
+
.catch((error) => ({
|
|
186
|
+
ok: false,
|
|
187
|
+
reason: redactRenderReason(`the renderer could not be loaded (${error instanceof Error ? error.name : 'unknown error'})`),
|
|
188
|
+
}));
|
|
189
|
+
}
|
|
190
|
+
return rendererLoad;
|
|
191
|
+
};
|
|
140
192
|
// Every rung funnels through here: an HTML body becomes Markdown (or plain
|
|
141
193
|
// text), a text body keeps its content, and both obey the character budget.
|
|
142
194
|
// The Host's own renderer then sees `kind: 'text'`, so it does not convert the
|
|
143
|
-
// same page a second time.
|
|
144
|
-
const finalize = (outcome) => {
|
|
195
|
+
// same page a second time. The render report rides along as an optional field.
|
|
196
|
+
const finalize = (outcome, render) => {
|
|
145
197
|
const format = liveFormat();
|
|
146
198
|
const maxLength = liveMaxLength();
|
|
147
199
|
if (outcome.body.kind === 'html') {
|
|
@@ -150,6 +202,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
150
202
|
...outcome,
|
|
151
203
|
body: { kind: 'text', content: rendered.content },
|
|
152
204
|
truncated: outcome.truncated || rendered.truncated,
|
|
205
|
+
render,
|
|
153
206
|
};
|
|
154
207
|
}
|
|
155
208
|
const limited = truncateFetchText(outcome.body.content, maxLength);
|
|
@@ -157,6 +210,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
157
210
|
...outcome,
|
|
158
211
|
body: { kind: 'text', content: limited.content },
|
|
159
212
|
truncated: outcome.truncated || limited.truncated,
|
|
213
|
+
render,
|
|
160
214
|
};
|
|
161
215
|
};
|
|
162
216
|
return {
|
|
@@ -170,17 +224,112 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
170
224
|
const failures = [];
|
|
171
225
|
/** True once the caller cancelled; a cancellation must not be retried. */
|
|
172
226
|
const cancelled = () => signal !== undefined && signal.aborted;
|
|
227
|
+
/** Reject with the caller's own abort reason when they cancelled. */
|
|
228
|
+
const stopIfCancelled = () => {
|
|
229
|
+
if (signal !== undefined && signal.aborted)
|
|
230
|
+
abortFetch(signal);
|
|
231
|
+
};
|
|
232
|
+
/** Best page from a transport rung, kept if rendering cannot improve it. */
|
|
233
|
+
let fallback;
|
|
234
|
+
/** The last render decision, attached to whatever result wins. */
|
|
235
|
+
let lastReport;
|
|
236
|
+
const makeReport = (mode, attempted, applied, reason) => ({ mode, attempted, applied, reason: redactRenderReason(reason) });
|
|
237
|
+
/**
|
|
238
|
+
* Decide whether to render, and render when warranted.
|
|
239
|
+
*
|
|
240
|
+
* `done: false` means the ladder should continue to extraction while the
|
|
241
|
+
* candidate stays as the fallback. A render never throws its own failure
|
|
242
|
+
* into the ladder: only a caller cancellation, or an AbortError raised by
|
|
243
|
+
* the renderer itself, rejects.
|
|
244
|
+
*/
|
|
245
|
+
const renderStep = async (candidate) => {
|
|
246
|
+
const mode = liveRenderMode();
|
|
247
|
+
if (mode === 'off') {
|
|
248
|
+
return candidate === undefined
|
|
249
|
+
? { done: false, report: makeReport(mode, false, false, 'renderJs is off') }
|
|
250
|
+
: { done: true, outcome: candidate, report: makeReport(mode, false, false, 'renderJs is off') };
|
|
251
|
+
}
|
|
252
|
+
if (candidate === undefined && mode === 'auto') {
|
|
253
|
+
// `auto` needs a body to judge. When every transport failed there is
|
|
254
|
+
// no shell verdict to make, so it must not launch a browser blindly.
|
|
255
|
+
return { done: false, report: makeReport(mode, false, false, 'no page body to judge as a shell') };
|
|
256
|
+
}
|
|
257
|
+
if (candidate !== undefined && mode === 'auto') {
|
|
258
|
+
const verdict = detectShellPage(candidate);
|
|
259
|
+
if (!verdict.shell) {
|
|
260
|
+
return { done: true, outcome: candidate, report: makeReport(mode, false, false, verdict.reason) };
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
stopIfCancelled();
|
|
264
|
+
const loaded = await loadRendererOnce();
|
|
265
|
+
if (!loaded.ok) {
|
|
266
|
+
// Silent skip: no banner, no thrown error, just a recorded reason.
|
|
267
|
+
return { done: false, report: makeReport(mode, false, false, loaded.reason) };
|
|
268
|
+
}
|
|
269
|
+
try {
|
|
270
|
+
const rendered = await deps.render(loaded.renderer, target, liveRenderTimeout(), signal);
|
|
271
|
+
stopIfCancelled();
|
|
272
|
+
if (rendered.html.trim().length === 0) {
|
|
273
|
+
return {
|
|
274
|
+
done: false,
|
|
275
|
+
report: makeReport(mode, true, false, `${loaded.moduleName} returned an empty page`),
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
const { content, truncated } = clamp(rendered.html, maxBytes);
|
|
279
|
+
const outcome = {
|
|
280
|
+
url: rendered.url.length > 0 ? rendered.url : target,
|
|
281
|
+
statusCode: rendered.statusCode,
|
|
282
|
+
body: { kind: 'html', content },
|
|
283
|
+
truncated,
|
|
284
|
+
};
|
|
285
|
+
return {
|
|
286
|
+
done: true,
|
|
287
|
+
outcome,
|
|
288
|
+
report: makeReport(mode, true, true, `${loaded.moduleName} rendered ${readableTextLength(rendered.html)} readable characters`),
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
catch (error) {
|
|
292
|
+
stopIfCancelled();
|
|
293
|
+
const failure = describeRenderError(error, liveRenderTimeout());
|
|
294
|
+
if (failure.aborted) {
|
|
295
|
+
// Even without the caller's signal, the renderer's abort is a
|
|
296
|
+
// cancellation rather than a page to fall back to.
|
|
297
|
+
throw error instanceof Error ? error : new DOMException('The page render was aborted', 'AbortError');
|
|
298
|
+
}
|
|
299
|
+
// Timeout, launch failure, navigation failure: keep the earlier page.
|
|
300
|
+
return { done: false, report: makeReport(mode, true, false, failure.reason) };
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
/** Run the render step for a transport result; `undefined` means keep going. */
|
|
304
|
+
const settle = async (candidate) => {
|
|
305
|
+
const step = await renderStep(candidate);
|
|
306
|
+
lastReport = step.report;
|
|
307
|
+
if (step.done)
|
|
308
|
+
return step.outcome ?? candidate;
|
|
309
|
+
fallback = candidate;
|
|
310
|
+
failures.push(`render: ${step.report.reason}`);
|
|
311
|
+
stopIfCancelled();
|
|
312
|
+
return undefined;
|
|
313
|
+
};
|
|
314
|
+
/** Report for a result that never reached the render step. */
|
|
315
|
+
const reportFor = () => lastReport ?? makeReport(liveRenderMode(), false, false, 'the render rung did not run');
|
|
173
316
|
// A blocked host usually fails by timing out, so the direct rung gets a
|
|
174
317
|
// shorter deadline than the tunnel: spending the whole budget on a request
|
|
175
318
|
// that cannot work would leave the rung that can work with nothing.
|
|
319
|
+
let direct;
|
|
176
320
|
try {
|
|
177
|
-
|
|
321
|
+
direct = await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
|
|
178
322
|
}
|
|
179
323
|
catch (error) {
|
|
180
324
|
if (cancelled())
|
|
181
325
|
throw error;
|
|
182
326
|
failures.push(`direct: ${error instanceof Error ? error.message : String(error)}`);
|
|
183
327
|
}
|
|
328
|
+
if (direct !== undefined) {
|
|
329
|
+
const rendered = await settle(direct);
|
|
330
|
+
if (rendered !== undefined)
|
|
331
|
+
return finalize(rendered, reportFor());
|
|
332
|
+
}
|
|
184
333
|
let proxy;
|
|
185
334
|
if (options.useProxy !== false) {
|
|
186
335
|
try {
|
|
@@ -191,18 +340,40 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
191
340
|
}
|
|
192
341
|
}
|
|
193
342
|
if (proxy !== undefined && proxy.length > 0) {
|
|
343
|
+
let tunneled;
|
|
194
344
|
try {
|
|
195
|
-
|
|
345
|
+
tunneled = await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
|
|
196
346
|
}
|
|
197
347
|
catch (error) {
|
|
198
348
|
if (cancelled())
|
|
199
349
|
throw error;
|
|
200
350
|
failures.push(`tunnel: ${error instanceof Error ? error.message : String(error)}`);
|
|
201
351
|
}
|
|
352
|
+
if (tunneled !== undefined) {
|
|
353
|
+
const rendered = await settle(tunneled);
|
|
354
|
+
if (rendered !== undefined)
|
|
355
|
+
return finalize(rendered, reportFor());
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
// `always` is also a standalone rescue: a transport that returns 403 or
|
|
359
|
+
// times out may still answer a real browser, and there is no earlier page
|
|
360
|
+
// to fall back to in that branch.
|
|
361
|
+
if (fallback === undefined && liveRenderMode() === 'always') {
|
|
362
|
+
const step = await renderStep(undefined);
|
|
363
|
+
lastReport = step.report;
|
|
364
|
+
if (step.done && step.outcome !== undefined)
|
|
365
|
+
return finalize(step.outcome, step.report);
|
|
366
|
+
if (!step.done)
|
|
367
|
+
failures.push(`render: ${step.report.reason}`);
|
|
202
368
|
}
|
|
203
369
|
if (apiKey.length > 0) {
|
|
204
370
|
try {
|
|
205
|
-
|
|
371
|
+
const extracted = await deps.extract(target, apiKey, timeoutMs, signal);
|
|
372
|
+
// Say where the body came from when the browser was tried first.
|
|
373
|
+
const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
|
|
374
|
+
? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the result came from AnySearch extraction`)
|
|
375
|
+
: reportFor();
|
|
376
|
+
return finalize(extracted, report);
|
|
206
377
|
}
|
|
207
378
|
catch (error) {
|
|
208
379
|
if (cancelled())
|
|
@@ -210,6 +381,14 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
210
381
|
failures.push(`extract: ${error instanceof Error ? error.message : String(error)}`);
|
|
211
382
|
}
|
|
212
383
|
}
|
|
384
|
+
// A failed render is not a failed fetch: hand back the page an earlier
|
|
385
|
+
// rung already produced, with the reason attached.
|
|
386
|
+
if (fallback !== undefined) {
|
|
387
|
+
const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
|
|
388
|
+
? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the earlier rung's result was kept`)
|
|
389
|
+
: reportFor();
|
|
390
|
+
return finalize(fallback, report);
|
|
391
|
+
}
|
|
213
392
|
throw new SearchError(`cannot fetch ${target} (${failures.join('; ')})`, 'SEARCH_NETWORK', true);
|
|
214
393
|
},
|
|
215
394
|
};
|
package/lib/index.d.ts
CHANGED
|
@@ -74,6 +74,8 @@ declare const _default: typeof apply & {
|
|
|
74
74
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
75
75
|
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
76
76
|
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
77
|
+
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
78
|
+
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
77
79
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
78
80
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
79
81
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -99,6 +101,8 @@ declare const _default: typeof apply & {
|
|
|
99
101
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
100
102
|
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
101
103
|
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
104
|
+
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
105
|
+
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
102
106
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
103
107
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
104
108
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -140,4 +144,6 @@ export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions }
|
|
|
140
144
|
export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
|
|
141
145
|
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
|
|
142
146
|
export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
|
|
147
|
+
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
|
|
148
|
+
export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
|
|
143
149
|
export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
|
package/lib/index.js
CHANGED
|
@@ -16,6 +16,7 @@ import { EngineHealth } from "./health.js";
|
|
|
16
16
|
import { createFetchProvider } from "./fetch.js";
|
|
17
17
|
import { registerDomainTool } from "./domain-tool.js";
|
|
18
18
|
import { registerSearchManyTool } from "./search-many.js";
|
|
19
|
+
import { registerReadPageTool } from "./read-page.js";
|
|
19
20
|
export const name = 'dsh-web-search';
|
|
20
21
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
21
22
|
export const inject = ['web'];
|
|
@@ -167,10 +168,12 @@ export function apply(ctx, config = {}) {
|
|
|
167
168
|
timeoutMs: mounted.timeoutMs,
|
|
168
169
|
format: () => resolveConfig(section, undefined).fetchFormat,
|
|
169
170
|
maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
|
|
171
|
+
renderJs: () => resolveConfig(section, undefined).renderJs,
|
|
172
|
+
renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
|
|
170
173
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
171
174
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
172
175
|
}));
|
|
173
|
-
// Optional half: `domain_search` and `
|
|
176
|
+
// Optional half: `domain_search`, `search_many` and `read_page`. Fail-safe by
|
|
174
177
|
// construction — a deployment without `ctx.tools` still loads this plugin and
|
|
175
178
|
// keeps `web_search`; the registration path is the host's
|
|
176
179
|
// `ToolRuntime.register(definition)`.
|
|
@@ -193,6 +196,24 @@ export function apply(ctx, config = {}) {
|
|
|
193
196
|
if (batch !== undefined) {
|
|
194
197
|
onDispose(ctx, batch.dispose);
|
|
195
198
|
}
|
|
199
|
+
// Resumable reading: `web_fetch` cannot continue from an offset because
|
|
200
|
+
// the host request carries only `url`, but a plugin-owned tool can. It
|
|
201
|
+
// reuses the same fetch ladder and the same live Settings readers.
|
|
202
|
+
const paged = registerReadPageTool(tools, () => {
|
|
203
|
+
const live = resolveConfig(section, undefined);
|
|
204
|
+
return {
|
|
205
|
+
maxLength: live.fetchMaxLength,
|
|
206
|
+
format: live.fetchFormat,
|
|
207
|
+
timeoutMs: live.timeoutMs,
|
|
208
|
+
renderJs: live.renderJs,
|
|
209
|
+
renderTimeoutMs: live.renderTimeoutMs,
|
|
210
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
211
|
+
...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
|
|
212
|
+
};
|
|
213
|
+
}, { warn: warnTool });
|
|
214
|
+
if (paged !== undefined) {
|
|
215
|
+
onDispose(ctx, paged.dispose);
|
|
216
|
+
}
|
|
196
217
|
}
|
|
197
218
|
}
|
|
198
219
|
catch (error) {
|
|
@@ -226,3 +247,4 @@ export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
|
226
247
|
export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
227
248
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
|
228
249
|
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
|
|
250
|
+
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";
|
package/lib/rank.js
CHANGED
|
@@ -195,6 +195,9 @@ function resolveOptions(options) {
|
|
|
195
195
|
freshnessDays: positiveOr(options?.freshnessDays, DEFAULT_FRESHNESS_DAYS),
|
|
196
196
|
timeRange: options?.timeRange,
|
|
197
197
|
weights: Array.isArray(options?.weights) ? options.weights : undefined,
|
|
198
|
+
now: typeof options?.now === 'number' && Number.isFinite(options.now)
|
|
199
|
+
? options.now
|
|
200
|
+
: Date.now(),
|
|
198
201
|
};
|
|
199
202
|
}
|
|
200
203
|
/**
|
|
@@ -320,7 +323,7 @@ function fuseEntries(entries, limit, options) {
|
|
|
320
323
|
if (budget === 0)
|
|
321
324
|
return [];
|
|
322
325
|
const config = resolveOptions(options);
|
|
323
|
-
const now =
|
|
326
|
+
const now = config.now;
|
|
324
327
|
// Per-engine lists, deduplicated so one engine gets at most one vote per
|
|
325
328
|
// document and the earliest row keeps its rank.
|
|
326
329
|
const lists = [];
|