@fffangchenghao/dsh-web-search 0.24.2 → 0.24.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +5 -5
- package/README.zh-CN.md +5 -5
- package/SECURITY.md +13 -7
- package/client.js +1 -1
- package/docs/comparison.md +1 -1
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +1 -1
- package/docs/store-evidence.md +1 -1
- package/lib/bridge.js +53 -2
- package/lib/cache-persist.js +9 -1
- package/lib/cache.js +15 -6
- package/lib/fetch.js +59 -7
- package/lib/index.js +2 -2
- package/lib/markdown.js +35 -5
- package/lib/official-release.js +6 -0
- package/lib/official-sources.js +4 -2
- package/lib/official-support.js +4 -0
- package/lib/parsers.js +38 -21
- package/lib/platforms.js +65 -2
- package/lib/proxy.d.ts +9 -6
- package/lib/proxy.js +66 -23
- package/lib/rank.js +2 -2
- package/lib/read-page.js +30 -11
- package/lib/research.d.ts +5 -0
- package/lib/research.js +76 -12
- package/lib/transport.js +3 -2
- package/lib/verticals.js +11 -5
- package/package.json +1 -1
package/lib/official-sources.js
CHANGED
|
@@ -13,14 +13,16 @@ async function fetchNpmPackage(name, options) {
|
|
|
13
13
|
try {
|
|
14
14
|
const result = await searchVerticals(name, {
|
|
15
15
|
ids: ['npm'],
|
|
16
|
-
|
|
16
|
+
// Search, not a registry exact lookup: ask for a few rows and verify the
|
|
17
|
+
// package identity instead of trusting the first fuzzy result.
|
|
18
|
+
limit: 10,
|
|
17
19
|
timeoutMs: typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
|
|
18
20
|
? Math.min(options.timeoutMs, 8_000)
|
|
19
21
|
: 8_000,
|
|
20
22
|
...(options.proxy === undefined ? {} : { proxy: options.proxy }),
|
|
21
23
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
22
24
|
});
|
|
23
|
-
return result.sources
|
|
25
|
+
return result.sources.find((source) => source.title === name || source.title?.startsWith(`${name}@`) === true);
|
|
24
26
|
}
|
|
25
27
|
catch {
|
|
26
28
|
return undefined;
|
package/lib/official-support.js
CHANGED
|
@@ -75,6 +75,10 @@ export function postgresSupportSource(releases) {
|
|
|
75
75
|
}
|
|
76
76
|
/** Read the page over the configured tunnel, or directly when none is set. */
|
|
77
77
|
async function fetchText(url, options, timeoutMs) {
|
|
78
|
+
if (options.signal?.aborted === true) {
|
|
79
|
+
const reason = options.signal.reason;
|
|
80
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
81
|
+
}
|
|
78
82
|
if (options.proxy !== undefined && options.proxy.length > 0) {
|
|
79
83
|
const response = await proxyGet(url, {
|
|
80
84
|
proxy: options.proxy,
|
package/lib/parsers.js
CHANGED
|
@@ -82,38 +82,52 @@ export function resolveHref(raw, baseUrl) {
|
|
|
82
82
|
if (typeof raw !== 'string' || raw.length === 0)
|
|
83
83
|
return undefined;
|
|
84
84
|
// Attribute values are HTML-escaped, so a Bing click tracker arrives as
|
|
85
|
-
// `...&u=a1<base64url>`; decode before
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
85
|
+
// `...&u=a1<base64url>`; decode before parsing.
|
|
86
|
+
const candidate = raw.replace(/&/gi, '&').replace(/�*38;/g, '&');
|
|
87
|
+
let parsed;
|
|
88
|
+
try {
|
|
89
|
+
parsed = new URL(candidate, baseUrl);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
// Only the search engine's own wrapper is unwrapped. A normal page whose
|
|
95
|
+
// query string happens to contain `uddg` or `u=a1...` must keep its URL.
|
|
96
|
+
const host = parsed.hostname.toLowerCase();
|
|
97
|
+
let target;
|
|
98
|
+
if ((host === 'bing.com' || host === 'www.bing.com') && parsed.pathname === '/ck/a') {
|
|
99
|
+
const payload = parsed.searchParams.get('u');
|
|
100
|
+
const match = payload === null ? null : /^a1([A-Za-z0-9_-]+={0,2})$/u.exec(payload);
|
|
101
|
+
if (match !== null) {
|
|
102
|
+
try {
|
|
103
|
+
target = Buffer.from(match[1].replace(/=+$/u, '').replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
94
108
|
}
|
|
95
109
|
}
|
|
96
|
-
else {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if (uddg) {
|
|
110
|
+
else if ((host === 'duckduckgo.com' || host.endsWith('.duckduckgo.com')) && parsed.pathname === '/l/') {
|
|
111
|
+
const wrapped = parsed.searchParams.get('uddg');
|
|
112
|
+
if (wrapped !== null && wrapped.length > 0) {
|
|
100
113
|
try {
|
|
101
|
-
|
|
114
|
+
target = decodeURIComponent(wrapped);
|
|
102
115
|
}
|
|
103
116
|
catch {
|
|
104
117
|
return undefined;
|
|
105
118
|
}
|
|
106
119
|
}
|
|
107
120
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
if (target !== undefined) {
|
|
122
|
+
try {
|
|
123
|
+
const unwrapped = new URL(target);
|
|
124
|
+
return unwrapped.protocol === 'http:' || unwrapped.protocol === 'https:' ? unwrapped.toString() : undefined;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
111
127
|
return undefined;
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
catch {
|
|
115
|
-
return undefined;
|
|
128
|
+
}
|
|
116
129
|
}
|
|
130
|
+
return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed.toString() : undefined;
|
|
117
131
|
}
|
|
118
132
|
/**
|
|
119
133
|
* Parse a 360 (so.com) results page. The real URL lives in `data-mdurl`;
|
|
@@ -457,6 +471,9 @@ export function parseExaMcp(payload, baseUrl) {
|
|
|
457
471
|
}
|
|
458
472
|
else if (/^highlights\s*:/i.test(line)) {
|
|
459
473
|
inHighlights = true;
|
|
474
|
+
const inline = line.replace(/^highlights\s*:/i, '').trim();
|
|
475
|
+
if (inline.length > 0)
|
|
476
|
+
highlights.push(inline);
|
|
460
477
|
}
|
|
461
478
|
else if (/^(published|author)\s*:/i.test(line)) {
|
|
462
479
|
inHighlights = false;
|
package/lib/platforms.js
CHANGED
|
@@ -267,6 +267,26 @@ function deadline(timeoutMs, signal) {
|
|
|
267
267
|
const timer = AbortSignal.timeout(timeoutMs);
|
|
268
268
|
return signal === undefined ? timer : AbortSignal.any([signal, timer]);
|
|
269
269
|
}
|
|
270
|
+
/** Race a promise against an abort signal, observing both outcomes. */
|
|
271
|
+
function withAbort(promise, signal) {
|
|
272
|
+
if (signal.aborted) {
|
|
273
|
+
return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
|
|
274
|
+
}
|
|
275
|
+
return new Promise((resolve, reject) => {
|
|
276
|
+
const onAbort = () => {
|
|
277
|
+
signal.removeEventListener('abort', onAbort);
|
|
278
|
+
reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
|
|
279
|
+
};
|
|
280
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
281
|
+
promise.then((value) => {
|
|
282
|
+
signal.removeEventListener('abort', onAbort);
|
|
283
|
+
resolve(value);
|
|
284
|
+
}, (error) => {
|
|
285
|
+
signal.removeEventListener('abort', onAbort);
|
|
286
|
+
reject(error);
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
}
|
|
270
290
|
/** Turn a non-2xx status into a reason. */
|
|
271
291
|
function statusError(status) {
|
|
272
292
|
return new Error(status === 0 ? 'no response' : `HTTP ${status}`);
|
|
@@ -828,7 +848,11 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
|
|
|
828
848
|
}
|
|
829
849
|
const fallbackQuery = `site:${spec.site} ${query}`;
|
|
830
850
|
try {
|
|
831
|
-
|
|
851
|
+
if (options.signal?.aborted === true) {
|
|
852
|
+
const reason = options.signal.reason;
|
|
853
|
+
throw reason instanceof Error ? reason : new Error('platform search was aborted');
|
|
854
|
+
}
|
|
855
|
+
const result = await withAbort(options.fallbackSearch(fallbackQuery, limit, options.signal), options.signal ?? new AbortController().signal);
|
|
832
856
|
// A search engine may ignore the site: operator and return generic pages.
|
|
833
857
|
// A platform result must at least live on that platform's domain.
|
|
834
858
|
const allowed = normaliseSources(result.sources, limit * 2)
|
|
@@ -875,7 +899,7 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
|
|
|
875
899
|
* @param timeoutMs - per-platform deadline.
|
|
876
900
|
* @returns the platform answer.
|
|
877
901
|
*/
|
|
878
|
-
async function
|
|
902
|
+
async function searchPlatformInner(spec, query, limit, options, timeoutMs) {
|
|
879
903
|
if (spec.fallbackOnly === true || spec.build === undefined || spec.parse === undefined) {
|
|
880
904
|
if (options.fallbackSearch === undefined) {
|
|
881
905
|
return {
|
|
@@ -927,6 +951,45 @@ async function searchPlatform(spec, query, limit, options, timeoutMs) {
|
|
|
927
951
|
}
|
|
928
952
|
return fallbackAnswer(spec, query, limit, options, directReason);
|
|
929
953
|
}
|
|
954
|
+
/**
|
|
955
|
+
* Run one platform under a single deadline that also covers the fallback.
|
|
956
|
+
*
|
|
957
|
+
* `requestText` has its own per-request timeout, but an alternate endpoint or
|
|
958
|
+
* an injected fallback search can otherwise get a fresh budget, and a fallback
|
|
959
|
+
* that ignores its signal can hang the whole `Promise.all` forever.
|
|
960
|
+
*
|
|
961
|
+
* @param spec - platform being served.
|
|
962
|
+
* @param query - original query, already trimmed.
|
|
963
|
+
* @param limit - per-platform budget.
|
|
964
|
+
* @param options - caller options.
|
|
965
|
+
* @param timeoutMs - total deadline for this platform.
|
|
966
|
+
* @returns the platform answer; timeout becomes a platform failure unless the
|
|
967
|
+
* caller itself cancelled.
|
|
968
|
+
*/
|
|
969
|
+
async function searchPlatform(spec, query, limit, options, timeoutMs) {
|
|
970
|
+
const controller = new AbortController();
|
|
971
|
+
const timer = setTimeout(() => controller.abort(new Error(`platform ${spec.id} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
972
|
+
const combined = options.signal === undefined
|
|
973
|
+
? controller.signal
|
|
974
|
+
: AbortSignal.any([options.signal, controller.signal]);
|
|
975
|
+
const scoped = { ...options, signal: combined };
|
|
976
|
+
try {
|
|
977
|
+
return await withAbort(searchPlatformInner(spec, query, limit, scoped, timeoutMs), combined);
|
|
978
|
+
}
|
|
979
|
+
catch (error) {
|
|
980
|
+
if (options.signal?.aborted === true)
|
|
981
|
+
throw error;
|
|
982
|
+
return {
|
|
983
|
+
id: spec.id,
|
|
984
|
+
sources: [],
|
|
985
|
+
via: spec.fallbackOnly === true ? 'fallback' : 'api',
|
|
986
|
+
failure: failureText(spec.id, messageOf(error)),
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
finally {
|
|
990
|
+
clearTimeout(timer);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
930
993
|
/**
|
|
931
994
|
* Search the requested platforms and merge their answers.
|
|
932
995
|
*
|
package/lib/proxy.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
export interface ProxyTarget {
|
|
3
3
|
/** Proxy host name or IP literal, without brackets or port. */
|
|
4
4
|
host: string;
|
|
5
|
-
/** Proxy port; already defaulted to 80
|
|
5
|
+
/** Proxy port; already defaulted to 80 by {@link parseProxy}. */
|
|
6
6
|
port: number;
|
|
7
7
|
/** Ready-to-send `Proxy-Authorization` header value, when credentials exist. */
|
|
8
8
|
authorization?: string;
|
|
@@ -34,6 +34,8 @@ export interface ProxyRequestOptions {
|
|
|
34
34
|
method?: 'GET' | 'POST';
|
|
35
35
|
/** Request body, sent verbatim for `POST`. */
|
|
36
36
|
body?: string;
|
|
37
|
+
/** Maximum response bytes to retain; defaults to 8 MiB. */
|
|
38
|
+
maxBytes?: number;
|
|
37
39
|
}
|
|
38
40
|
/** A raw response split into its parts, before it is mapped to `{ url }`. */
|
|
39
41
|
export interface ParsedResponse {
|
|
@@ -42,11 +44,12 @@ export interface ParsedResponse {
|
|
|
42
44
|
text: string;
|
|
43
45
|
}
|
|
44
46
|
/**
|
|
45
|
-
* Parse a proxy URL
|
|
47
|
+
* Parse a proxy URL.
|
|
46
48
|
*
|
|
47
|
-
* `
|
|
48
|
-
*
|
|
49
|
-
*
|
|
49
|
+
* `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
|
|
50
|
+
* proxy would require a TLS handshake to the proxy before that CONNECT; this
|
|
51
|
+
* transport does not implement it, so the scheme is rejected instead of being
|
|
52
|
+
* half-supported (which used to fail with an opaque `socket hang up`).
|
|
50
53
|
*
|
|
51
54
|
* @param raw - candidate `http://[user:pass@]host[:port]` string.
|
|
52
55
|
* @returns the proxy host, port and optional `Proxy-Authorization` value.
|
|
@@ -119,7 +122,7 @@ export declare function resolveProxy(explicit?: string): Promise<string | undefi
|
|
|
119
122
|
* @param buffer - the bytes received from the socket.
|
|
120
123
|
* @returns the parsed response, or `undefined` when the head is incomplete.
|
|
121
124
|
*/
|
|
122
|
-
export declare function readResponse(buffer: Buffer): ParsedResponse | undefined;
|
|
125
|
+
export declare function readResponse(buffer: Buffer, maxBytes?: number): ParsedResponse | undefined;
|
|
123
126
|
/**
|
|
124
127
|
* Perform a request through the proxy, following redirects.
|
|
125
128
|
*
|
package/lib/proxy.js
CHANGED
|
@@ -30,6 +30,8 @@ const CACHE_MS = 60_000;
|
|
|
30
30
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
31
31
|
/** Default number of redirect hops to follow. */
|
|
32
32
|
const DEFAULT_REDIRECTS = 4;
|
|
33
|
+
/** Hard ceiling for one tunnel response, before decompression. */
|
|
34
|
+
const DEFAULT_MAX_BYTES = 8 * 1024 * 1024;
|
|
33
35
|
/**
|
|
34
36
|
* Environment variables read by {@link proxyFromEnv}, highest precedence first.
|
|
35
37
|
* Both casings are listed because shells disagree about which one they export.
|
|
@@ -55,11 +57,12 @@ function decodeCredential(value) {
|
|
|
55
57
|
}
|
|
56
58
|
}
|
|
57
59
|
/**
|
|
58
|
-
* Parse a proxy URL
|
|
60
|
+
* Parse a proxy URL.
|
|
59
61
|
*
|
|
60
|
-
* `
|
|
61
|
-
*
|
|
62
|
-
*
|
|
62
|
+
* `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
|
|
63
|
+
* proxy would require a TLS handshake to the proxy before that CONNECT; this
|
|
64
|
+
* transport does not implement it, so the scheme is rejected instead of being
|
|
65
|
+
* half-supported (which used to fail with an opaque `socket hang up`).
|
|
63
66
|
*
|
|
64
67
|
* @param raw - candidate `http://[user:pass@]host[:port]` string.
|
|
65
68
|
* @returns the proxy host, port and optional `Proxy-Authorization` value.
|
|
@@ -76,19 +79,19 @@ export function parseProxy(raw) {
|
|
|
76
79
|
catch {
|
|
77
80
|
throw new Error(`unparsable proxy URL: ${JSON.stringify(raw)}`);
|
|
78
81
|
}
|
|
79
|
-
if (url.protocol !== 'http:'
|
|
80
|
-
throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http://
|
|
82
|
+
if (url.protocol !== 'http:') {
|
|
83
|
+
throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http:// proxies are supported`);
|
|
81
84
|
}
|
|
82
85
|
if (url.hostname.length === 0)
|
|
83
86
|
throw new Error(`proxy URL has no host: ${JSON.stringify(raw)}`);
|
|
84
|
-
const port = url.port === '' ?
|
|
87
|
+
const port = url.port === '' ? 80 : Number(url.port);
|
|
85
88
|
if (!Number.isInteger(port) || port <= 0 || port > 65_535) {
|
|
86
89
|
throw new Error(`proxy URL has an invalid port: ${JSON.stringify(raw)}`);
|
|
87
90
|
}
|
|
88
91
|
const username = decodeCredential(url.username);
|
|
89
92
|
const password = decodeCredential(url.password);
|
|
90
93
|
const authorization = username.length === 0 && password.length === 0 ? undefined : `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
|
|
91
|
-
return { host: url.hostname, port, ...(authorization === undefined ? {} : { authorization }) };
|
|
94
|
+
return { host: url.hostname.replace(/^\[|\]$/g, ''), port, ...(authorization === undefined ? {} : { authorization }) };
|
|
92
95
|
}
|
|
93
96
|
/**
|
|
94
97
|
* Parse a proxy string for the detection paths, which treat an unusable value
|
|
@@ -215,7 +218,8 @@ async function captureProbe(command, args, timeoutMs = 2_000) {
|
|
|
215
218
|
* @returns an `http://` proxy URL, or `undefined` when disabled/unusable.
|
|
216
219
|
*/
|
|
217
220
|
export function proxyFromWindowsRegistry(output) {
|
|
218
|
-
|
|
221
|
+
const enabled = Number.parseInt(/ProxyEnable\s+REG_DWORD\s+0x([0-9a-f]+)/iu.exec(output)?.[1] ?? '0', 16);
|
|
222
|
+
if (!Number.isFinite(enabled) || enabled === 0)
|
|
219
223
|
return undefined;
|
|
220
224
|
const raw = /ProxyServer\s+REG_SZ\s+(.+)/iu.exec(output)?.[1]?.trim();
|
|
221
225
|
if (raw === undefined || raw.length === 0)
|
|
@@ -272,7 +276,7 @@ export async function proxyFromSystem() {
|
|
|
272
276
|
/** Check that something is listening, so a stale setting cannot stall a search. */
|
|
273
277
|
async function reachable(proxy, timeoutMs = 400) {
|
|
274
278
|
return await new Promise((resolve) => {
|
|
275
|
-
const socket = net.connect({ host: proxy.hostname, port: Number(proxy.port || 80) });
|
|
279
|
+
const socket = net.connect({ host: proxy.hostname.replace(/^\[|\]$/g, ''), port: Number(proxy.port || 80) });
|
|
276
280
|
const done = (value) => {
|
|
277
281
|
socket.destroy();
|
|
278
282
|
resolve(value);
|
|
@@ -314,13 +318,15 @@ export async function resolveProxy(explicit) {
|
|
|
314
318
|
}
|
|
315
319
|
/** Open a `CONNECT` tunnel to `host:port` through the proxy. */
|
|
316
320
|
async function openTunnel(proxy, host, port, timeoutMs, signal) {
|
|
321
|
+
const bare = host.replace(/^\[|\]$/g, '');
|
|
322
|
+
const authority = net.isIP(bare) === 6 ? `[${bare}]:${port}` : `${bare}:${port}`;
|
|
317
323
|
const request = http.request({
|
|
318
324
|
host: proxy.host,
|
|
319
325
|
port: proxy.port,
|
|
320
326
|
method: 'CONNECT',
|
|
321
|
-
path:
|
|
327
|
+
path: authority,
|
|
322
328
|
headers: {
|
|
323
|
-
host:
|
|
329
|
+
host: authority,
|
|
324
330
|
...(proxy.authorization === undefined ? {} : { 'proxy-authorization': proxy.authorization }),
|
|
325
331
|
},
|
|
326
332
|
});
|
|
@@ -344,9 +350,10 @@ async function openTunnel(proxy, host, port, timeoutMs, signal) {
|
|
|
344
350
|
}
|
|
345
351
|
}
|
|
346
352
|
/** Write one request over a connected socket and collect the raw response. */
|
|
347
|
-
function exchange(socket, head, timeoutMs, signal) {
|
|
353
|
+
function exchange(socket, head, timeoutMs, signal, maxBytes = DEFAULT_MAX_BYTES) {
|
|
348
354
|
return new Promise((resolve, reject) => {
|
|
349
355
|
const chunks = [];
|
|
356
|
+
let size = 0;
|
|
350
357
|
let settled = false;
|
|
351
358
|
const done = (settle, value) => {
|
|
352
359
|
if (settled)
|
|
@@ -369,7 +376,23 @@ function exchange(socket, head, timeoutMs, signal) {
|
|
|
369
376
|
const timer = setTimeout(() => fail(new Error('request timed out')), timeoutMs);
|
|
370
377
|
const abort = () => fail(new Error('aborted'));
|
|
371
378
|
signal?.addEventListener('abort', abort, { once: true });
|
|
372
|
-
socket.on('data', (chunk) =>
|
|
379
|
+
socket.on('data', (chunk) => {
|
|
380
|
+
if (settled)
|
|
381
|
+
return;
|
|
382
|
+
const left = maxBytes - size;
|
|
383
|
+
if (left <= 0) {
|
|
384
|
+
done(resolve, Buffer.concat(chunks));
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
if (chunk.length > left) {
|
|
388
|
+
chunks.push(chunk.subarray(0, left));
|
|
389
|
+
size += left;
|
|
390
|
+
done(resolve, Buffer.concat(chunks));
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
chunks.push(chunk);
|
|
394
|
+
size += chunk.length;
|
|
395
|
+
});
|
|
373
396
|
socket.on('end', () => done(resolve, Buffer.concat(chunks)));
|
|
374
397
|
socket.on('error', (error) => fail(error));
|
|
375
398
|
socket.write(head);
|
|
@@ -402,7 +425,7 @@ function dechunk(buffer) {
|
|
|
402
425
|
* @param buffer - the bytes received from the socket.
|
|
403
426
|
* @returns the parsed response, or `undefined` when the head is incomplete.
|
|
404
427
|
*/
|
|
405
|
-
export function readResponse(buffer) {
|
|
428
|
+
export function readResponse(buffer, maxBytes = DEFAULT_MAX_BYTES) {
|
|
406
429
|
const separator = buffer.indexOf('\r\n\r\n');
|
|
407
430
|
if (separator === -1)
|
|
408
431
|
return undefined;
|
|
@@ -424,15 +447,24 @@ export function readResponse(buffer) {
|
|
|
424
447
|
const encoding = (headers['content-encoding'] ?? '').toLowerCase();
|
|
425
448
|
try {
|
|
426
449
|
if (encoding.includes('gzip'))
|
|
427
|
-
body = zlib.gunzipSync(body);
|
|
450
|
+
body = zlib.gunzipSync(body, { maxOutputLength: maxBytes });
|
|
428
451
|
else if (encoding.includes('deflate'))
|
|
429
|
-
body = zlib.inflateSync(body);
|
|
452
|
+
body = zlib.inflateSync(body, { maxOutputLength: maxBytes });
|
|
430
453
|
else if (encoding.includes('br'))
|
|
431
|
-
body = zlib.brotliDecompressSync(body);
|
|
454
|
+
body = zlib.brotliDecompressSync(body, { maxOutputLength: maxBytes });
|
|
432
455
|
}
|
|
433
|
-
catch {
|
|
434
|
-
//
|
|
456
|
+
catch (error) {
|
|
457
|
+
// If a decompression bomb hit the output ceiling, never fall back to the
|
|
458
|
+
// compressed bytes: they are not text. Keep a short explicit marker so the
|
|
459
|
+
// caller can see that the page was too large. A genuinely malformed or
|
|
460
|
+
// mislabelled body stays as-is, as before.
|
|
461
|
+
const code = error.code;
|
|
462
|
+
if (code === 'ERR_BUFFER_TOO_LARGE' || code === 'Z_BUF_ERROR') {
|
|
463
|
+
body = Buffer.from('[response body exceeded the decompression limit]');
|
|
464
|
+
}
|
|
435
465
|
}
|
|
466
|
+
if (body.length > maxBytes)
|
|
467
|
+
body = body.subarray(0, maxBytes);
|
|
436
468
|
return { status, headers, text: body.toString('utf8') };
|
|
437
469
|
}
|
|
438
470
|
/**
|
|
@@ -454,19 +486,30 @@ export function readResponse(buffer) {
|
|
|
454
486
|
* @returns `{ status, text, url, headers }` for the final hop.
|
|
455
487
|
*/
|
|
456
488
|
export async function proxyGet(url, options) {
|
|
489
|
+
if (options.signal?.aborted === true) {
|
|
490
|
+
const reason = options.signal.reason;
|
|
491
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
492
|
+
}
|
|
457
493
|
const proxy = parseProxy(options.proxy);
|
|
458
494
|
const timeoutMs = options.timeoutMs !== undefined && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
|
|
459
495
|
const redirects = options.redirects !== undefined && Number.isInteger(options.redirects) ? options.redirects : DEFAULT_REDIRECTS;
|
|
496
|
+
const maxBytes = options.maxBytes !== undefined && Number.isFinite(options.maxBytes) && options.maxBytes > 0
|
|
497
|
+
? Math.floor(options.maxBytes)
|
|
498
|
+
: DEFAULT_MAX_BYTES;
|
|
460
499
|
let current = new URL(url);
|
|
461
500
|
let method = options.method === 'POST' ? 'POST' : 'GET';
|
|
462
501
|
let payload = method === 'POST' && typeof options.body === 'string' ? options.body : undefined;
|
|
463
502
|
for (let hop = 0; hop <= redirects; hop += 1) {
|
|
464
503
|
const secure = current.protocol === 'https:';
|
|
465
504
|
const port = Number(current.port || (secure ? 443 : 80));
|
|
466
|
-
const
|
|
505
|
+
const targetHost = current.hostname.replace(/^\[|\]$/g, '');
|
|
506
|
+
const raw = await openTunnel(proxy, targetHost, port, timeoutMs, options.signal);
|
|
467
507
|
let socket = raw;
|
|
468
508
|
if (secure) {
|
|
469
|
-
const
|
|
509
|
+
const servername = net.isIP(targetHost) === 0 ? targetHost : undefined;
|
|
510
|
+
// `host` is the identity Node checks the certificate against. Omitting it
|
|
511
|
+
// makes the check fall back to the proxy host for IP targets.
|
|
512
|
+
const upgraded = tls.connect({ socket: raw, host: targetHost, ...(servername === undefined ? {} : { servername }) });
|
|
470
513
|
// `once` also rejects on 'error', so a refused TLS handshake surfaces
|
|
471
514
|
// instead of becoming an unhandled socket event.
|
|
472
515
|
await once(upgraded, 'secureConnect');
|
|
@@ -485,7 +528,7 @@ export async function proxyGet(url, options) {
|
|
|
485
528
|
'\r\n' +
|
|
486
529
|
(payload ?? '');
|
|
487
530
|
// Follow one redirect hop at a time, re-opening the tunnel per hop.
|
|
488
|
-
const response = readResponse(await exchange(socket, head, timeoutMs, options.signal));
|
|
531
|
+
const response = readResponse(await exchange(socket, head, timeoutMs, options.signal, maxBytes), maxBytes);
|
|
489
532
|
const location = response?.headers.location;
|
|
490
533
|
if (response !== undefined && response.status >= 300 && response.status < 400 && location !== undefined && hop < redirects) {
|
|
491
534
|
// 307/308 preserve the method and body; the older statuses downgrade to GET.
|
package/lib/rank.js
CHANGED
|
@@ -360,7 +360,7 @@ function fuseEntries(entries, limit, options) {
|
|
|
360
360
|
weights.push(engineWeight(config.weights, entry.engineIndex, entry.host));
|
|
361
361
|
}
|
|
362
362
|
const rounds = lists.reduce((deepest, rows, index) => {
|
|
363
|
-
const weight = weights
|
|
363
|
+
const weight = slotWeight(weights, index);
|
|
364
364
|
if (weight === 0 || rows.length === 0)
|
|
365
365
|
return deepest;
|
|
366
366
|
return Math.max(deepest, Math.ceil(rows.length / weight));
|
|
@@ -377,7 +377,7 @@ function fuseEntries(entries, limit, options) {
|
|
|
377
377
|
const rows = lists[index];
|
|
378
378
|
if (rows === undefined)
|
|
379
379
|
continue;
|
|
380
|
-
const weight = weights
|
|
380
|
+
const weight = slotWeight(weights, index);
|
|
381
381
|
for (let offset = 0; offset < weight; offset += 1) {
|
|
382
382
|
const row = rows[round * weight + offset];
|
|
383
383
|
if (row === undefined)
|
package/lib/read-page.js
CHANGED
|
@@ -401,20 +401,39 @@ export function createReadPageDefinition(name, readSettings, options = {}) {
|
|
|
401
401
|
if (parsed.offset > totalLength) {
|
|
402
402
|
throw new Error(`read_page: offset ${parsed.offset} is past the end of ${parsed.url} (totalLength ${totalLength}); use offset 0 to start over or ${totalLength} to read the tail`);
|
|
403
403
|
}
|
|
404
|
-
//
|
|
405
|
-
//
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
if (
|
|
410
|
-
end
|
|
404
|
+
// Walk the real page boundaries. A surrogate pair at a cut point moves
|
|
405
|
+
// that boundary one code unit back, so the chunk count and index cannot
|
|
406
|
+
// be derived from `ceil(totalLength / maxLength)` without drifting.
|
|
407
|
+
const pageEnd = (start) => {
|
|
408
|
+
let end = Math.min(totalLength, start + parsed.maxLength);
|
|
409
|
+
if (end > start && end < totalLength) {
|
|
410
|
+
const last = page.content.charCodeAt(end - 1);
|
|
411
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
412
|
+
end -= 1;
|
|
413
|
+
}
|
|
414
|
+
// A one-character budget on a surrogate pair must still advance.
|
|
415
|
+
return end > start ? end : Math.min(totalLength, start + 1);
|
|
416
|
+
};
|
|
417
|
+
let count = 0;
|
|
418
|
+
let index = 0;
|
|
419
|
+
let cursor = 0;
|
|
420
|
+
while (cursor < totalLength) {
|
|
421
|
+
const boundary = pageEnd(cursor);
|
|
422
|
+
count += 1;
|
|
423
|
+
if (parsed.offset >= cursor && parsed.offset < boundary)
|
|
424
|
+
index = count;
|
|
425
|
+
cursor = boundary;
|
|
426
|
+
}
|
|
427
|
+
if (totalLength === 0) {
|
|
428
|
+
count = 1;
|
|
429
|
+
index = 1;
|
|
430
|
+
}
|
|
431
|
+
else if (parsed.offset === totalLength) {
|
|
432
|
+
index = count;
|
|
411
433
|
}
|
|
434
|
+
const end = pageEnd(parsed.offset);
|
|
412
435
|
const content = page.content.slice(parsed.offset, end);
|
|
413
436
|
const nextOffset = end < totalLength ? end : null;
|
|
414
|
-
const count = totalLength === 0 ? 1 : Math.ceil(totalLength / parsed.maxLength);
|
|
415
|
-
const index = totalLength === 0
|
|
416
|
-
? 1
|
|
417
|
-
: Math.min(count, Math.floor(parsed.offset / parsed.maxLength) + 1);
|
|
418
437
|
const value = {
|
|
419
438
|
url: parsed.url,
|
|
420
439
|
content,
|
package/lib/research.d.ts
CHANGED
|
@@ -87,6 +87,11 @@ export interface ResearchValue {
|
|
|
87
87
|
}>;
|
|
88
88
|
/** Fetch failures, one entry per URL. */
|
|
89
89
|
failures: ResearchFetchFailure[];
|
|
90
|
+
/** Search errors, one entry per failed query; omitted when none failed. */
|
|
91
|
+
searchFailures?: Array<{
|
|
92
|
+
query: string;
|
|
93
|
+
reason: string;
|
|
94
|
+
}>;
|
|
90
95
|
/** Number of search rows seen before the fetch budget cut them. */
|
|
91
96
|
found: number;
|
|
92
97
|
/** True when the search row list was cut by the fetch budget. */
|