@fffangchenghao/dsh-web-search 0.24.2 → 0.24.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,14 +13,16 @@ async function fetchNpmPackage(name, options) {
13
13
  try {
14
14
  const result = await searchVerticals(name, {
15
15
  ids: ['npm'],
16
- limit: 1,
16
+ // Search, not a registry exact lookup: ask for a few rows and verify the
17
+ // package identity instead of trusting the first fuzzy result.
18
+ limit: 10,
17
19
  timeoutMs: typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
18
20
  ? Math.min(options.timeoutMs, 8_000)
19
21
  : 8_000,
20
22
  ...(options.proxy === undefined ? {} : { proxy: options.proxy }),
21
23
  ...(options.signal === undefined ? {} : { signal: options.signal }),
22
24
  });
23
- return result.sources[0];
25
+ return result.sources.find((source) => source.title === name || source.title?.startsWith(`${name}@`) === true);
24
26
  }
25
27
  catch {
26
28
  return undefined;
@@ -75,6 +75,10 @@ export function postgresSupportSource(releases) {
75
75
  }
76
76
  /** Read the page over the configured tunnel, or directly when none is set. */
77
77
  async function fetchText(url, options, timeoutMs) {
78
+ if (options.signal?.aborted === true) {
79
+ const reason = options.signal.reason;
80
+ throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
81
+ }
78
82
  if (options.proxy !== undefined && options.proxy.length > 0) {
79
83
  const response = await proxyGet(url, {
80
84
  proxy: options.proxy,
package/lib/parsers.js CHANGED
@@ -82,38 +82,52 @@ export function resolveHref(raw, baseUrl) {
82
82
  if (typeof raw !== 'string' || raw.length === 0)
83
83
  return undefined;
84
84
  // Attribute values are HTML-escaped, so a Bing click tracker arrives as
85
- // `...&amp;u=a1<base64url>`; decode before looking for the payload.
86
- let candidate = raw.replace(/&amp;/gi, '&').replace(/&#0*38;/g, '&');
87
- const wrapped = /[?&]u=a1([A-Za-z0-9_-]+)/.exec(candidate);
88
- if (wrapped) {
89
- try {
90
- candidate = Buffer.from(wrapped[1].replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
91
- }
92
- catch {
93
- return undefined;
85
+ // `...&amp;u=a1<base64url>`; decode before parsing.
86
+ const candidate = raw.replace(/&amp;/gi, '&').replace(/&#0*38;/g, '&');
87
+ let parsed;
88
+ try {
89
+ parsed = new URL(candidate, baseUrl);
90
+ }
91
+ catch {
92
+ return undefined;
93
+ }
94
+ // Only the search engine's own wrapper is unwrapped. A normal page whose
95
+ // query string happens to contain `uddg` or `u=a1...` must keep its URL.
96
+ const host = parsed.hostname.toLowerCase();
97
+ let target;
98
+ if ((host === 'bing.com' || host === 'www.bing.com') && parsed.pathname === '/ck/a') {
99
+ const payload = parsed.searchParams.get('u');
100
+ const match = payload === null ? null : /^a1([A-Za-z0-9_-]+={0,2})$/u.exec(payload);
101
+ if (match !== null) {
102
+ try {
103
+ target = Buffer.from(match[1].replace(/=+$/u, '').replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
104
+ }
105
+ catch {
106
+ return undefined;
107
+ }
94
108
  }
95
109
  }
96
- else {
97
- // DuckDuckGo wraps outbound links as `/l/?uddg=<percent-encoded>`.
98
- const uddg = /[?&]uddg=([^&]+)/.exec(candidate);
99
- if (uddg) {
110
+ else if ((host === 'duckduckgo.com' || host.endsWith('.duckduckgo.com')) && parsed.pathname === '/l/') {
111
+ const wrapped = parsed.searchParams.get('uddg');
112
+ if (wrapped !== null && wrapped.length > 0) {
100
113
  try {
101
- candidate = decodeURIComponent(uddg[1]);
114
+ target = decodeURIComponent(wrapped);
102
115
  }
103
116
  catch {
104
117
  return undefined;
105
118
  }
106
119
  }
107
120
  }
108
- try {
109
- const url = new URL(candidate, baseUrl);
110
- if (url.protocol !== 'http:' && url.protocol !== 'https:')
121
+ if (target !== undefined) {
122
+ try {
123
+ const unwrapped = new URL(target);
124
+ return unwrapped.protocol === 'http:' || unwrapped.protocol === 'https:' ? unwrapped.toString() : undefined;
125
+ }
126
+ catch {
111
127
  return undefined;
112
- return url.toString();
113
- }
114
- catch {
115
- return undefined;
128
+ }
116
129
  }
130
+ return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed.toString() : undefined;
117
131
  }
118
132
  /**
119
133
  * Parse a 360 (so.com) results page. The real URL lives in `data-mdurl`;
@@ -457,6 +471,9 @@ export function parseExaMcp(payload, baseUrl) {
457
471
  }
458
472
  else if (/^highlights\s*:/i.test(line)) {
459
473
  inHighlights = true;
474
+ const inline = line.replace(/^highlights\s*:/i, '').trim();
475
+ if (inline.length > 0)
476
+ highlights.push(inline);
460
477
  }
461
478
  else if (/^(published|author)\s*:/i.test(line)) {
462
479
  inHighlights = false;
package/lib/platforms.js CHANGED
@@ -267,6 +267,26 @@ function deadline(timeoutMs, signal) {
267
267
  const timer = AbortSignal.timeout(timeoutMs);
268
268
  return signal === undefined ? timer : AbortSignal.any([signal, timer]);
269
269
  }
270
+ /** Race a promise against an abort signal, observing both outcomes. */
271
+ function withAbort(promise, signal) {
272
+ if (signal.aborted) {
273
+ return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
274
+ }
275
+ return new Promise((resolve, reject) => {
276
+ const onAbort = () => {
277
+ signal.removeEventListener('abort', onAbort);
278
+ reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
279
+ };
280
+ signal.addEventListener('abort', onAbort, { once: true });
281
+ promise.then((value) => {
282
+ signal.removeEventListener('abort', onAbort);
283
+ resolve(value);
284
+ }, (error) => {
285
+ signal.removeEventListener('abort', onAbort);
286
+ reject(error);
287
+ });
288
+ });
289
+ }
270
290
  /** Turn a non-2xx status into a reason. */
271
291
  function statusError(status) {
272
292
  return new Error(status === 0 ? 'no response' : `HTTP ${status}`);
@@ -828,7 +848,11 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
828
848
  }
829
849
  const fallbackQuery = `site:${spec.site} ${query}`;
830
850
  try {
831
- const result = await options.fallbackSearch(fallbackQuery, limit, options.signal);
851
+ if (options.signal?.aborted === true) {
852
+ const reason = options.signal.reason;
853
+ throw reason instanceof Error ? reason : new Error('platform search was aborted');
854
+ }
855
+ const result = await withAbort(options.fallbackSearch(fallbackQuery, limit, options.signal), options.signal ?? new AbortController().signal);
832
856
  // A search engine may ignore the site: operator and return generic pages.
833
857
  // A platform result must at least live on that platform's domain.
834
858
  const allowed = normaliseSources(result.sources, limit * 2)
@@ -875,7 +899,7 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
875
899
  * @param timeoutMs - per-platform deadline.
876
900
  * @returns the platform answer.
877
901
  */
878
- async function searchPlatform(spec, query, limit, options, timeoutMs) {
902
+ async function searchPlatformInner(spec, query, limit, options, timeoutMs) {
879
903
  if (spec.fallbackOnly === true || spec.build === undefined || spec.parse === undefined) {
880
904
  if (options.fallbackSearch === undefined) {
881
905
  return {
@@ -927,6 +951,45 @@ async function searchPlatform(spec, query, limit, options, timeoutMs) {
927
951
  }
928
952
  return fallbackAnswer(spec, query, limit, options, directReason);
929
953
  }
954
+ /**
955
+ * Run one platform under a single deadline that also covers the fallback.
956
+ *
957
+ * `requestText` has its own per-request timeout, but an alternate endpoint or
958
+ * an injected fallback search can otherwise get a fresh budget, and a fallback
959
+ * that ignores its signal can hang the whole `Promise.all` forever.
960
+ *
961
+ * @param spec - platform being served.
962
+ * @param query - original query, already trimmed.
963
+ * @param limit - per-platform budget.
964
+ * @param options - caller options.
965
+ * @param timeoutMs - total deadline for this platform.
966
+ * @returns the platform answer; timeout becomes a platform failure unless the
967
+ * caller itself cancelled.
968
+ */
969
+ async function searchPlatform(spec, query, limit, options, timeoutMs) {
970
+ const controller = new AbortController();
971
+ const timer = setTimeout(() => controller.abort(new Error(`platform ${spec.id} timed out after ${timeoutMs}ms`)), timeoutMs);
972
+ const combined = options.signal === undefined
973
+ ? controller.signal
974
+ : AbortSignal.any([options.signal, controller.signal]);
975
+ const scoped = { ...options, signal: combined };
976
+ try {
977
+ return await withAbort(searchPlatformInner(spec, query, limit, scoped, timeoutMs), combined);
978
+ }
979
+ catch (error) {
980
+ if (options.signal?.aborted === true)
981
+ throw error;
982
+ return {
983
+ id: spec.id,
984
+ sources: [],
985
+ via: spec.fallbackOnly === true ? 'fallback' : 'api',
986
+ failure: failureText(spec.id, messageOf(error)),
987
+ };
988
+ }
989
+ finally {
990
+ clearTimeout(timer);
991
+ }
992
+ }
930
993
  /**
931
994
  * Search the requested platforms and merge their answers.
932
995
  *
package/lib/proxy.d.ts CHANGED
@@ -2,7 +2,7 @@
2
2
  export interface ProxyTarget {
3
3
  /** Proxy host name or IP literal, without brackets or port. */
4
4
  host: string;
5
- /** Proxy port; already defaulted to 80 or 443 by {@link parseProxy}. */
5
+ /** Proxy port; already defaulted to 80 by {@link parseProxy}. */
6
6
  port: number;
7
7
  /** Ready-to-send `Proxy-Authorization` header value, when credentials exist. */
8
8
  authorization?: string;
@@ -34,6 +34,8 @@ export interface ProxyRequestOptions {
34
34
  method?: 'GET' | 'POST';
35
35
  /** Request body, sent verbatim for `POST`. */
36
36
  body?: string;
37
+ /** Maximum response bytes to retain; defaults to 8 MiB. */
38
+ maxBytes?: number;
37
39
  }
38
40
  /** A raw response split into its parts, before it is mapped to `{ url }`. */
39
41
  export interface ParsedResponse {
@@ -42,11 +44,12 @@ export interface ParsedResponse {
42
44
  text: string;
43
45
  }
44
46
  /**
45
- * Parse a proxy URL, accepting only the schemes `CONNECT` can tunnel through.
47
+ * Parse a proxy URL.
46
48
  *
47
- * `http:` is the normal case; `https:` is accepted for its default port only —
48
- * the tunnel itself is always opened with a plain `CONNECT`, because the
49
- * request written inside it is already TLS for `https:` targets.
49
+ * `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
50
+ * proxy would require a TLS handshake to the proxy before that CONNECT; this
51
+ * transport does not implement it, so the scheme is rejected instead of being
52
+ * half-supported (which used to fail with an opaque `socket hang up`).
50
53
  *
51
54
  * @param raw - candidate `http://[user:pass@]host[:port]` string.
52
55
  * @returns the proxy host, port and optional `Proxy-Authorization` value.
@@ -119,7 +122,7 @@ export declare function resolveProxy(explicit?: string): Promise<string | undefi
119
122
  * @param buffer - the bytes received from the socket.
120
123
  * @returns the parsed response, or `undefined` when the head is incomplete.
121
124
  */
122
- export declare function readResponse(buffer: Buffer): ParsedResponse | undefined;
125
+ export declare function readResponse(buffer: Buffer, maxBytes?: number): ParsedResponse | undefined;
123
126
  /**
124
127
  * Perform a request through the proxy, following redirects.
125
128
  *
package/lib/proxy.js CHANGED
@@ -30,6 +30,8 @@ const CACHE_MS = 60_000;
30
30
  const DEFAULT_TIMEOUT_MS = 15_000;
31
31
  /** Default number of redirect hops to follow. */
32
32
  const DEFAULT_REDIRECTS = 4;
33
+ /** Hard ceiling for one tunnel response, before decompression. */
34
+ const DEFAULT_MAX_BYTES = 8 * 1024 * 1024;
33
35
  /**
34
36
  * Environment variables read by {@link proxyFromEnv}, highest precedence first.
35
37
  * Both casings are listed because shells disagree about which one they export.
@@ -55,11 +57,12 @@ function decodeCredential(value) {
55
57
  }
56
58
  }
57
59
  /**
58
- * Parse a proxy URL, accepting only the schemes `CONNECT` can tunnel through.
60
+ * Parse a proxy URL.
59
61
  *
60
- * `http:` is the normal case; `https:` is accepted for its default port only —
61
- * the tunnel itself is always opened with a plain `CONNECT`, because the
62
- * request written inside it is already TLS for `https:` targets.
62
+ * `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
63
+ * proxy would require a TLS handshake to the proxy before that CONNECT; this
64
+ * transport does not implement it, so the scheme is rejected instead of being
65
+ * half-supported (which used to fail with an opaque `socket hang up`).
63
66
  *
64
67
  * @param raw - candidate `http://[user:pass@]host[:port]` string.
65
68
  * @returns the proxy host, port and optional `Proxy-Authorization` value.
@@ -76,19 +79,19 @@ export function parseProxy(raw) {
76
79
  catch {
77
80
  throw new Error(`unparsable proxy URL: ${JSON.stringify(raw)}`);
78
81
  }
79
- if (url.protocol !== 'http:' && url.protocol !== 'https:') {
80
- throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http:// and https:// proxies are supported`);
82
+ if (url.protocol !== 'http:') {
83
+ throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http:// proxies are supported`);
81
84
  }
82
85
  if (url.hostname.length === 0)
83
86
  throw new Error(`proxy URL has no host: ${JSON.stringify(raw)}`);
84
- const port = url.port === '' ? (url.protocol === 'https:' ? 443 : 80) : Number(url.port);
87
+ const port = url.port === '' ? 80 : Number(url.port);
85
88
  if (!Number.isInteger(port) || port <= 0 || port > 65_535) {
86
89
  throw new Error(`proxy URL has an invalid port: ${JSON.stringify(raw)}`);
87
90
  }
88
91
  const username = decodeCredential(url.username);
89
92
  const password = decodeCredential(url.password);
90
93
  const authorization = username.length === 0 && password.length === 0 ? undefined : `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
91
- return { host: url.hostname, port, ...(authorization === undefined ? {} : { authorization }) };
94
+ return { host: url.hostname.replace(/^\[|\]$/g, ''), port, ...(authorization === undefined ? {} : { authorization }) };
92
95
  }
93
96
  /**
94
97
  * Parse a proxy string for the detection paths, which treat an unusable value
@@ -215,7 +218,8 @@ async function captureProbe(command, args, timeoutMs = 2_000) {
215
218
  * @returns an `http://` proxy URL, or `undefined` when disabled/unusable.
216
219
  */
217
220
  export function proxyFromWindowsRegistry(output) {
218
- if ((/ProxyEnable\s+REG_DWORD\s+0x([0-9a-f]+)/iu.exec(output)?.[1] ?? '0') === '0')
221
+ const enabled = Number.parseInt(/ProxyEnable\s+REG_DWORD\s+0x([0-9a-f]+)/iu.exec(output)?.[1] ?? '0', 16);
222
+ if (!Number.isFinite(enabled) || enabled === 0)
219
223
  return undefined;
220
224
  const raw = /ProxyServer\s+REG_SZ\s+(.+)/iu.exec(output)?.[1]?.trim();
221
225
  if (raw === undefined || raw.length === 0)
@@ -272,7 +276,7 @@ export async function proxyFromSystem() {
272
276
  /** Check that something is listening, so a stale setting cannot stall a search. */
273
277
  async function reachable(proxy, timeoutMs = 400) {
274
278
  return await new Promise((resolve) => {
275
- const socket = net.connect({ host: proxy.hostname, port: Number(proxy.port || 80) });
279
+ const socket = net.connect({ host: proxy.hostname.replace(/^\[|\]$/g, ''), port: Number(proxy.port || 80) });
276
280
  const done = (value) => {
277
281
  socket.destroy();
278
282
  resolve(value);
@@ -314,13 +318,15 @@ export async function resolveProxy(explicit) {
314
318
  }
315
319
  /** Open a `CONNECT` tunnel to `host:port` through the proxy. */
316
320
  async function openTunnel(proxy, host, port, timeoutMs, signal) {
321
+ const bare = host.replace(/^\[|\]$/g, '');
322
+ const authority = net.isIP(bare) === 6 ? `[${bare}]:${port}` : `${bare}:${port}`;
317
323
  const request = http.request({
318
324
  host: proxy.host,
319
325
  port: proxy.port,
320
326
  method: 'CONNECT',
321
- path: `${host}:${port}`,
327
+ path: authority,
322
328
  headers: {
323
- host: `${host}:${port}`,
329
+ host: authority,
324
330
  ...(proxy.authorization === undefined ? {} : { 'proxy-authorization': proxy.authorization }),
325
331
  },
326
332
  });
@@ -344,9 +350,10 @@ async function openTunnel(proxy, host, port, timeoutMs, signal) {
344
350
  }
345
351
  }
346
352
  /** Write one request over a connected socket and collect the raw response. */
347
- function exchange(socket, head, timeoutMs, signal) {
353
+ function exchange(socket, head, timeoutMs, signal, maxBytes = DEFAULT_MAX_BYTES) {
348
354
  return new Promise((resolve, reject) => {
349
355
  const chunks = [];
356
+ let size = 0;
350
357
  let settled = false;
351
358
  const done = (settle, value) => {
352
359
  if (settled)
@@ -369,7 +376,23 @@ function exchange(socket, head, timeoutMs, signal) {
369
376
  const timer = setTimeout(() => fail(new Error('request timed out')), timeoutMs);
370
377
  const abort = () => fail(new Error('aborted'));
371
378
  signal?.addEventListener('abort', abort, { once: true });
372
- socket.on('data', (chunk) => chunks.push(chunk));
379
+ socket.on('data', (chunk) => {
380
+ if (settled)
381
+ return;
382
+ const left = maxBytes - size;
383
+ if (left <= 0) {
384
+ done(resolve, Buffer.concat(chunks));
385
+ return;
386
+ }
387
+ if (chunk.length > left) {
388
+ chunks.push(chunk.subarray(0, left));
389
+ size += left;
390
+ done(resolve, Buffer.concat(chunks));
391
+ return;
392
+ }
393
+ chunks.push(chunk);
394
+ size += chunk.length;
395
+ });
373
396
  socket.on('end', () => done(resolve, Buffer.concat(chunks)));
374
397
  socket.on('error', (error) => fail(error));
375
398
  socket.write(head);
@@ -402,7 +425,7 @@ function dechunk(buffer) {
402
425
  * @param buffer - the bytes received from the socket.
403
426
  * @returns the parsed response, or `undefined` when the head is incomplete.
404
427
  */
405
- export function readResponse(buffer) {
428
+ export function readResponse(buffer, maxBytes = DEFAULT_MAX_BYTES) {
406
429
  const separator = buffer.indexOf('\r\n\r\n');
407
430
  if (separator === -1)
408
431
  return undefined;
@@ -424,15 +447,24 @@ export function readResponse(buffer) {
424
447
  const encoding = (headers['content-encoding'] ?? '').toLowerCase();
425
448
  try {
426
449
  if (encoding.includes('gzip'))
427
- body = zlib.gunzipSync(body);
450
+ body = zlib.gunzipSync(body, { maxOutputLength: maxBytes });
428
451
  else if (encoding.includes('deflate'))
429
- body = zlib.inflateSync(body);
452
+ body = zlib.inflateSync(body, { maxOutputLength: maxBytes });
430
453
  else if (encoding.includes('br'))
431
- body = zlib.brotliDecompressSync(body);
454
+ body = zlib.brotliDecompressSync(body, { maxOutputLength: maxBytes });
432
455
  }
433
- catch {
434
- // A truncated or mislabelled body is still worth parsing as-is.
456
+ catch (error) {
457
+ // If a decompression bomb hit the output ceiling, never fall back to the
458
+ // compressed bytes: they are not text. Keep a short explicit marker so the
459
+ // caller can see that the page was too large. A genuinely malformed or
460
+ // mislabelled body stays as-is, as before.
461
+ const code = error.code;
462
+ if (code === 'ERR_BUFFER_TOO_LARGE' || code === 'Z_BUF_ERROR') {
463
+ body = Buffer.from('[response body exceeded the decompression limit]');
464
+ }
435
465
  }
466
+ if (body.length > maxBytes)
467
+ body = body.subarray(0, maxBytes);
436
468
  return { status, headers, text: body.toString('utf8') };
437
469
  }
438
470
  /**
@@ -454,19 +486,30 @@ export function readResponse(buffer) {
454
486
  * @returns `{ status, text, url, headers }` for the final hop.
455
487
  */
456
488
  export async function proxyGet(url, options) {
489
+ if (options.signal?.aborted === true) {
490
+ const reason = options.signal.reason;
491
+ throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
492
+ }
457
493
  const proxy = parseProxy(options.proxy);
458
494
  const timeoutMs = options.timeoutMs !== undefined && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
459
495
  const redirects = options.redirects !== undefined && Number.isInteger(options.redirects) ? options.redirects : DEFAULT_REDIRECTS;
496
+ const maxBytes = options.maxBytes !== undefined && Number.isFinite(options.maxBytes) && options.maxBytes > 0
497
+ ? Math.floor(options.maxBytes)
498
+ : DEFAULT_MAX_BYTES;
460
499
  let current = new URL(url);
461
500
  let method = options.method === 'POST' ? 'POST' : 'GET';
462
501
  let payload = method === 'POST' && typeof options.body === 'string' ? options.body : undefined;
463
502
  for (let hop = 0; hop <= redirects; hop += 1) {
464
503
  const secure = current.protocol === 'https:';
465
504
  const port = Number(current.port || (secure ? 443 : 80));
466
- const raw = await openTunnel(proxy, current.hostname, port, timeoutMs, options.signal);
505
+ const targetHost = current.hostname.replace(/^\[|\]$/g, '');
506
+ const raw = await openTunnel(proxy, targetHost, port, timeoutMs, options.signal);
467
507
  let socket = raw;
468
508
  if (secure) {
469
- const upgraded = tls.connect({ socket: raw, servername: current.hostname });
509
+ const servername = net.isIP(targetHost) === 0 ? targetHost : undefined;
510
+ // `host` is the identity Node checks the certificate against. Omitting it
511
+ // makes the check fall back to the proxy host for IP targets.
512
+ const upgraded = tls.connect({ socket: raw, host: targetHost, ...(servername === undefined ? {} : { servername }) });
470
513
  // `once` also rejects on 'error', so a refused TLS handshake surfaces
471
514
  // instead of becoming an unhandled socket event.
472
515
  await once(upgraded, 'secureConnect');
@@ -485,7 +528,7 @@ export async function proxyGet(url, options) {
485
528
  '\r\n' +
486
529
  (payload ?? '');
487
530
  // Follow one redirect hop at a time, re-opening the tunnel per hop.
488
- const response = readResponse(await exchange(socket, head, timeoutMs, options.signal));
531
+ const response = readResponse(await exchange(socket, head, timeoutMs, options.signal, maxBytes), maxBytes);
489
532
  const location = response?.headers.location;
490
533
  if (response !== undefined && response.status >= 300 && response.status < 400 && location !== undefined && hop < redirects) {
491
534
  // 307/308 preserve the method and body; the older statuses downgrade to GET.
package/lib/rank.js CHANGED
@@ -360,7 +360,7 @@ function fuseEntries(entries, limit, options) {
360
360
  weights.push(engineWeight(config.weights, entry.engineIndex, entry.host));
361
361
  }
362
362
  const rounds = lists.reduce((deepest, rows, index) => {
363
- const weight = weights[index] ?? 1;
363
+ const weight = slotWeight(weights, index);
364
364
  if (weight === 0 || rows.length === 0)
365
365
  return deepest;
366
366
  return Math.max(deepest, Math.ceil(rows.length / weight));
@@ -377,7 +377,7 @@ function fuseEntries(entries, limit, options) {
377
377
  const rows = lists[index];
378
378
  if (rows === undefined)
379
379
  continue;
380
- const weight = weights[index] ?? 1;
380
+ const weight = slotWeight(weights, index);
381
381
  for (let offset = 0; offset < weight; offset += 1) {
382
382
  const row = rows[round * weight + offset];
383
383
  if (row === undefined)
package/lib/read-page.js CHANGED
@@ -401,20 +401,39 @@ export function createReadPageDefinition(name, readSettings, options = {}) {
401
401
  if (parsed.offset > totalLength) {
402
402
  throw new Error(`read_page: offset ${parsed.offset} is past the end of ${parsed.url} (totalLength ${totalLength}); use offset 0 to start over or ${totalLength} to read the tail`);
403
403
  }
404
- // Keep surrogate pairs intact at the cut point. `nextOffset` follows the
405
- // adjusted end, so the concatenated chunks still equal the full body.
406
- let end = Math.min(totalLength, parsed.offset + parsed.maxLength);
407
- if (end > parsed.offset && end < totalLength) {
408
- const last = page.content.charCodeAt(end - 1);
409
- if (last >= 0xd800 && last <= 0xdbff)
410
- end -= 1;
404
+ // Walk the real page boundaries. A surrogate pair at a cut point moves
405
+ // that boundary one code unit back, so the chunk count and index cannot
406
+ // be derived from `ceil(totalLength / maxLength)` without drifting.
407
+ const pageEnd = (start) => {
408
+ let end = Math.min(totalLength, start + parsed.maxLength);
409
+ if (end > start && end < totalLength) {
410
+ const last = page.content.charCodeAt(end - 1);
411
+ if (last >= 0xd800 && last <= 0xdbff)
412
+ end -= 1;
413
+ }
414
+ // A one-character budget on a surrogate pair must still advance.
415
+ return end > start ? end : Math.min(totalLength, start + 1);
416
+ };
417
+ let count = 0;
418
+ let index = 0;
419
+ let cursor = 0;
420
+ while (cursor < totalLength) {
421
+ const boundary = pageEnd(cursor);
422
+ count += 1;
423
+ if (parsed.offset >= cursor && parsed.offset < boundary)
424
+ index = count;
425
+ cursor = boundary;
426
+ }
427
+ if (totalLength === 0) {
428
+ count = 1;
429
+ index = 1;
430
+ }
431
+ else if (parsed.offset === totalLength) {
432
+ index = count;
411
433
  }
434
+ const end = pageEnd(parsed.offset);
412
435
  const content = page.content.slice(parsed.offset, end);
413
436
  const nextOffset = end < totalLength ? end : null;
414
- const count = totalLength === 0 ? 1 : Math.ceil(totalLength / parsed.maxLength);
415
- const index = totalLength === 0
416
- ? 1
417
- : Math.min(count, Math.floor(parsed.offset / parsed.maxLength) + 1);
418
437
  const value = {
419
438
  url: parsed.url,
420
439
  content,
package/lib/research.d.ts CHANGED
@@ -87,6 +87,11 @@ export interface ResearchValue {
87
87
  }>;
88
88
  /** Fetch failures, one entry per URL. */
89
89
  failures: ResearchFetchFailure[];
90
+ /** Search errors, one entry per failed query; omitted when none failed. */
91
+ searchFailures?: Array<{
92
+ query: string;
93
+ reason: string;
94
+ }>;
90
95
  /** Number of search rows seen before the fetch budget cut them. */
91
96
  found: number;
92
97
  /** True when the search row list was cut by the fetch budget. */