@yeaft/webchat-agent 1.0.638 → 1.0.640

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,6 +29,7 @@ import { stat, readFile } from 'fs/promises';
29
29
  import { existsSync } from 'fs';
30
30
  import { resolve, extname, isAbsolute } from 'path';
31
31
  import { checkPathAllowed } from './path-safety.js';
32
+ import { readPersistedAttachmentFile, resolvePersistedAttachmentPath } from '../attachments.js';
32
33
 
33
34
  /** Default max image size in bytes (20 MiB). Override via ctx.maxImageBytes. */
34
35
  const DEFAULT_MAX_IMAGE_BYTES = 20 * 1024 * 1024;
@@ -149,7 +150,7 @@ Path examples:
149
150
 
150
151
  Supported formats: PNG, JPEG (.jpg/.jpeg/.jfif), GIF, WebP.
151
152
  Max size: 20 MiB by default (configurable via ctx.maxImageBytes).
152
- Path must live under the project directory or an explicit host allowlist.`,
153
+ Path must live under the project directory, the current Session's attachments, or an explicit host allowlist. Attachments from other Sessions are not allowed.`,
153
154
  zh: `加载本地图片文件并附加到对话中,使 LLM 可以查看。
154
155
 
155
156
  返回 base64 data URI(image 字段)及元数据(格式、尺寸、大小)。引擎将图片作为原生多模态工具结果交给模型。
@@ -170,7 +171,7 @@ Path must live under the project directory or an explicit host allowlist.`,
170
171
 
171
172
  支持格式:PNG、JPEG(.jpg/.jpeg/.jfif)、GIF、WebP。
172
173
  最大文件:默认 20 MiB(可通过 ctx.maxImageBytes 配置)。
173
- 路径必须在项目目录下或已加入宿主白名单。`
174
+ 路径必须在项目目录、当前 Session 的附件目录下或已加入宿主白名单;不允许读取其他 Session 的附件。`
174
175
  },
175
176
  parameters: {
176
177
  type: 'object',
@@ -216,8 +217,14 @@ Path must live under the project directory or an explicit host allowlist.`,
216
217
  : DEFAULT_MAX_IMAGE_BYTES;
217
218
  const absPath = resolve(cwd, file_path);
218
219
 
220
+ // Session uploads intentionally live outside workDir. Reuse the same
221
+ // owner + no-symlink resolver as history/preview, never allow the whole
222
+ // instance's sessions directory just to make an uploaded image readable.
223
+ const attachmentOptions = { yeaftDir: ctx?.yeaftDir, sessionId: ctx?.sessionId, cwd };
224
+ const sessionAttachment = Boolean(ctx?.yeaftDir && ctx?.sessionId
225
+ && resolvePersistedAttachmentPath(file_path, attachmentOptions));
219
226
  const pathErr = checkPathAllowed(absPath, cwd, allowlist);
220
- if (pathErr) {
227
+ if (pathErr && !sessionAttachment) {
221
228
  // prev-3 P2: split "absolute outside project" from "relative ..".
222
229
  // The `..` case is already handled above, so anything reaching here
223
230
  // is either an absolute path outside cwd/allowlist or a relative
@@ -229,6 +236,9 @@ Path must live under the project directory or an explicit host allowlist.`,
229
236
  : 'Resolved path is outside the project directory. ';
230
237
  return JSON.stringify({
231
238
  error: hint + pathErr.message,
239
+ code: 'image_path_denied',
240
+ errorEffect: 'none',
241
+ hint: 'Use the current Session attachment path, a project image, or an explicitly allowlisted directory.',
232
242
  path: absPath,
233
243
  });
234
244
  }
@@ -254,37 +264,48 @@ Path must live under the project directory or an explicit host allowlist.`,
254
264
  });
255
265
  }
256
266
 
257
- if (!existsSync(absPath)) {
258
- return JSON.stringify({ error: `Image not found: ${absPath}` });
259
- }
267
+ let buffer;
268
+ let modified;
269
+ if (sessionAttachment) {
270
+ try {
271
+ // Recheck ownership and symlinks at the actual bounded file read.
272
+ ({ buffer, modified } = readPersistedAttachmentFile(file_path, attachmentOptions, maxBytes));
273
+ } catch (err) {
274
+ return JSON.stringify({ error: `Failed to read Session image: ${err.message}`, code: 'image_attachment_unavailable', errorEffect: 'none' });
275
+ }
276
+ } else {
277
+ if (!existsSync(absPath)) {
278
+ return JSON.stringify({ error: `Image not found: ${absPath}`, code: 'image_not_found', errorEffect: 'none' });
279
+ }
260
280
 
261
- let fileStat;
262
- try {
263
- fileStat = await stat(absPath);
264
- } catch (err) {
265
- return JSON.stringify({ error: `Failed to stat image: ${err.message}` });
266
- }
281
+ let fileStat;
282
+ try {
283
+ fileStat = await stat(absPath);
284
+ } catch (err) {
285
+ return JSON.stringify({ error: `Failed to stat image: ${err.message}` });
286
+ }
267
287
 
268
- if (!fileStat.isFile()) {
269
- return JSON.stringify({ error: 'file_path does not point to a regular file' });
270
- }
288
+ if (!fileStat.isFile()) {
289
+ return JSON.stringify({ error: 'file_path does not point to a regular file' });
290
+ }
271
291
 
272
- if (fileStat.size > maxBytes) {
273
- return JSON.stringify({
274
- error:
275
- `Image exceeds ${formatBytes(maxBytes)} (${formatBytes(fileStat.size)} actual). ` +
276
- `Reduce image size (resize/crop), or set \`maxImageBytes\` in ` +
277
- `~/.yeaft/config.json if your LLM supports more.`,
278
- size: fileStat.size,
279
- maxSize: maxBytes,
280
- });
281
- }
292
+ if (fileStat.size > maxBytes) {
293
+ return JSON.stringify({
294
+ error:
295
+ `Image exceeds ${formatBytes(maxBytes)} (${formatBytes(fileStat.size)} actual). ` +
296
+ `Reduce image size (resize/crop), or set \`maxImageBytes\` in ` +
297
+ `~/.yeaft/config.json if your LLM supports more.`,
298
+ size: fileStat.size,
299
+ maxSize: maxBytes,
300
+ });
301
+ }
282
302
 
283
- let buffer;
284
- try {
285
- buffer = await readFile(absPath);
286
- } catch (err) {
287
- return JSON.stringify({ error: `Failed to read image: ${err.message}` });
303
+ try {
304
+ buffer = await readFile(absPath);
305
+ } catch (err) {
306
+ return JSON.stringify({ error: `Failed to read image: ${err.message}` });
307
+ }
308
+ modified = fileStat.mtime.toISOString();
288
309
  }
289
310
 
290
311
  const mediaType = EXT_TO_MIME[ext];
@@ -296,9 +317,9 @@ Path must live under the project directory or an explicit host allowlist.`,
296
317
  path: absPath,
297
318
  format: ext.slice(1).toUpperCase() === 'JPG' ? 'JPEG' : ext.slice(1).toUpperCase(),
298
319
  media_type: mediaType,
299
- size: fileStat.size,
300
- sizeFormatted: formatBytes(fileStat.size),
301
- modified: fileStat.mtime.toISOString(),
320
+ size: buffer.length,
321
+ sizeFormatted: formatBytes(buffer.length),
322
+ modified,
302
323
  image: dataUri,
303
324
  };
304
325
  // Normalise .jfif to JPEG in display format too, for consistency.
@@ -6,6 +6,10 @@
6
6
  */
7
7
 
8
8
  import { defineTool } from './types.js';
9
+ import {
10
+ attachResearchGuidance, canonicalWebUrl, recordWebResearchCall, safeWebUrl, setBounded,
11
+ shareWebRequest, webFetchKey, webHttpFailure, webRequestFailure, withWebDeadline,
12
+ } from '../web-research.js';
9
13
 
10
14
  /** Strip HTML tags and normalize whitespace for readability. */
11
15
  function htmlToText(html) {
@@ -98,69 +102,73 @@ Guidelines:
98
102
  },
99
103
  isConcurrencySafe: () => true,
100
104
  isReadOnly: () => true,
101
- // Live waiting is explicit; raw formatting alone never exempts static reads.
105
+ // Success-only reuse and active sharing belong to the query ledger, not Engine's cache.
102
106
  duplicateCallPolicy: (input) => input?.polling === true ? 'allow' : 'warn',
103
107
  async execute(input, ctx) {
104
- const { url, max_length = 50000, raw = false } = input;
105
- if (!url) return JSON.stringify({ error: 'url is required' });
106
-
108
+ const { url, max_length = 50000, raw = false, polling = false } = input;
109
+ const research = ctx?.webResearch;
110
+ const signal = ctx?.signal;
111
+ const visibleUrl = safeWebUrl(url);
112
+ let canonicalUrl;
107
113
  try {
108
- // Validate URL
109
- let parsedUrl;
110
- try {
111
- parsedUrl = new URL(url);
112
- } catch {
113
- return JSON.stringify({ error: `Invalid URL: ${url}` });
114
- }
115
-
116
- // Only allow http(s)
117
- if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
118
- return JSON.stringify({ error: `Unsupported protocol: ${parsedUrl.protocol}` });
119
- }
114
+ canonicalUrl = canonicalWebUrl(url);
115
+ if (!['http:', 'https:'].includes(new URL(canonicalUrl).protocol)) throw new Error('Only HTTP(S) URLs are supported');
116
+ if (!Number.isInteger(max_length) || max_length < 1) throw new Error('max_length must be a positive integer');
117
+ } catch {
118
+ return JSON.stringify({ error: 'Invalid fetch input: a full HTTP(S) URL and positive integer max_length are required', code: 'invalid_arguments', errorEffect: 'none', hint: 'Provide a full HTTP(S) URL and a positive integer max_length.' });
119
+ }
120
120
 
121
- const response = await fetch(url, {
122
- signal: ctx?.signal,
123
- headers: {
124
- 'User-Agent': 'Yeaft/1.0 (compatible; bot)',
125
- 'Accept': 'text/html, application/json, text/plain, */*',
126
- },
127
- redirect: 'follow',
121
+ try {
122
+ signal?.throwIfAborted();
123
+ const key = webFetchKey(canonicalUrl, { raw, maxLength: max_length });
124
+ const prior = !polling && research?.fetches.get(key);
125
+ if (prior) return attachResearchGuidance(prior, research, {
126
+ reused: true, reuseReason: 'This URL and response mode already succeeded in this query. Use polling:true only for requested live status checks.',
128
127
  });
129
-
130
- if (!response.ok) {
131
- return JSON.stringify({
132
- error: `HTTP ${response.status}: ${response.statusText}`,
133
- url,
134
- });
135
- }
136
-
137
- const contentType = response.headers.get('content-type') || '';
138
- const body = await response.text();
139
-
140
- let content;
141
- if (raw || contentType.includes('json') || contentType.includes('text/plain')) {
142
- content = body;
143
- } else {
144
- content = htmlToText(body);
145
- }
146
-
147
- // Truncate if too long
148
- const truncated = content.length > max_length;
149
- if (truncated) {
150
- content = content.slice(0, max_length);
151
- }
152
-
153
- return JSON.stringify({
154
- url: response.url, // final URL after redirects
155
- status: response.status,
156
- contentType,
157
- contentLength: content.length,
158
- truncated,
159
- content,
128
+ const output = await shareWebRequest(research, JSON.stringify(['fetch', key, polling]), signal, async () => {
129
+ let result;
130
+ try {
131
+ result = await withWebDeadline(signal, async bounded => {
132
+ const response = await fetch(canonicalUrl, {
133
+ signal: bounded,
134
+ headers: { 'User-Agent': 'Yeaft/1.0 (compatible; bot)', Accept: 'text/html, application/json, text/plain, */*' },
135
+ redirect: 'follow',
136
+ });
137
+ const finalUrl = safeWebUrl(response.url || canonicalUrl) || visibleUrl;
138
+ if (!response.ok) return { ...webHttpFailure(response.status), url: finalUrl };
139
+ const contentType = response.headers.get('content-type') || '';
140
+ const body = await response.text();
141
+ const content = raw || contentType.includes('json') || contentType.includes('text/plain') ? body : htmlToText(body);
142
+ return {
143
+ url: finalUrl, status: response.status, contentType,
144
+ contentLength: Math.min(content.length, max_length), truncated: content.length > max_length,
145
+ content: content.slice(0, max_length),
146
+ };
147
+ });
148
+ signal?.throwIfAborted();
149
+ } catch (error) {
150
+ result = { ...webRequestFailure(error, signal), url: visibleUrl };
151
+ }
152
+ recordWebResearchCall(research);
153
+ const previousAuth = research?.authorizationFailures.get(canonicalUrl);
154
+ if (previousAuth && result.error) {
155
+ result.priorAuthorizationFailure = previousAuth;
156
+ result.hint += ' This same URL previously denied access in this query; check public access before retrying.';
157
+ }
158
+ const output = JSON.stringify(result);
159
+ if (research) {
160
+ if (!result.error) {
161
+ setBounded(research.fetches, key, output);
162
+ research.authorizationFailures.delete(canonicalUrl);
163
+ } else if ([401, 403].includes(result.status)) {
164
+ setBounded(research.authorizationFailures, canonicalUrl, { status: result.status, url: result.url });
165
+ }
166
+ }
167
+ return output;
160
168
  });
161
- } catch (err) {
162
- if (err.name === 'AbortError') return JSON.stringify({ error: 'Fetch cancelled' });
163
- return JSON.stringify({ error: `Fetch failed: ${err.message}`, url });
169
+ return attachResearchGuidance(output, research);
170
+ } catch (error) {
171
+ return attachResearchGuidance({ ...webRequestFailure(error, signal), url: visibleUrl }, research);
164
172
  }
165
173
  },
166
174
  });
@@ -22,6 +22,11 @@
22
22
  */
23
23
 
24
24
  import { defineTool } from './types.js';
25
+ import {
26
+ attachResearchGuidance, findPriorSearch, findRelatedSearch, normalizeSearchQuery,
27
+ recordSearch, recordWebResearchCall, shareWebRequest, webHttpFailure,
28
+ webRequestFailure, withWebDeadline,
29
+ } from '../web-research.js';
25
30
 
26
31
  export default defineTool({
27
32
  name: 'WebSearch',
@@ -66,147 +71,134 @@ Guidelines:
66
71
  },
67
72
  isConcurrencySafe: () => true,
68
73
  isReadOnly: () => true,
74
+ // The ledger caches successes only; Engine's general cache would freeze errors.
69
75
  async execute(input, ctx) {
70
- const { query, limit = 5 } = input;
71
- if (!query || typeof query !== 'string') {
72
- return JSON.stringify({ error: 'query is required' });
76
+ const query = normalizeSearchQuery(input.query);
77
+ const limit = input.limit ?? 5;
78
+ if (typeof input.query !== 'string' || !query || !Number.isInteger(limit) || limit < 1) {
79
+ return JSON.stringify({ error: 'query must be non-empty text and limit a positive integer', code: 'invalid_arguments', errorEffect: 'none', hint: 'Supply a targeted query and a positive integer result limit.' });
73
80
  }
74
81
 
75
- const search = ctx?.config?.search || {};
82
+ const research = ctx?.webResearch;
76
83
  const signal = ctx?.signal;
77
- const errors = [];
78
-
79
- // Backend preference set in YeaftSettings → Search tab. When the
80
- // user picks `playwright` we'd normally call the playwright-service;
81
- // that service is not yet shipped (next PR), so the preference is
82
- // recorded for forward-compat and we transparently fall through to
83
- // Tavily / HTML scrape. Once the service lands we'll insert a
84
- // tryPlaywright backend here ahead of Tavily.
85
- // Anything other than 'playwright' is treated as 'tavily'.
86
-
87
- // 1. Tavily — default, fast, structured.
88
- if (search.tavilyApiKey) {
89
- const r = await tryTavily(query, limit, search.tavilyApiKey, signal);
90
- if (r.ok) return JSON.stringify(r.data, null, 2);
91
- errors.push(`tavily: ${r.error}`);
92
- }
93
-
94
- // 2. Generic JSON endpoint (legacy escape hatch — SearXNG, custom proxy, etc).
95
- const genericUrl = search.searchApiUrl || ctx?.config?.searchApiUrl;
96
- if (genericUrl) {
97
- const r = await tryGenericApi(query, limit, genericUrl, signal);
98
- if (r.ok) return JSON.stringify(r.data, null, 2);
99
- errors.push(`searchApiUrl: ${r.error}`);
84
+ try {
85
+ signal?.throwIfAborted();
86
+ const prior = findPriorSearch(research, query, limit);
87
+ if (prior) {
88
+ const data = JSON.parse(prior.output);
89
+ return attachResearchGuidance({ ...data, results: data.results.slice(0, limit) }, research, {
90
+ reused: true, priorQuery: prior.query,
91
+ reuseReason: 'This exact trimmed query already succeeded with at least the requested limit in this query.',
92
+ });
93
+ }
94
+ const related = findRelatedSearch(research, query);
95
+ const output = await shareWebRequest(research, JSON.stringify(['search', query, limit]), signal, async () => {
96
+ const result = await searchWeb(query, limit, ctx);
97
+ recordWebResearchCall(research);
98
+ const output = JSON.stringify(result);
99
+ if (!result.error) recordSearch(research, query, limit, output);
100
+ return output;
101
+ });
102
+ return attachResearchGuidance(output, research, related ? {
103
+ relatedPriorQuery: related.query,
104
+ convergenceHint: 'A similar query already completed. Use this result for the specific unresolved fact; semantic variants are not interchangeable.',
105
+ } : {});
106
+ } catch (error) {
107
+ return attachResearchGuidance(webRequestFailure(error, signal), research);
100
108
  }
101
-
102
- // 3. HTML-scrape fallback. Often blocked on cloud IPs; useful for
103
- // self-hosted / residential setups with no API key.
104
- if (!search.disableHtmlFallback) {
105
- const r = await tryHtmlScrape(query, limit, signal);
106
- if (r.ok) return JSON.stringify(r.data, null, 2);
107
- errors.push(`html: ${r.error}`);
108
- }
109
-
110
- return JSON.stringify({
111
- error: 'No web search backend succeeded.',
112
- attempted: errors,
113
- hint: 'Set search.tavilyApiKey in ~/.yeaft/config.json (free tier: https://tavily.com).',
114
- });
115
109
  },
116
110
  });
117
111
 
112
+ async function searchWeb(query, limit, ctx) {
113
+ const search = ctx?.config?.search || {};
114
+ const signal = ctx?.signal;
115
+ const failures = [];
116
+ const backends = [];
117
+ if (search.tavilyApiKey) backends.push(['tavily', bounded => tryTavily(query, limit, search.tavilyApiKey, bounded)]);
118
+ const genericUrl = search.searchApiUrl || ctx?.config?.searchApiUrl;
119
+ if (genericUrl) backends.push(['generic', bounded => tryGenericApi(query, limit, genericUrl, bounded)]);
120
+ if (!search.disableHtmlFallback) {
121
+ backends.push(['duckduckgo-html', bounded => tryHtmlScrape(query, limit, 'duckduckgo-html', bounded)]);
122
+ backends.push(['bing-html', bounded => tryHtmlScrape(query, limit, 'bing-html', bounded)]);
123
+ }
124
+ for (const [provider, work] of backends) {
125
+ signal?.throwIfAborted();
126
+ try {
127
+ const result = await withWebDeadline(signal, work);
128
+ signal?.throwIfAborted();
129
+ if (!result.error) return result;
130
+ failures.push({ provider, ...result });
131
+ } catch (error) {
132
+ const failure = webRequestFailure(error, signal);
133
+ // Cancellation is terminal: never start the next backend.
134
+ if (failure.code === 'cancelled') return { ...failure, query, failures };
135
+ failures.push({ provider, ...failure });
136
+ }
137
+ }
138
+ const last = failures.at(-1);
139
+ return {
140
+ error: 'No web search backend succeeded.', code: 'search_unavailable', errorEffect: 'none', query,
141
+ ...(last?.status ? { status: last.status } : {}),
142
+ attempted: failures.map(failure => `${failure.provider}: ${failure.error}`), failures,
143
+ hint: 'Inspect backend failures: HTML search may be bot-blocked or its markup may have changed. Use a known official URL with WebFetch or another available source; verify an already configured search endpoint if applicable. Rephrasing alone will not fix access failures.',
144
+ };
145
+ }
146
+
118
147
  // ─── Backend implementations ────────────────────────────────────────
119
148
 
120
149
  async function tryTavily(query, limit, apiKey, signal) {
121
- try {
122
- const res = await fetch('https://api.tavily.com/search', {
123
- method: 'POST',
124
- signal,
125
- headers: { 'Content-Type': 'application/json' },
126
- body: JSON.stringify({
127
- api_key: apiKey,
128
- query,
129
- max_results: Math.max(1, Math.min(limit, 10)),
130
- include_answer: true,
131
- search_depth: 'basic',
132
- }),
133
- });
134
- if (!res.ok) {
135
- const text = await res.text().catch(() => '');
136
- return { ok: false, error: `${res.status} ${res.statusText} ${text.slice(0, 200)}` };
137
- }
138
- const data = await res.json();
139
- return {
140
- ok: true,
141
- data: {
142
- provider: 'tavily',
143
- query,
144
- answer: data.answer || null,
145
- results: (data.results || []).slice(0, limit).map((r) => ({
146
- title: r.title,
147
- url: r.url,
148
- snippet: r.content,
149
- score: r.score,
150
- })),
151
- },
152
- };
153
- } catch (err) {
154
- if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
155
- return { ok: false, error: err.message || String(err) };
156
- }
150
+ const res = await fetch('https://api.tavily.com/search', {
151
+ method: 'POST', signal,
152
+ headers: { 'Content-Type': 'application/json' },
153
+ body: JSON.stringify({ api_key: apiKey, query, max_results: Math.min(limit, 10), include_answer: true, search_depth: 'basic' }),
154
+ });
155
+ if (!res.ok) return webHttpFailure(res.status);
156
+ const data = await res.json();
157
+ if (!Array.isArray(data?.results) || data.error) return invalidSearchResponse();
158
+ return {
159
+ provider: 'tavily', query, answer: data.answer || null,
160
+ results: data.results.slice(0, limit).map(r => ({ title: r.title, url: r.url, snippet: r.content, score: r.score })),
161
+ };
157
162
  }
158
163
 
159
164
  async function tryGenericApi(query, limit, urlStr, signal) {
160
- try {
161
- const url = new URL(urlStr);
162
- url.searchParams.set('q', query);
163
- url.searchParams.set('limit', String(limit));
164
- const res = await fetch(url.toString(), {
165
- signal,
166
- headers: { 'User-Agent': 'Yeaft/1.0', Accept: 'application/json' },
167
- });
168
- if (!res.ok) return { ok: false, error: `${res.status} ${res.statusText}` };
169
- const data = await res.json();
170
- return { ok: true, data: { provider: 'generic', query, ...data } };
171
- } catch (err) {
172
- if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
173
- return { ok: false, error: err.message || String(err) };
174
- }
165
+ const url = new URL(urlStr);
166
+ url.searchParams.set('q', query);
167
+ url.searchParams.set('limit', String(limit));
168
+ const res = await fetch(url.toString(), { signal, headers: { 'User-Agent': 'Yeaft/1.0', Accept: 'application/json' } });
169
+ if (!res.ok) return webHttpFailure(res.status);
170
+ const data = await res.json();
171
+ if (!Array.isArray(data?.results) || data.error) return invalidSearchResponse();
172
+ return { ...data, provider: 'generic', query, results: data.results.slice(0, limit) };
175
173
  }
176
174
 
177
- /**
178
- * HTML-scrape fallback. Tries DuckDuckGo's lite HTML endpoint first
179
- * (smaller markup, but more aggressive bot detection on cloud IPs),
180
- * then Bing. We intentionally keep the regex-based parsers minimal —
181
- * they break less than full DOM selectors when sites tweak markup.
182
- */
183
- async function tryHtmlScrape(query, limit, signal) {
184
- const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
185
- const headers = { 'User-Agent': ua, 'Accept-Language': 'en-US,en;q=0.9' };
186
-
187
- try {
188
- const ddg = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { signal, headers });
189
- if (ddg.ok) {
190
- const html = await ddg.text();
191
- const results = parseDdgHtml(html, limit);
192
- if (results.length) return { ok: true, data: { provider: 'duckduckgo-html', query, results } };
193
- }
194
- } catch (err) {
195
- if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
196
- }
197
-
198
- try {
199
- const bing = await fetch(`https://www.bing.com/search?q=${encodeURIComponent(query)}`, { signal, headers });
200
- if (bing.ok) {
201
- const html = await bing.text();
202
- const results = parseBingHtml(html, limit);
203
- if (results.length) return { ok: true, data: { provider: 'bing-html', query, results } };
204
- }
205
- } catch (err) {
206
- if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
207
- }
175
+ function invalidSearchResponse() {
176
+ return { error: 'Search endpoint did not return a results array', code: 'invalid_response', errorEffect: 'none', hint: 'Verify the configured endpoint returns a JSON object with results: [], including for no matches.' };
177
+ }
208
178
 
209
- return { ok: false, error: 'all HTML scrape backends returned 0 results (likely bot-blocked)' };
179
+ /** Zero matches need an explicit no-results marker, not an empty parser output. */
180
+ async function tryHtmlScrape(query, limit, provider, signal) {
181
+ const ddg = provider === 'duckduckgo-html';
182
+ const endpoint = ddg ? 'https://html.duckduckgo.com/html/?q=' : 'https://www.bing.com/search?q=';
183
+ const res = await fetch(endpoint + encodeURIComponent(query), {
184
+ signal,
185
+ headers: {
186
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
187
+ 'Accept-Language': 'en-US,en;q=0.9',
188
+ },
189
+ });
190
+ if (!res.ok) return webHttpFailure(res.status);
191
+ const html = await res.text();
192
+ const blocked = /anomaly\.js|anomaly-modal|challenge-form|verify (?:that )?you(?:'re| are) (?:a )?human|unusual traffic|<title[^>]*>[^<]*(?:captcha|access denied)/i.test(html);
193
+ const results = ddg ? parseDdgHtml(html, limit) : parseBingHtml(html, limit);
194
+ const noMatches = ddg ? /class=["'][^"']*\bno-results\b|No results found for/i.test(html)
195
+ : /class=["'][^"']*\bb_no\b|There are no results for|No results found for/i.test(html);
196
+ if (!blocked && (results.length || noMatches)) return { provider, query, results };
197
+ return {
198
+ error: blocked ? 'HTML search returned a bot challenge' : 'HTML search markup was not recognized; zero matches are not confirmed',
199
+ code: blocked ? 'bot_blocked' : 'unrecognized_response', status: res.status, errorEffect: 'none',
200
+ hint: 'Use a known official URL or another available source. Retrying a rephrased query will not fix a bot challenge or parser mismatch.',
201
+ };
210
202
  }
211
203
 
212
204
  /**