@yeaft/webchat-agent 1.0.639 → 1.0.641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local-runtime/version.json +1 -1
- package/local-runtime/web/app.bundle.js +102 -102
- package/local-runtime/web/app.bundle.js.gz +0 -0
- package/local-runtime/web/index.html +2 -2
- package/local-runtime/web/style.bundle.css +1 -1
- package/local-runtime/web/style.bundle.css.gz +0 -0
- package/package.json +1 -1
- package/yeaft/attachments.js +22 -9
- package/yeaft/engine.js +95 -574
- package/yeaft/history-window.js +132 -5
- package/yeaft/query-progress.js +15 -2
- package/yeaft/response-presentation.js +27 -5
- package/yeaft/sessions/project-doc.js +29 -6
- package/yeaft/stats/tool-usage.js +90 -22
- package/yeaft/tools/git-read.js +7 -5
- package/yeaft/tools/view-image.js +54 -33
- package/yeaft/tools/web-fetch.js +65 -57
- package/yeaft/tools/web-search.js +116 -124
- package/yeaft/web-research.js +169 -0
|
@@ -29,6 +29,7 @@ import { stat, readFile } from 'fs/promises';
|
|
|
29
29
|
import { existsSync } from 'fs';
|
|
30
30
|
import { resolve, extname, isAbsolute } from 'path';
|
|
31
31
|
import { checkPathAllowed } from './path-safety.js';
|
|
32
|
+
import { readPersistedAttachmentFile, resolvePersistedAttachmentPath } from '../attachments.js';
|
|
32
33
|
|
|
33
34
|
/** Default max image size in bytes (20 MiB). Override via ctx.maxImageBytes. */
|
|
34
35
|
const DEFAULT_MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
@@ -149,7 +150,7 @@ Path examples:
|
|
|
149
150
|
|
|
150
151
|
Supported formats: PNG, JPEG (.jpg/.jpeg/.jfif), GIF, WebP.
|
|
151
152
|
Max size: 20 MiB by default (configurable via ctx.maxImageBytes).
|
|
152
|
-
Path must live under the project directory or an explicit host allowlist.`,
|
|
153
|
+
Path must live under the project directory, the current Session's attachments, or an explicit host allowlist. Attachments from other Sessions are not allowed.`,
|
|
153
154
|
zh: `加载本地图片文件并附加到对话中,使 LLM 可以查看。
|
|
154
155
|
|
|
155
156
|
返回 base64 data URI(image 字段)及元数据(格式、尺寸、大小)。引擎将图片作为原生多模态工具结果交给模型。
|
|
@@ -170,7 +171,7 @@ Path must live under the project directory or an explicit host allowlist.`,
|
|
|
170
171
|
|
|
171
172
|
支持格式:PNG、JPEG(.jpg/.jpeg/.jfif)、GIF、WebP。
|
|
172
173
|
最大文件:默认 20 MiB(可通过 ctx.maxImageBytes 配置)。
|
|
173
|
-
|
|
174
|
+
路径必须在项目目录、当前 Session 的附件目录下或已加入宿主白名单;不允许读取其他 Session 的附件。`
|
|
174
175
|
},
|
|
175
176
|
parameters: {
|
|
176
177
|
type: 'object',
|
|
@@ -216,8 +217,14 @@ Path must live under the project directory or an explicit host allowlist.`,
|
|
|
216
217
|
: DEFAULT_MAX_IMAGE_BYTES;
|
|
217
218
|
const absPath = resolve(cwd, file_path);
|
|
218
219
|
|
|
220
|
+
// Session uploads intentionally live outside workDir. Reuse the same
|
|
221
|
+
// owner + no-symlink resolver as history/preview, never allow the whole
|
|
222
|
+
// instance's sessions directory just to make an uploaded image readable.
|
|
223
|
+
const attachmentOptions = { yeaftDir: ctx?.yeaftDir, sessionId: ctx?.sessionId, cwd };
|
|
224
|
+
const sessionAttachment = Boolean(ctx?.yeaftDir && ctx?.sessionId
|
|
225
|
+
&& resolvePersistedAttachmentPath(file_path, attachmentOptions));
|
|
219
226
|
const pathErr = checkPathAllowed(absPath, cwd, allowlist);
|
|
220
|
-
if (pathErr) {
|
|
227
|
+
if (pathErr && !sessionAttachment) {
|
|
221
228
|
// prev-3 P2: split "absolute outside project" from "relative ..".
|
|
222
229
|
// The `..` case is already handled above, so anything reaching here
|
|
223
230
|
// is either an absolute path outside cwd/allowlist or a relative
|
|
@@ -229,6 +236,9 @@ Path must live under the project directory or an explicit host allowlist.`,
|
|
|
229
236
|
: 'Resolved path is outside the project directory. ';
|
|
230
237
|
return JSON.stringify({
|
|
231
238
|
error: hint + pathErr.message,
|
|
239
|
+
code: 'image_path_denied',
|
|
240
|
+
errorEffect: 'none',
|
|
241
|
+
hint: 'Use the current Session attachment path, a project image, or an explicitly allowlisted directory.',
|
|
232
242
|
path: absPath,
|
|
233
243
|
});
|
|
234
244
|
}
|
|
@@ -254,37 +264,48 @@ Path must live under the project directory or an explicit host allowlist.`,
|
|
|
254
264
|
});
|
|
255
265
|
}
|
|
256
266
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
267
|
+
let buffer;
|
|
268
|
+
let modified;
|
|
269
|
+
if (sessionAttachment) {
|
|
270
|
+
try {
|
|
271
|
+
// Recheck ownership and symlinks at the actual bounded file read.
|
|
272
|
+
({ buffer, modified } = readPersistedAttachmentFile(file_path, attachmentOptions, maxBytes));
|
|
273
|
+
} catch (err) {
|
|
274
|
+
return JSON.stringify({ error: `Failed to read Session image: ${err.message}`, code: 'image_attachment_unavailable', errorEffect: 'none' });
|
|
275
|
+
}
|
|
276
|
+
} else {
|
|
277
|
+
if (!existsSync(absPath)) {
|
|
278
|
+
return JSON.stringify({ error: `Image not found: ${absPath}`, code: 'image_not_found', errorEffect: 'none' });
|
|
279
|
+
}
|
|
260
280
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
281
|
+
let fileStat;
|
|
282
|
+
try {
|
|
283
|
+
fileStat = await stat(absPath);
|
|
284
|
+
} catch (err) {
|
|
285
|
+
return JSON.stringify({ error: `Failed to stat image: ${err.message}` });
|
|
286
|
+
}
|
|
267
287
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
288
|
+
if (!fileStat.isFile()) {
|
|
289
|
+
return JSON.stringify({ error: 'file_path does not point to a regular file' });
|
|
290
|
+
}
|
|
271
291
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
292
|
+
if (fileStat.size > maxBytes) {
|
|
293
|
+
return JSON.stringify({
|
|
294
|
+
error:
|
|
295
|
+
`Image exceeds ${formatBytes(maxBytes)} (${formatBytes(fileStat.size)} actual). ` +
|
|
296
|
+
`Reduce image size (resize/crop), or set \`maxImageBytes\` in ` +
|
|
297
|
+
`~/.yeaft/config.json if your LLM supports more.`,
|
|
298
|
+
size: fileStat.size,
|
|
299
|
+
maxSize: maxBytes,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
282
302
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
303
|
+
try {
|
|
304
|
+
buffer = await readFile(absPath);
|
|
305
|
+
} catch (err) {
|
|
306
|
+
return JSON.stringify({ error: `Failed to read image: ${err.message}` });
|
|
307
|
+
}
|
|
308
|
+
modified = fileStat.mtime.toISOString();
|
|
288
309
|
}
|
|
289
310
|
|
|
290
311
|
const mediaType = EXT_TO_MIME[ext];
|
|
@@ -296,9 +317,9 @@ Path must live under the project directory or an explicit host allowlist.`,
|
|
|
296
317
|
path: absPath,
|
|
297
318
|
format: ext.slice(1).toUpperCase() === 'JPG' ? 'JPEG' : ext.slice(1).toUpperCase(),
|
|
298
319
|
media_type: mediaType,
|
|
299
|
-
size:
|
|
300
|
-
sizeFormatted: formatBytes(
|
|
301
|
-
modified
|
|
320
|
+
size: buffer.length,
|
|
321
|
+
sizeFormatted: formatBytes(buffer.length),
|
|
322
|
+
modified,
|
|
302
323
|
image: dataUri,
|
|
303
324
|
};
|
|
304
325
|
// Normalise .jfif to JPEG in display format too, for consistency.
|
package/yeaft/tools/web-fetch.js
CHANGED
|
@@ -6,6 +6,10 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { defineTool } from './types.js';
|
|
9
|
+
import {
|
|
10
|
+
attachResearchGuidance, canonicalWebUrl, recordWebResearchCall, safeWebUrl, setBounded,
|
|
11
|
+
shareWebRequest, webFetchKey, webHttpFailure, webRequestFailure, withWebDeadline,
|
|
12
|
+
} from '../web-research.js';
|
|
9
13
|
|
|
10
14
|
/** Strip HTML tags and normalize whitespace for readability. */
|
|
11
15
|
function htmlToText(html) {
|
|
@@ -98,69 +102,73 @@ Guidelines:
|
|
|
98
102
|
},
|
|
99
103
|
isConcurrencySafe: () => true,
|
|
100
104
|
isReadOnly: () => true,
|
|
101
|
-
//
|
|
105
|
+
// Success-only reuse and active sharing belong to the query ledger, not Engine's cache.
|
|
102
106
|
duplicateCallPolicy: (input) => input?.polling === true ? 'allow' : 'warn',
|
|
103
107
|
async execute(input, ctx) {
|
|
104
|
-
const { url, max_length = 50000, raw = false } = input;
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
const { url, max_length = 50000, raw = false, polling = false } = input;
|
|
109
|
+
const research = ctx?.webResearch;
|
|
110
|
+
const signal = ctx?.signal;
|
|
111
|
+
const visibleUrl = safeWebUrl(url);
|
|
112
|
+
let canonicalUrl;
|
|
107
113
|
try {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
// Only allow http(s)
|
|
117
|
-
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
|
118
|
-
return JSON.stringify({ error: `Unsupported protocol: ${parsedUrl.protocol}` });
|
|
119
|
-
}
|
|
114
|
+
canonicalUrl = canonicalWebUrl(url);
|
|
115
|
+
if (!['http:', 'https:'].includes(new URL(canonicalUrl).protocol)) throw new Error('Only HTTP(S) URLs are supported');
|
|
116
|
+
if (!Number.isInteger(max_length) || max_length < 1) throw new Error('max_length must be a positive integer');
|
|
117
|
+
} catch {
|
|
118
|
+
return JSON.stringify({ error: 'Invalid fetch input: a full HTTP(S) URL and positive integer max_length are required', code: 'invalid_arguments', errorEffect: 'none', hint: 'Provide a full HTTP(S) URL and a positive integer max_length.' });
|
|
119
|
+
}
|
|
120
120
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
redirect: 'follow',
|
|
121
|
+
try {
|
|
122
|
+
signal?.throwIfAborted();
|
|
123
|
+
const key = webFetchKey(canonicalUrl, { raw, maxLength: max_length });
|
|
124
|
+
const prior = !polling && research?.fetches.get(key);
|
|
125
|
+
if (prior) return attachResearchGuidance(prior, research, {
|
|
126
|
+
reused: true, reuseReason: 'This URL and response mode already succeeded in this query. Use polling:true only for requested live status checks.',
|
|
128
127
|
});
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
128
|
+
const output = await shareWebRequest(research, JSON.stringify(['fetch', key, polling]), signal, async () => {
|
|
129
|
+
let result;
|
|
130
|
+
try {
|
|
131
|
+
result = await withWebDeadline(signal, async bounded => {
|
|
132
|
+
const response = await fetch(canonicalUrl, {
|
|
133
|
+
signal: bounded,
|
|
134
|
+
headers: { 'User-Agent': 'Yeaft/1.0 (compatible; bot)', Accept: 'text/html, application/json, text/plain, */*' },
|
|
135
|
+
redirect: 'follow',
|
|
136
|
+
});
|
|
137
|
+
const finalUrl = safeWebUrl(response.url || canonicalUrl) || visibleUrl;
|
|
138
|
+
if (!response.ok) return { ...webHttpFailure(response.status), url: finalUrl };
|
|
139
|
+
const contentType = response.headers.get('content-type') || '';
|
|
140
|
+
const body = await response.text();
|
|
141
|
+
const content = raw || contentType.includes('json') || contentType.includes('text/plain') ? body : htmlToText(body);
|
|
142
|
+
return {
|
|
143
|
+
url: finalUrl, status: response.status, contentType,
|
|
144
|
+
contentLength: Math.min(content.length, max_length), truncated: content.length > max_length,
|
|
145
|
+
content: content.slice(0, max_length),
|
|
146
|
+
};
|
|
147
|
+
});
|
|
148
|
+
signal?.throwIfAborted();
|
|
149
|
+
} catch (error) {
|
|
150
|
+
result = { ...webRequestFailure(error, signal), url: visibleUrl };
|
|
151
|
+
}
|
|
152
|
+
recordWebResearchCall(research);
|
|
153
|
+
const previousAuth = research?.authorizationFailures.get(canonicalUrl);
|
|
154
|
+
if (previousAuth && result.error) {
|
|
155
|
+
result.priorAuthorizationFailure = previousAuth;
|
|
156
|
+
result.hint += ' This same URL previously denied access in this query; check public access before retrying.';
|
|
157
|
+
}
|
|
158
|
+
const output = JSON.stringify(result);
|
|
159
|
+
if (research) {
|
|
160
|
+
if (!result.error) {
|
|
161
|
+
setBounded(research.fetches, key, output);
|
|
162
|
+
research.authorizationFailures.delete(canonicalUrl);
|
|
163
|
+
} else if ([401, 403].includes(result.status)) {
|
|
164
|
+
setBounded(research.authorizationFailures, canonicalUrl, { status: result.status, url: result.url });
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return output;
|
|
160
168
|
});
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
return
|
|
169
|
+
return attachResearchGuidance(output, research);
|
|
170
|
+
} catch (error) {
|
|
171
|
+
return attachResearchGuidance({ ...webRequestFailure(error, signal), url: visibleUrl }, research);
|
|
164
172
|
}
|
|
165
173
|
},
|
|
166
174
|
});
|
|
@@ -22,6 +22,11 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import { defineTool } from './types.js';
|
|
25
|
+
import {
|
|
26
|
+
attachResearchGuidance, findPriorSearch, findRelatedSearch, normalizeSearchQuery,
|
|
27
|
+
recordSearch, recordWebResearchCall, shareWebRequest, webHttpFailure,
|
|
28
|
+
webRequestFailure, withWebDeadline,
|
|
29
|
+
} from '../web-research.js';
|
|
25
30
|
|
|
26
31
|
export default defineTool({
|
|
27
32
|
name: 'WebSearch',
|
|
@@ -66,147 +71,134 @@ Guidelines:
|
|
|
66
71
|
},
|
|
67
72
|
isConcurrencySafe: () => true,
|
|
68
73
|
isReadOnly: () => true,
|
|
74
|
+
// The ledger caches successes only; Engine's general cache would freeze errors.
|
|
69
75
|
async execute(input, ctx) {
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
76
|
+
const query = normalizeSearchQuery(input.query);
|
|
77
|
+
const limit = input.limit ?? 5;
|
|
78
|
+
if (typeof input.query !== 'string' || !query || !Number.isInteger(limit) || limit < 1) {
|
|
79
|
+
return JSON.stringify({ error: 'query must be non-empty text and limit a positive integer', code: 'invalid_arguments', errorEffect: 'none', hint: 'Supply a targeted query and a positive integer result limit.' });
|
|
73
80
|
}
|
|
74
81
|
|
|
75
|
-
const
|
|
82
|
+
const research = ctx?.webResearch;
|
|
76
83
|
const signal = ctx?.signal;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
84
|
+
try {
|
|
85
|
+
signal?.throwIfAborted();
|
|
86
|
+
const prior = findPriorSearch(research, query, limit);
|
|
87
|
+
if (prior) {
|
|
88
|
+
const data = JSON.parse(prior.output);
|
|
89
|
+
return attachResearchGuidance({ ...data, results: data.results.slice(0, limit) }, research, {
|
|
90
|
+
reused: true, priorQuery: prior.query,
|
|
91
|
+
reuseReason: 'This exact trimmed query already succeeded with at least the requested limit in this query.',
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
const related = findRelatedSearch(research, query);
|
|
95
|
+
const output = await shareWebRequest(research, JSON.stringify(['search', query, limit]), signal, async () => {
|
|
96
|
+
const result = await searchWeb(query, limit, ctx);
|
|
97
|
+
recordWebResearchCall(research);
|
|
98
|
+
const output = JSON.stringify(result);
|
|
99
|
+
if (!result.error) recordSearch(research, query, limit, output);
|
|
100
|
+
return output;
|
|
101
|
+
});
|
|
102
|
+
return attachResearchGuidance(output, research, related ? {
|
|
103
|
+
relatedPriorQuery: related.query,
|
|
104
|
+
convergenceHint: 'A similar query already completed. Use this result for the specific unresolved fact; semantic variants are not interchangeable.',
|
|
105
|
+
} : {});
|
|
106
|
+
} catch (error) {
|
|
107
|
+
return attachResearchGuidance(webRequestFailure(error, signal), research);
|
|
100
108
|
}
|
|
101
|
-
|
|
102
|
-
// 3. HTML-scrape fallback. Often blocked on cloud IPs; useful for
|
|
103
|
-
// self-hosted / residential setups with no API key.
|
|
104
|
-
if (!search.disableHtmlFallback) {
|
|
105
|
-
const r = await tryHtmlScrape(query, limit, signal);
|
|
106
|
-
if (r.ok) return JSON.stringify(r.data, null, 2);
|
|
107
|
-
errors.push(`html: ${r.error}`);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
return JSON.stringify({
|
|
111
|
-
error: 'No web search backend succeeded.',
|
|
112
|
-
attempted: errors,
|
|
113
|
-
hint: 'Set search.tavilyApiKey in ~/.yeaft/config.json (free tier: https://tavily.com).',
|
|
114
|
-
});
|
|
115
109
|
},
|
|
116
110
|
});
|
|
117
111
|
|
|
112
|
+
async function searchWeb(query, limit, ctx) {
|
|
113
|
+
const search = ctx?.config?.search || {};
|
|
114
|
+
const signal = ctx?.signal;
|
|
115
|
+
const failures = [];
|
|
116
|
+
const backends = [];
|
|
117
|
+
if (search.tavilyApiKey) backends.push(['tavily', bounded => tryTavily(query, limit, search.tavilyApiKey, bounded)]);
|
|
118
|
+
const genericUrl = search.searchApiUrl || ctx?.config?.searchApiUrl;
|
|
119
|
+
if (genericUrl) backends.push(['generic', bounded => tryGenericApi(query, limit, genericUrl, bounded)]);
|
|
120
|
+
if (!search.disableHtmlFallback) {
|
|
121
|
+
backends.push(['duckduckgo-html', bounded => tryHtmlScrape(query, limit, 'duckduckgo-html', bounded)]);
|
|
122
|
+
backends.push(['bing-html', bounded => tryHtmlScrape(query, limit, 'bing-html', bounded)]);
|
|
123
|
+
}
|
|
124
|
+
for (const [provider, work] of backends) {
|
|
125
|
+
signal?.throwIfAborted();
|
|
126
|
+
try {
|
|
127
|
+
const result = await withWebDeadline(signal, work);
|
|
128
|
+
signal?.throwIfAborted();
|
|
129
|
+
if (!result.error) return result;
|
|
130
|
+
failures.push({ provider, ...result });
|
|
131
|
+
} catch (error) {
|
|
132
|
+
const failure = webRequestFailure(error, signal);
|
|
133
|
+
// Cancellation is terminal: never start the next backend.
|
|
134
|
+
if (failure.code === 'cancelled') return { ...failure, query, failures };
|
|
135
|
+
failures.push({ provider, ...failure });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const last = failures.at(-1);
|
|
139
|
+
return {
|
|
140
|
+
error: 'No web search backend succeeded.', code: 'search_unavailable', errorEffect: 'none', query,
|
|
141
|
+
...(last?.status ? { status: last.status } : {}),
|
|
142
|
+
attempted: failures.map(failure => `${failure.provider}: ${failure.error}`), failures,
|
|
143
|
+
hint: 'Inspect backend failures: HTML search may be bot-blocked or its markup may have changed. Use a known official URL with WebFetch or another available source; verify an already configured search endpoint if applicable. Rephrasing alone will not fix access failures.',
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
118
147
|
// ─── Backend implementations ────────────────────────────────────────
|
|
119
148
|
|
|
120
149
|
async function tryTavily(query, limit, apiKey, signal) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
});
|
|
134
|
-
if (!res.ok) {
|
|
135
|
-
const text = await res.text().catch(() => '');
|
|
136
|
-
return { ok: false, error: `${res.status} ${res.statusText} ${text.slice(0, 200)}` };
|
|
137
|
-
}
|
|
138
|
-
const data = await res.json();
|
|
139
|
-
return {
|
|
140
|
-
ok: true,
|
|
141
|
-
data: {
|
|
142
|
-
provider: 'tavily',
|
|
143
|
-
query,
|
|
144
|
-
answer: data.answer || null,
|
|
145
|
-
results: (data.results || []).slice(0, limit).map((r) => ({
|
|
146
|
-
title: r.title,
|
|
147
|
-
url: r.url,
|
|
148
|
-
snippet: r.content,
|
|
149
|
-
score: r.score,
|
|
150
|
-
})),
|
|
151
|
-
},
|
|
152
|
-
};
|
|
153
|
-
} catch (err) {
|
|
154
|
-
if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
|
|
155
|
-
return { ok: false, error: err.message || String(err) };
|
|
156
|
-
}
|
|
150
|
+
const res = await fetch('https://api.tavily.com/search', {
|
|
151
|
+
method: 'POST', signal,
|
|
152
|
+
headers: { 'Content-Type': 'application/json' },
|
|
153
|
+
body: JSON.stringify({ api_key: apiKey, query, max_results: Math.min(limit, 10), include_answer: true, search_depth: 'basic' }),
|
|
154
|
+
});
|
|
155
|
+
if (!res.ok) return webHttpFailure(res.status);
|
|
156
|
+
const data = await res.json();
|
|
157
|
+
if (!Array.isArray(data?.results) || data.error) return invalidSearchResponse();
|
|
158
|
+
return {
|
|
159
|
+
provider: 'tavily', query, answer: data.answer || null,
|
|
160
|
+
results: data.results.slice(0, limit).map(r => ({ title: r.title, url: r.url, snippet: r.content, score: r.score })),
|
|
161
|
+
};
|
|
157
162
|
}
|
|
158
163
|
|
|
159
164
|
async function tryGenericApi(query, limit, urlStr, signal) {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if (!res.ok) return { ok: false, error: `${res.status} ${res.statusText}` };
|
|
169
|
-
const data = await res.json();
|
|
170
|
-
return { ok: true, data: { provider: 'generic', query, ...data } };
|
|
171
|
-
} catch (err) {
|
|
172
|
-
if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
|
|
173
|
-
return { ok: false, error: err.message || String(err) };
|
|
174
|
-
}
|
|
165
|
+
const url = new URL(urlStr);
|
|
166
|
+
url.searchParams.set('q', query);
|
|
167
|
+
url.searchParams.set('limit', String(limit));
|
|
168
|
+
const res = await fetch(url.toString(), { signal, headers: { 'User-Agent': 'Yeaft/1.0', Accept: 'application/json' } });
|
|
169
|
+
if (!res.ok) return webHttpFailure(res.status);
|
|
170
|
+
const data = await res.json();
|
|
171
|
+
if (!Array.isArray(data?.results) || data.error) return invalidSearchResponse();
|
|
172
|
+
return { ...data, provider: 'generic', query, results: data.results.slice(0, limit) };
|
|
175
173
|
}
|
|
176
174
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
* then Bing. We intentionally keep the regex-based parsers minimal —
|
|
181
|
-
* they break less than full DOM selectors when sites tweak markup.
|
|
182
|
-
*/
|
|
183
|
-
async function tryHtmlScrape(query, limit, signal) {
|
|
184
|
-
const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
185
|
-
const headers = { 'User-Agent': ua, 'Accept-Language': 'en-US,en;q=0.9' };
|
|
186
|
-
|
|
187
|
-
try {
|
|
188
|
-
const ddg = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, { signal, headers });
|
|
189
|
-
if (ddg.ok) {
|
|
190
|
-
const html = await ddg.text();
|
|
191
|
-
const results = parseDdgHtml(html, limit);
|
|
192
|
-
if (results.length) return { ok: true, data: { provider: 'duckduckgo-html', query, results } };
|
|
193
|
-
}
|
|
194
|
-
} catch (err) {
|
|
195
|
-
if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
try {
|
|
199
|
-
const bing = await fetch(`https://www.bing.com/search?q=${encodeURIComponent(query)}`, { signal, headers });
|
|
200
|
-
if (bing.ok) {
|
|
201
|
-
const html = await bing.text();
|
|
202
|
-
const results = parseBingHtml(html, limit);
|
|
203
|
-
if (results.length) return { ok: true, data: { provider: 'bing-html', query, results } };
|
|
204
|
-
}
|
|
205
|
-
} catch (err) {
|
|
206
|
-
if (err?.name === 'AbortError') return { ok: false, error: 'cancelled' };
|
|
207
|
-
}
|
|
175
|
+
function invalidSearchResponse() {
|
|
176
|
+
return { error: 'Search endpoint did not return a results array', code: 'invalid_response', errorEffect: 'none', hint: 'Verify the configured endpoint returns a JSON object with results: [], including for no matches.' };
|
|
177
|
+
}
|
|
208
178
|
|
|
209
|
-
|
|
179
|
+
/** Zero matches need an explicit no-results marker, not an empty parser output. */
|
|
180
|
+
async function tryHtmlScrape(query, limit, provider, signal) {
|
|
181
|
+
const ddg = provider === 'duckduckgo-html';
|
|
182
|
+
const endpoint = ddg ? 'https://html.duckduckgo.com/html/?q=' : 'https://www.bing.com/search?q=';
|
|
183
|
+
const res = await fetch(endpoint + encodeURIComponent(query), {
|
|
184
|
+
signal,
|
|
185
|
+
headers: {
|
|
186
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
187
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
188
|
+
},
|
|
189
|
+
});
|
|
190
|
+
if (!res.ok) return webHttpFailure(res.status);
|
|
191
|
+
const html = await res.text();
|
|
192
|
+
const blocked = /anomaly\.js|anomaly-modal|challenge-form|verify (?:that )?you(?:'re| are) (?:a )?human|unusual traffic|<title[^>]*>[^<]*(?:captcha|access denied)/i.test(html);
|
|
193
|
+
const results = ddg ? parseDdgHtml(html, limit) : parseBingHtml(html, limit);
|
|
194
|
+
const noMatches = ddg ? /class=["'][^"']*\bno-results\b|No results found for/i.test(html)
|
|
195
|
+
: /class=["'][^"']*\bb_no\b|There are no results for|No results found for/i.test(html);
|
|
196
|
+
if (!blocked && (results.length || noMatches)) return { provider, query, results };
|
|
197
|
+
return {
|
|
198
|
+
error: blocked ? 'HTML search returned a bot challenge' : 'HTML search markup was not recognized; zero matches are not confirmed',
|
|
199
|
+
code: blocked ? 'bot_blocked' : 'unrecognized_response', status: res.status, errorEffect: 'none',
|
|
200
|
+
hint: 'Use a known official URL or another available source. Retrying a rephrased query will not fix a bot challenge or parser mismatch.',
|
|
201
|
+
};
|
|
210
202
|
}
|
|
211
203
|
|
|
212
204
|
/**
|