@yeaft/webchat-agent 1.0.590 → 1.0.592
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local-runtime/server/handlers/client-conversation.js +76 -11
- package/local-runtime/version.json +1 -1
- package/local-runtime/web/app.bundle.js +103 -103
- package/local-runtime/web/app.bundle.js.gz +0 -0
- package/local-runtime/web/index.html +2 -2
- package/local-runtime/web/style.bundle.css +1 -1
- package/local-runtime/web/style.bundle.css.gz +0 -0
- package/package.json +1 -1
- package/yeaft/attachments.js +190 -46
- package/yeaft/conversation/persist.js +3 -1
- package/yeaft/engine.js +39 -4
- package/yeaft/llm/adapter.js +2 -2
- package/yeaft/llm/openai-responses.js +4 -1
- package/yeaft/tool-folding/index.js +12 -2
- package/yeaft/tools/registry.js +11 -0
- package/yeaft/tools/view-image.js +5 -4
- package/yeaft/web-bridge.js +12 -8
|
Binary file
|
package/package.json
CHANGED
package/yeaft/attachments.js
CHANGED
|
@@ -9,16 +9,17 @@
|
|
|
9
9
|
* user message into that query's `inputStream`.
|
|
10
10
|
* - Yeaft mode has many VPs taking turns inside a group, no per-VP
|
|
11
11
|
* workDir, and the Engine accepts the user message via `query()`
|
|
12
|
-
* args.
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* args. Session uploads belong to the instance's
|
|
13
|
+
* `sessions/<sessionId>/attachments` directory, independent of workDir.
|
|
14
|
+
* Non-Session callers retain the legacy CWD-relative folder.
|
|
15
|
+
* We hand back the persisted-form
|
|
15
16
|
* metadata AND a `promptParts` content array (image blocks +
|
|
16
17
|
* synthesized [Uploaded files] suffix) for the LLM call.
|
|
17
18
|
*
|
|
18
19
|
* Inputs (`files`) come from the server-side resolver in
|
|
19
20
|
* `client-conversation.js`: each entry is
|
|
20
21
|
* `{ name, mimeType, data: <base64>, isImage }` — the `pendingFiles`
|
|
21
|
-
* `fileId`
|
|
22
|
+
* `fileId` is consumed only after the server successfully forwards the message.
|
|
22
23
|
*
|
|
23
24
|
* Output (single bundle, all named for the role each piece plays in
|
|
24
25
|
* the LLM call):
|
|
@@ -40,8 +41,8 @@
|
|
|
40
41
|
* than swallowing it in a console.warn.
|
|
41
42
|
*/
|
|
42
43
|
|
|
43
|
-
import {
|
|
44
|
-
import { basename, extname, join, resolve, relative, isAbsolute } from 'node:path';
|
|
44
|
+
import { mkdirSync, writeFileSync, lstatSync, openSync, closeSync, fstatSync, readSync, constants } from 'node:fs';
|
|
45
|
+
import { basename, extname, join, resolve, relative, isAbsolute, sep } from 'node:path';
|
|
45
46
|
import { randomBytes } from 'node:crypto';
|
|
46
47
|
|
|
47
48
|
// Same dir name Chat mode uses, so ".gitignore" rules and tool-side
|
|
@@ -89,6 +90,8 @@ function sanitizeBaseName(base) {
|
|
|
89
90
|
* @param {Array<{name:string, mimeType:string, data:string, isImage?:boolean}>} files
|
|
90
91
|
* Resolved files from server (pendingFiles → base64).
|
|
91
92
|
* @param {Object} [opts]
|
|
93
|
+
* @param {string} [opts.yeaftDir] Instance data root for Session uploads.
|
|
94
|
+
* @param {string} [opts.sessionId] Session owner (required with yeaftDir).
|
|
92
95
|
* @param {string} [opts.subdir] Sub-folder under TEMP_UPLOAD_DIR
|
|
93
96
|
* (e.g. sessionId). Lets multiple groups co-exist without clobbering.
|
|
94
97
|
* @param {string} [opts.cwd] Override base dir; defaults to process.cwd()
|
|
@@ -107,9 +110,11 @@ export function persistYeaftAttachments(files, opts = {}) {
|
|
|
107
110
|
// from sessionId. Keep the existing strict policy here; this is NOT
|
|
108
111
|
// user-visible.
|
|
109
112
|
const subdir = opts.subdir ? String(opts.subdir).replace(/[^a-zA-Z0-9._-]/g, '_') : '';
|
|
110
|
-
const
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
const sessionUpload = opts.yeaftDir != null || opts.sessionId != null;
|
|
114
|
+
const uploadDir = sessionUpload
|
|
115
|
+
? sessionAttachmentRoot(opts)
|
|
116
|
+
: (subdir && subdir !== '.' && subdir !== '..'
|
|
117
|
+
? resolve(cwd, TEMP_UPLOAD_DIR, subdir) : resolve(cwd, TEMP_UPLOAD_DIR));
|
|
113
118
|
|
|
114
119
|
if (!Array.isArray(files) || files.length === 0) {
|
|
115
120
|
return { promptAttachments: [], promptSuffix: '', promptParts: [], failed: [] };
|
|
@@ -120,8 +125,17 @@ export function persistYeaftAttachments(files, opts = {}) {
|
|
|
120
125
|
const accepted = files.slice(0, MAX_FILES_PER_TURN);
|
|
121
126
|
const rejectedByCount = files.slice(MAX_FILES_PER_TURN);
|
|
122
127
|
|
|
123
|
-
|
|
128
|
+
try {
|
|
129
|
+
if (!uploadDir) throw new Error('invalid Session attachment owner');
|
|
130
|
+
const root = sessionUpload ? resolve(opts.yeaftDir) : resolve(cwd);
|
|
131
|
+
assertNoSymlinks(root, uploadDir, true);
|
|
124
132
|
mkdirSync(uploadDir, { recursive: true });
|
|
133
|
+
assertNoSymlinks(root, uploadDir);
|
|
134
|
+
} catch (error) {
|
|
135
|
+
return {
|
|
136
|
+
promptAttachments: [], promptSuffix: '', promptParts: [],
|
|
137
|
+
failed: files.map(file => ({ name: file?.name || '<unknown>', error: error.message })),
|
|
138
|
+
};
|
|
125
139
|
}
|
|
126
140
|
|
|
127
141
|
const promptAttachments = [];
|
|
@@ -141,17 +155,18 @@ export function persistYeaftAttachments(files, opts = {}) {
|
|
|
141
155
|
continue;
|
|
142
156
|
}
|
|
143
157
|
try {
|
|
144
|
-
const
|
|
145
|
-
const
|
|
158
|
+
const rawExt = extname(file.name);
|
|
159
|
+
const ext = rawExt.replace(/[\\\/\0-\x1f\x7f]/g, '_');
|
|
160
|
+
const base = basename(file.name, rawExt);
|
|
146
161
|
const safeBase = sanitizeBaseName(base);
|
|
147
162
|
// Identity comes from random bytes — a clock is not an identity.
|
|
148
163
|
// 4 bytes (2^32) is plenty for a 16-file cap.
|
|
149
164
|
const suffix = randomBytes(4).toString('hex');
|
|
150
165
|
const uniqueName = `${safeBase}_${suffix}${ext || ''}`;
|
|
151
166
|
const absPath = join(uploadDir, uniqueName);
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
167
|
+
// Absolute Session references keep file tools independent of workDir;
|
|
168
|
+
// readers still require this instance and Session's exact asset root.
|
|
169
|
+
const persistedPath = sessionUpload ? absPath : relative(resolve(cwd), absPath);
|
|
155
170
|
|
|
156
171
|
const buffer = Buffer.from(file.data, 'base64');
|
|
157
172
|
|
|
@@ -166,12 +181,12 @@ export function persistYeaftAttachments(files, opts = {}) {
|
|
|
166
181
|
}
|
|
167
182
|
totalBytes += buffer.length;
|
|
168
183
|
|
|
169
|
-
writeFileSync(absPath, buffer);
|
|
184
|
+
writeFileSync(absPath, buffer, { flag: 'wx', mode: 0o600 });
|
|
170
185
|
|
|
171
186
|
const isImage = !!file.isImage || (file.mimeType || '').startsWith('image/');
|
|
172
187
|
promptAttachments.push({
|
|
173
188
|
name: file.name,
|
|
174
|
-
path:
|
|
189
|
+
path: persistedPath,
|
|
175
190
|
mimeType: file.mimeType || 'application/octet-stream',
|
|
176
191
|
isImage,
|
|
177
192
|
});
|
|
@@ -230,48 +245,177 @@ export function attachmentsForPersistence(promptAttachments) {
|
|
|
230
245
|
}));
|
|
231
246
|
}
|
|
232
247
|
|
|
248
|
+
/** Session IDs are identities, never sanitized into another Session's directory. */
|
|
249
|
+
function validSessionId(value) {
|
|
250
|
+
return typeof value === 'string' && /^[a-zA-Z0-9_-][a-zA-Z0-9._-]*$/.test(value);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function sessionAttachmentRoot({ yeaftDir, sessionId }) {
|
|
254
|
+
return typeof yeaftDir === 'string' && yeaftDir && validSessionId(sessionId)
|
|
255
|
+
? resolve(yeaftDir, 'sessions', sessionId, 'attachments') : null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function isWithin(root, path) {
|
|
259
|
+
const rel = relative(root, path);
|
|
260
|
+
return rel !== '' && rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Check the trusted base itself and every descendant, including intermediate
|
|
264
|
+
// directories. O_NOFOLLOW below also guards replacement of the final file.
|
|
265
|
+
// This is not a sandbox against concurrently replaced ancestor directories.
|
|
266
|
+
function assertNoSymlinks(root, path, allowMissing = false) {
|
|
267
|
+
if (path !== root && !isWithin(root, path)) throw new Error('attachment path outside owner root');
|
|
268
|
+
let current = root;
|
|
269
|
+
for (const part of ['', ...relative(root, path).split(sep).filter(Boolean)]) {
|
|
270
|
+
if (part) current = join(current, part);
|
|
271
|
+
try {
|
|
272
|
+
if (lstatSync(current).isSymbolicLink()) throw new Error('attachment symlink is not allowed');
|
|
273
|
+
} catch (error) {
|
|
274
|
+
if (allowMissing && error.code === 'ENOENT') continue;
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
233
280
|
/**
|
|
234
|
-
* Resolve
|
|
235
|
-
*
|
|
236
|
-
*
|
|
237
|
-
*
|
|
281
|
+
* Resolve only this Session's assets, or its CWD-relative legacy uploads.
|
|
282
|
+
* Absolute paths are allowed ONLY in <yeaftDir>/sessions/<sessionId>/attachments.
|
|
283
|
+
* No workDir search/fallback: legacy Web uploads were written under Agent CWD.
|
|
284
|
+
* Non-Session callers retain their original CWD-relative upload root.
|
|
238
285
|
* @param {string} attachmentPath
|
|
239
|
-
* @param {{
|
|
286
|
+
* @param {{cwd?:string, legacyRoot?:string, yeaftDir?:string, sessionId?:string}} [opts]
|
|
287
|
+
* legacyRoot is the original Agent CWD (not workDir or the upload subdir).
|
|
240
288
|
* @returns {string|null}
|
|
241
289
|
*/
|
|
242
290
|
export function resolvePersistedAttachmentPath(attachmentPath, opts = {}) {
|
|
243
|
-
if (!attachmentPath || typeof attachmentPath !== 'string') return null;
|
|
244
|
-
|
|
245
|
-
const
|
|
246
|
-
const
|
|
291
|
+
if (!attachmentPath || typeof attachmentPath !== 'string' || attachmentPath.includes('\0')) return null;
|
|
292
|
+
const cwd = resolve(opts.legacyRoot || opts.cwd || process.cwd());
|
|
293
|
+
const scoped = opts.sessionId != null || opts.yeaftDir != null;
|
|
294
|
+
const sessionRoot = sessionAttachmentRoot(opts);
|
|
295
|
+
if (scoped && !sessionRoot) return null;
|
|
247
296
|
const absPath = resolve(cwd, attachmentPath);
|
|
248
|
-
|
|
249
|
-
if (
|
|
250
|
-
|
|
297
|
+
let root;
|
|
298
|
+
if (isAbsolute(attachmentPath)) {
|
|
299
|
+
if (!sessionRoot || !isWithin(sessionRoot, absPath)) return null;
|
|
300
|
+
root = resolve(opts.yeaftDir);
|
|
301
|
+
} else {
|
|
302
|
+
const legacyRoot = scoped
|
|
303
|
+
? resolve(cwd, TEMP_UPLOAD_DIR, opts.sessionId) : resolve(cwd, TEMP_UPLOAD_DIR);
|
|
304
|
+
if (!isWithin(legacyRoot, absPath)) return null;
|
|
305
|
+
root = cwd;
|
|
306
|
+
}
|
|
307
|
+
try {
|
|
308
|
+
assertNoSymlinks(root, absPath);
|
|
309
|
+
return absPath;
|
|
310
|
+
} catch {
|
|
311
|
+
return null;
|
|
312
|
+
}
|
|
251
313
|
}
|
|
252
314
|
|
|
253
|
-
/**
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
* @param {{name?:string, path?:string, mimeType?:string, isImage?:boolean}} att
|
|
259
|
-
* @param {{ cwd?: string }} [opts]
|
|
260
|
-
* @returns {{data:string,mimeType:string,filename:string}|null}
|
|
261
|
-
*/
|
|
262
|
-
export function persistedAttachmentPreviewPayload(att, opts = {}) {
|
|
263
|
-
if (!att || !att.isImage || !att.path) return null;
|
|
264
|
-
const absPath = resolvePersistedAttachmentPath(att.path, opts);
|
|
265
|
-
if (!absPath) return null;
|
|
315
|
+
/** Shared bounded read for model history and browser preview. Never follows symlinks. */
|
|
316
|
+
function readPersistedAttachment(att, opts, maxBytes) {
|
|
317
|
+
const absPath = resolvePersistedAttachmentPath(att?.path, opts);
|
|
318
|
+
if (!absPath) throw new Error('file missing or outside this Session / 文件缺失或不属于本会话');
|
|
319
|
+
const fd = openSync(absPath, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
|
|
266
320
|
try {
|
|
267
|
-
const
|
|
268
|
-
if (!
|
|
321
|
+
const stat = fstatSync(fd);
|
|
322
|
+
if (!stat.isFile()) throw new Error('not a regular file');
|
|
323
|
+
if (stat.size > maxBytes) throw new Error('image exceeds attachment size limit');
|
|
324
|
+
const buffer = Buffer.alloc(stat.size);
|
|
325
|
+
let offset = 0;
|
|
326
|
+
while (offset < buffer.length) {
|
|
327
|
+
const read = readSync(fd, buffer, offset, buffer.length - offset, null);
|
|
328
|
+
if (!read) throw new Error('image changed while reading');
|
|
329
|
+
offset += read;
|
|
330
|
+
}
|
|
269
331
|
return {
|
|
270
|
-
data:
|
|
271
|
-
mimeType: att.mimeType || '
|
|
332
|
+
data: buffer.toString('base64'),
|
|
333
|
+
mimeType: att.mimeType || 'image/png',
|
|
272
334
|
filename: att.name || basename(absPath),
|
|
335
|
+
bytes: buffer.length,
|
|
273
336
|
};
|
|
337
|
+
} finally {
|
|
338
|
+
closeSync(fd);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/** Short-lived web preview; missing/denied/oversized images have no preview. */
|
|
343
|
+
export function persistedAttachmentPreviewPayload(att, opts = {}) {
|
|
344
|
+
if (!att?.isImage || !att.path) return null;
|
|
345
|
+
try {
|
|
346
|
+
const { bytes, ...payload } = readPersistedAttachment(att, opts, MAX_PREVIEW_BYTES);
|
|
347
|
+
return payload;
|
|
274
348
|
} catch {
|
|
275
349
|
return null;
|
|
276
350
|
}
|
|
277
351
|
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* Build provider content from one canonical user row, without a Web runtime.
|
|
355
|
+
* Call once on raw transcript rows, before history-window budgeting (including
|
|
356
|
+
* recalled turns). Never persist the returned base64 or unavailable notices.
|
|
357
|
+
* Ownership comes from the requested Session, not an attachment's metadata.
|
|
358
|
+
*
|
|
359
|
+
* @param {{role:string, sessionId?:string, content?:string|Array, attachments?:Array}} row
|
|
360
|
+
* @param {{yeaftDir:string, sessionId:string, legacyRoot?:string, cwd?:string}} opts
|
|
361
|
+
* legacyRoot is the original Agent CWD; cwd is its compatibility alias.
|
|
362
|
+
* @returns {Array} Standard provider content parts; the input row is unchanged.
|
|
363
|
+
*/
|
|
364
|
+
export function hydratePersistedUserAttachmentContent(row, opts = {}) {
|
|
365
|
+
const content = Array.isArray(row?.content) ? [...row.content]
|
|
366
|
+
: (row?.content ? [{ type: 'text', text: String(row.content) }] : []);
|
|
367
|
+
if (row?.role !== 'user' || row.sessionId !== opts.sessionId || !validSessionId(opts.sessionId)) return content;
|
|
368
|
+
const images = Array.isArray(row.attachments) ? row.attachments.filter(att => att?.isImage) : [];
|
|
369
|
+
let totalBytes = 0;
|
|
370
|
+
for (const att of images.slice(0, MAX_FILES_PER_TURN)) {
|
|
371
|
+
try {
|
|
372
|
+
const payload = readPersistedAttachment(att, opts, MAX_TOTAL_BYTES - totalBytes);
|
|
373
|
+
totalBytes += payload.bytes;
|
|
374
|
+
content.push({ type: 'image', source: { type: 'base64', media_type: payload.mimeType, data: payload.data } });
|
|
375
|
+
} catch (error) {
|
|
376
|
+
content.push({ type: 'text', text: `[Uploaded image unavailable / 上传图片不可用: ${att.name || 'image'} — ${error.message}. Ask the user to upload it again / 请用户重新上传。]` });
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (images.length > MAX_FILES_PER_TURN) content.push({ type: 'text', text: '[Uploaded image unavailable: attachment count limit exceeded.]' });
|
|
380
|
+
return content;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/** Provider-only projection for a canonical history array; no store decoration. */
|
|
384
|
+
export function hydratePersistedAttachmentHistory(messages, opts = {}) {
|
|
385
|
+
return messages.map(row => {
|
|
386
|
+
if (row?.role !== 'user' || row.sessionId !== opts.sessionId || !validSessionId(opts.sessionId)
|
|
387
|
+
|| !Array.isArray(row.attachments) || !row.attachments.some(att => att?.isImage)) return row;
|
|
388
|
+
return { ...row, content: hydratePersistedUserAttachmentContent(row, opts) };
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Clone selected transcript attachments into the target Session's owned root.
|
|
394
|
+
* Missing/denied legacy references remain unavailable (never broaden access).
|
|
395
|
+
* Copy errors for available files fail the fork so CRUD can roll it back.
|
|
396
|
+
* @param {Array} rows Selected canonical source rows, including cold history.
|
|
397
|
+
* @param {{yeaftDir:string, sourceSessionId:string, targetSessionId:string}} opts
|
|
398
|
+
* @returns {Array} New rows; callers remap message and Session identities.
|
|
399
|
+
*/
|
|
400
|
+
export function copyPersistedSessionAttachments(rows, { yeaftDir, sourceSessionId, targetSessionId }) {
|
|
401
|
+
const sourceOptions = { yeaftDir, sessionId: sourceSessionId };
|
|
402
|
+
const copiedPaths = new Map();
|
|
403
|
+
return rows.map(row => {
|
|
404
|
+
if (row.sessionId !== sourceSessionId || !Array.isArray(row.attachments)) return row;
|
|
405
|
+
return { ...row, attachments: row.attachments.map(att => {
|
|
406
|
+
if (!att?.path) return att;
|
|
407
|
+
if (copiedPaths.has(att.path)) return { ...att, path: copiedPaths.get(att.path) };
|
|
408
|
+
if (!resolvePersistedAttachmentPath(att.path, sourceOptions)) return att;
|
|
409
|
+
const payload = readPersistedAttachment(att, sourceOptions, MAX_TOTAL_BYTES);
|
|
410
|
+
const bundle = persistYeaftAttachments([{ ...att, name: payload.filename, data: payload.data }], {
|
|
411
|
+
yeaftDir, sessionId: targetSessionId,
|
|
412
|
+
});
|
|
413
|
+
if (bundle.failed.length || bundle.promptAttachments.length !== 1) {
|
|
414
|
+
throw new Error(`Session attachment copy failed: ${bundle.failed[0]?.error || 'missing output'}`);
|
|
415
|
+
}
|
|
416
|
+
const path = bundle.promptAttachments[0].path;
|
|
417
|
+
copiedPaths.set(att.path, path);
|
|
418
|
+
return { ...att, path };
|
|
419
|
+
}) };
|
|
420
|
+
});
|
|
421
|
+
}
|
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
iterateCanonicalVisibleEntriesNewestFirst,
|
|
32
32
|
normalizeLiteralSearch,
|
|
33
33
|
} from './visible-entry.js';
|
|
34
|
+
import { copyPersistedSessionAttachments } from '../attachments.js';
|
|
34
35
|
import { markConversationDirty } from './history-index-state.js';
|
|
35
36
|
|
|
36
37
|
/**
|
|
@@ -2035,7 +2036,8 @@ export class ConversationStore {
|
|
|
2035
2036
|
`m${String(firstSeq + index).padStart(4, '0')}`,
|
|
2036
2037
|
]));
|
|
2037
2038
|
const remapId = id => idMap.get(id) || id;
|
|
2038
|
-
const
|
|
2039
|
+
const assetRows = copyPersistedSessionAttachments(rows, { yeaftDir: this.#dir, sourceSessionId, targetSessionId });
|
|
2040
|
+
const copies = assetRows.map(row => {
|
|
2039
2041
|
const copy = {
|
|
2040
2042
|
...row,
|
|
2041
2043
|
id: idMap.get(row.id),
|
package/yeaft/engine.js
CHANGED
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
} from './history-window.js';
|
|
43
43
|
import { recallConversationTurns } from './conversation/history-index.js';
|
|
44
44
|
import { parseSeqFromId } from './conversation/persist.js';
|
|
45
|
+
import { hydratePersistedAttachmentHistory } from './attachments.js';
|
|
45
46
|
import { isVpForeign, readContent as readScopeContent } from './memory/store.js';
|
|
46
47
|
import { ActiveMemorySet } from './memory/ams.js';
|
|
47
48
|
import { cleanMemoryPromptText } from './memory/prompt-cleanup.js';
|
|
@@ -2180,7 +2181,13 @@ export class Engine {
|
|
|
2180
2181
|
if (Number.isFinite(beforeSeq)) {
|
|
2181
2182
|
const loadHistory = this.#conversationStore.loadProviderHistoryBySession
|
|
2182
2183
|
|| this.#conversationStore.loadRecentBySession;
|
|
2183
|
-
|
|
2184
|
+
// Materialize only provider copies from this Session's controlled assets.
|
|
2185
|
+
// Hydrate before budgeting; never mutate/persist base64 into canonical rows.
|
|
2186
|
+
const attachmentOptions = { yeaftDir: this.#yeaftDir, sessionId: runtimeSessionId, legacyRoot: process.cwd() };
|
|
2187
|
+
const tail = hydratePersistedAttachmentHistory(
|
|
2188
|
+
await loadHistory.call(this.#conversationStore, runtimeSessionId, recentTurnCap, { beforeSeq }),
|
|
2189
|
+
attachmentOptions,
|
|
2190
|
+
);
|
|
2184
2191
|
messages = tail.filter(m => parseSeqFromId(m.id) < beforeSeq
|
|
2185
2192
|
&& (m.role !== 'tool' || !queryVpId || m.speakerVpId === queryVpId))
|
|
2186
2193
|
.map(m => {
|
|
@@ -2193,7 +2200,19 @@ export class Engine {
|
|
|
2193
2200
|
const recalled = await recallConversationTurns(this.#yeaftDir, runtimeSessionId, prompt, {
|
|
2194
2201
|
beforeSeq, limit: relatedTurnCap,
|
|
2195
2202
|
});
|
|
2196
|
-
relatedHistoryTurns = recalled.turns || []
|
|
2203
|
+
relatedHistoryTurns = (recalled.turns || []).map(turn => ({
|
|
2204
|
+
...turn,
|
|
2205
|
+
// Recall indexes intentionally contain text/identities, not assets.
|
|
2206
|
+
// Reload only the exact same-Session user anchor's references.
|
|
2207
|
+
messages: hydratePersistedAttachmentHistory((turn.messages || []).map(row => {
|
|
2208
|
+
if (row.role !== 'user' || row.sessionId !== runtimeSessionId
|
|
2209
|
+
|| !Number.isFinite(row.seq) || row.seq >= beforeSeq) return row;
|
|
2210
|
+
const anchor = this.#conversationStore.loadRecentBySession(runtimeSessionId, 1, { beforeSeq: row.seq + 1 })
|
|
2211
|
+
.find(candidate => candidate.role === 'user' && candidate.sessionId === runtimeSessionId
|
|
2212
|
+
&& candidate.id === row.messageId && parseSeqFromId(candidate.id) === row.seq);
|
|
2213
|
+
return anchor?.attachments ? { ...row, attachments: anchor.attachments } : row;
|
|
2214
|
+
}), attachmentOptions),
|
|
2215
|
+
}));
|
|
2197
2216
|
historyRecallMeta = { source: 'messages', status: 'ready', ...recalled.meta };
|
|
2198
2217
|
} catch (error) {
|
|
2199
2218
|
// Cold/stale index degrades to recent history, never a full scan.
|
|
@@ -4279,6 +4298,7 @@ export class Engine {
|
|
|
4279
4298
|
// break out of the outer while-loop cleanly once the current
|
|
4280
4299
|
// tool batch finishes reporting.
|
|
4281
4300
|
let abortedDuringTools = false;
|
|
4301
|
+
let batchHasUnseenImages = false;
|
|
4282
4302
|
/** @type {string[]} */
|
|
4283
4303
|
const pendingDupReminders = [];
|
|
4284
4304
|
let terminateAfterDuplicateBatch = false;
|
|
@@ -4439,6 +4459,7 @@ export class Engine {
|
|
|
4439
4459
|
|
|
4440
4460
|
let output;
|
|
4441
4461
|
let displayImages = [];
|
|
4462
|
+
let modelImages = [];
|
|
4442
4463
|
let isError = false;
|
|
4443
4464
|
let reusedReadOnlyResult = false;
|
|
4444
4465
|
let reusedReadOnlyCallId = null;
|
|
@@ -4575,6 +4596,7 @@ export class Engine {
|
|
|
4575
4596
|
const cachedReadOnly = readOnlyToolResults.get(duplicateKey);
|
|
4576
4597
|
if (!readOnlyToolReuseDisabled && cachedReadOnly && cacheableTool) {
|
|
4577
4598
|
output = cachedReadOnly.output;
|
|
4599
|
+
modelImages = cachedReadOnly.modelImages || [];
|
|
4578
4600
|
isError = Boolean(cachedReadOnly.isError);
|
|
4579
4601
|
reusedReadOnlyResult = true;
|
|
4580
4602
|
reusedReadOnlyCallId = cachedReadOnly.callId || null;
|
|
@@ -4619,6 +4641,13 @@ export class Engine {
|
|
|
4619
4641
|
}
|
|
4620
4642
|
displayImages = extractDisplayImages(tc.name, output);
|
|
4621
4643
|
if (displayImages.length > 0) {
|
|
4644
|
+
// UI asset delivery and model vision are independent consumers.
|
|
4645
|
+
// Capture provider parts before yielding to the UI bridge; never
|
|
4646
|
+
// persist base64 in transcript/exec-log or textual tool output.
|
|
4647
|
+
modelImages = displayImages.map(image => ({
|
|
4648
|
+
type: 'image',
|
|
4649
|
+
source: { type: 'base64', media_type: image.mimeType, data: image.previewData.data },
|
|
4650
|
+
}));
|
|
4622
4651
|
output = stripDisplayImageData(output, displayImages);
|
|
4623
4652
|
}
|
|
4624
4653
|
isError = toolErrorOutput === 'json-error-envelope' && isToolErrorOutput(output);
|
|
@@ -4752,6 +4781,7 @@ export class Engine {
|
|
|
4752
4781
|
&& !readOnlyToolReuseDisabled && cacheableTool) {
|
|
4753
4782
|
readOnlyToolResults.set(duplicateKey, {
|
|
4754
4783
|
output,
|
|
4784
|
+
modelImages,
|
|
4755
4785
|
isError,
|
|
4756
4786
|
callId: tc.id,
|
|
4757
4787
|
});
|
|
@@ -4764,10 +4794,13 @@ export class Engine {
|
|
|
4764
4794
|
toolName: tc.name,
|
|
4765
4795
|
language: this.#config?.language,
|
|
4766
4796
|
});
|
|
4797
|
+
if (modelImages.length > 0) batchHasUnseenImages = true;
|
|
4767
4798
|
const toolMessage = {
|
|
4768
4799
|
role: 'tool',
|
|
4769
4800
|
toolCallId: tc.id,
|
|
4770
|
-
content:
|
|
4801
|
+
content: modelImages.length > 0
|
|
4802
|
+
? [{ type: 'text', text: contextOutput }, ...modelImages]
|
|
4803
|
+
: contextOutput,
|
|
4771
4804
|
isError,
|
|
4772
4805
|
};
|
|
4773
4806
|
conversationMessages.push(toolMessage);
|
|
@@ -4896,7 +4929,9 @@ export class Engine {
|
|
|
4896
4929
|
const completedToolLoops = toolLoopTurns + 1;
|
|
4897
4930
|
const t1BatchDue = completedToolLoops - lastT1AtLoopCount
|
|
4898
4931
|
>= TOOL_LOOP_REFLECTION_INTERVAL;
|
|
4899
|
-
|
|
4932
|
+
// Text-only reflection cannot consume visual input. Defer folding until
|
|
4933
|
+
// the primary provider has received this image batch at least once.
|
|
4934
|
+
if (t1BatchDue && !batchHasUnseenImages && !toolBatchBarrier
|
|
4900
4935
|
&& !abortedDuringTools && !signal?.aborted) {
|
|
4901
4936
|
const t1DedupKey = `${queryNumber}:t1-loop:${completedToolLoops}`;
|
|
4902
4937
|
if (this.#reflectedTurns.has(t1DedupKey)) {
|
package/yeaft/llm/adapter.js
CHANGED
|
@@ -59,9 +59,9 @@ import { utf8PrefixWithinBytes } from '../utf8.js';
|
|
|
59
59
|
* @typedef {{ thinking: string, signature: string }} ThinkingBlock
|
|
60
60
|
*
|
|
61
61
|
* @typedef {{ role: 'system', content: string }} SystemMessage
|
|
62
|
-
* @typedef {{ role: 'user', content: string }} UserMessage
|
|
62
|
+
* @typedef {{ role: 'user', content: string|Array<object> }} UserMessage
|
|
63
63
|
* @typedef {{ role: 'assistant', content: string, toolCalls?: UnifiedToolCall[], thinkingBlocks?: ThinkingBlock[] }} AssistantMessage
|
|
64
|
-
* @typedef {{ role: 'tool', toolCallId: string, content: string
|
|
64
|
+
* @typedef {{ role: 'tool', toolCallId: string, content: string|Array<object>, isError?: boolean }} ToolMessage
|
|
65
65
|
*
|
|
66
66
|
* @typedef {SystemMessage | UserMessage | AssistantMessage | ToolMessage} UnifiedMessage
|
|
67
67
|
*/
|
|
@@ -205,7 +205,10 @@ export class OpenAIResponsesAdapter extends LLMAdapter {
|
|
|
205
205
|
input.push({
|
|
206
206
|
type: 'function_call_output',
|
|
207
207
|
call_id: msg.toolCallId,
|
|
208
|
-
output:
|
|
208
|
+
output: Array.isArray(msg.content) && msg.content.length > 0
|
|
209
|
+
&& msg.content.every(part => part && ['text', 'image', 'document'].includes(part.type))
|
|
210
|
+
? this.#translateUserContent(msg.content)
|
|
211
|
+
: (typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content)),
|
|
209
212
|
});
|
|
210
213
|
}
|
|
211
214
|
}
|
|
@@ -125,6 +125,16 @@ export function buildDuplicateReminder({ toolName, count, lastResultBrief }) {
|
|
|
125
125
|
+ `Consider whether re-running this tool is necessary or if you should try a different approach.`;
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
+
function toolOutputForReflection(content) {
|
|
129
|
+
if (typeof content === 'string') return content;
|
|
130
|
+
if (Array.isArray(content) && content.every(part => part && ['text', 'image', 'document'].includes(part.type))) {
|
|
131
|
+
// Reflection consumes text, not a base64 dump of visual context. The image
|
|
132
|
+
// has already been delivered to the primary model in the paired result.
|
|
133
|
+
return content.map(part => part.type === 'text' ? part.text : `[${part.type} content omitted from reflection]`).join('\n');
|
|
134
|
+
}
|
|
135
|
+
return JSON.stringify(content);
|
|
136
|
+
}
|
|
137
|
+
|
|
128
138
|
/**
|
|
129
139
|
* Convert assistant.toolCalls + matching tool results into the
|
|
130
140
|
* { name, input, output, isError } pairs the reflector prompt expects.
|
|
@@ -159,14 +169,14 @@ export function extractToolPairsFromRange(messages, startIdx, endIdx) {
|
|
|
159
169
|
} else if (m.role === 'tool') {
|
|
160
170
|
const ent = m.toolCallId ? byId.get(m.toolCallId) : null;
|
|
161
171
|
if (ent) {
|
|
162
|
-
ent.output =
|
|
172
|
+
ent.output = toolOutputForReflection(m.content);
|
|
163
173
|
ent.isError = !!m.isError;
|
|
164
174
|
} else {
|
|
165
175
|
// Orphan tool result — still include for completeness.
|
|
166
176
|
pairs.push({
|
|
167
177
|
name: '(orphan)',
|
|
168
178
|
input: {},
|
|
169
|
-
output:
|
|
179
|
+
output: toolOutputForReflection(m.content),
|
|
170
180
|
isError: !!m.isError,
|
|
171
181
|
});
|
|
172
182
|
}
|
package/yeaft/tools/registry.js
CHANGED
|
@@ -238,6 +238,17 @@ function truncateUtf8(text, maxBytes) {
|
|
|
238
238
|
}
|
|
239
239
|
|
|
240
240
|
export function truncateToolResultIfNeeded(output, { toolName, language } = {}) {
|
|
241
|
+
// Provider content parts are not JSON text. Bound their text together while
|
|
242
|
+
// preserving binary blocks; history-window budgets/omits attachments whole.
|
|
243
|
+
// Arbitrary structured tool output still follows the legacy JSON path.
|
|
244
|
+
if (Array.isArray(output) && output.length > 0
|
|
245
|
+
&& output.every(part => part && ['text', 'image', 'document'].includes(part.type))) {
|
|
246
|
+
const text = output.filter(part => part.type === 'text').map(part => part.text || '').join('\n');
|
|
247
|
+
return [
|
|
248
|
+
...(text ? [{ type: 'text', text: truncateToolResultIfNeeded(text, { toolName, language }) }] : []),
|
|
249
|
+
...output.filter(part => part.type !== 'text'),
|
|
250
|
+
];
|
|
251
|
+
}
|
|
241
252
|
const text = normalizeToolOutput(output);
|
|
242
253
|
const originalBytes = Buffer.byteLength(text, 'utf8');
|
|
243
254
|
if (originalBytes <= TOOL_RESULT_MAX_BYTES) return text;
|
|
@@ -123,8 +123,9 @@ export default defineTool({
|
|
|
123
123
|
en: `Load a local image file and attach it to the conversation so the LLM can see it.
|
|
124
124
|
|
|
125
125
|
Returns a base64 data URI (\`image\` field) plus metadata (format, dimensions,
|
|
126
|
-
size). The
|
|
127
|
-
|
|
126
|
+
size). The engine delivers the image as a native multimodal tool result.
|
|
127
|
+
Inspect the whole image first. Crop/zoom only details that are unreadable; OCR
|
|
128
|
+
is optional for text extraction, not a prerequisite for seeing the image.
|
|
128
129
|
|
|
129
130
|
When to call:
|
|
130
131
|
- User references a local image path (screenshot, design, log/chart) and
|
|
@@ -151,8 +152,8 @@ Max size: 20 MiB by default (configurable via ctx.maxImageBytes).
|
|
|
151
152
|
Path must live under the project directory or an explicit host allowlist.`,
|
|
152
153
|
zh: `加载本地图片文件并附加到对话中,使 LLM 可以查看。
|
|
153
154
|
|
|
154
|
-
返回 base64 data URI(image
|
|
155
|
-
|
|
155
|
+
返回 base64 data URI(image 字段)及元数据(格式、尺寸、大小)。引擎将图片作为原生多模态工具结果交给模型。
|
|
156
|
+
先直接查看整图;仅对确实看不清的细节裁剪/放大。OCR 用于需要时提取文字,不是看图的前置步骤。
|
|
156
157
|
|
|
157
158
|
何时调用:
|
|
158
159
|
- 用户引用本地图片路径(截图、设计稿、日志/图表)并要求你读取、分析或描述它。
|
package/yeaft/web-bridge.js
CHANGED
|
@@ -1454,12 +1454,15 @@ function projectPersistedToVisibleHistoryEntry(m) {
|
|
|
1454
1454
|
return entry && (entry.role === 'user' || entry.role === 'assistant') ? entry : null;
|
|
1455
1455
|
}
|
|
1456
1456
|
|
|
1457
|
-
function hydrateHistoryAttachmentPreviews(attachments) {
|
|
1457
|
+
function hydrateHistoryAttachmentPreviews(attachments, sessionId) {
|
|
1458
1458
|
if (!Array.isArray(attachments) || attachments.length === 0) return [];
|
|
1459
1459
|
return attachments.map((att) => {
|
|
1460
1460
|
if (!att || typeof att !== 'object') return att;
|
|
1461
1461
|
if (!att.isImage || att.preview || att.previewData) return att;
|
|
1462
|
-
const payload = persistedAttachmentPreviewPayload(att
|
|
1462
|
+
const payload = sessionId ? persistedAttachmentPreviewPayload(att, {
|
|
1463
|
+
yeaftDir: session?.yeaftDir || ctx.CONFIG?.yeaftDir || DEFAULT_YEAFT_DIR,
|
|
1464
|
+
sessionId,
|
|
1465
|
+
}) : null;
|
|
1463
1466
|
return payload ? { ...att, previewData: payload } : att;
|
|
1464
1467
|
});
|
|
1465
1468
|
}
|
|
@@ -1538,7 +1541,7 @@ function projectVisibleHistoryChunkMessages(messages = []) {
|
|
|
1538
1541
|
threadId: m.threadId || m.turnId || 'main',
|
|
1539
1542
|
...(m.turnId ? { turnId: m.turnId } : {}),
|
|
1540
1543
|
...(m.imageAssetAnchor === true ? { imageAssetAnchor: true } : {}),
|
|
1541
|
-
...(Array.isArray(m.attachments) && m.attachments.length > 0 ? { attachments: hydrateHistoryAttachmentPreviews(m.attachments) } : {}),
|
|
1544
|
+
...(Array.isArray(m.attachments) && m.attachments.length > 0 ? { attachments: hydrateHistoryAttachmentPreviews(m.attachments, m.sessionId) } : {}),
|
|
1542
1545
|
...(m.quote ? { quote: m.quote } : {}),
|
|
1543
1546
|
...(Array.isArray(m.images) && m.images.length > 0 ? { images: m.images } : {}),
|
|
1544
1547
|
...(m.speakerVpId ? { speakerVpId: m.speakerVpId } : {}),
|
|
@@ -1596,7 +1599,7 @@ function emitLegacyHistoryOutputFrames(replayEntries) {
|
|
|
1596
1599
|
message: {
|
|
1597
1600
|
content: entry.content,
|
|
1598
1601
|
id: entry.id || null,
|
|
1599
|
-
...(Array.isArray(entry.attachments) && entry.attachments.length > 0 ? { attachments: hydrateHistoryAttachmentPreviews(entry.attachments) } : {}),
|
|
1602
|
+
...(Array.isArray(entry.attachments) && entry.attachments.length > 0 ? { attachments: hydrateHistoryAttachmentPreviews(entry.attachments, entry.sessionId) } : {}),
|
|
1600
1603
|
...(entry.quote ? { quote: entry.quote } : {}),
|
|
1601
1604
|
},
|
|
1602
1605
|
ts: entry.ts || null,
|
|
@@ -4920,9 +4923,9 @@ async function runYeaftSessionSend(msg) {
|
|
|
4920
4923
|
// ── Attachments (images + files) ───────────────────────────────
|
|
4921
4924
|
// Server has already resolved fileId → { name, mimeType, data:base64,
|
|
4922
4925
|
// isImage } via the client-conversation.js relay
|
|
4923
|
-
// for `yeaft_*`).
|
|
4924
|
-
//
|
|
4925
|
-
//
|
|
4926
|
+
// for `yeaft_*`). Session files belong to the instance data root, not
|
|
4927
|
+
// the agent CWD or project workDir. Absolute controlled references work
|
|
4928
|
+
// with file-tools; per-image content blocks go directly to the model. The
|
|
4926
4929
|
// resolved metadata WITHOUT base64 rides on coord.ingest meta so it
|
|
4927
4930
|
// shows up in the persisted group log and on the envelope every VP
|
|
4928
4931
|
// driver receives.
|
|
@@ -4931,9 +4934,10 @@ async function runYeaftSessionSend(msg) {
|
|
|
4931
4934
|
const attachmentsStart = perfNowMs();
|
|
4932
4935
|
if (inboundFiles.length > 0) {
|
|
4933
4936
|
try {
|
|
4934
|
-
attachmentBundle = persistYeaftAttachments(inboundFiles, {
|
|
4937
|
+
attachmentBundle = persistYeaftAttachments(inboundFiles, { yeaftDir, sessionId });
|
|
4935
4938
|
} catch (err) {
|
|
4936
4939
|
console.warn('[Yeaft] yeaft_session_chat: attachment persist failed', err?.message || err);
|
|
4940
|
+
attachmentBundle.failed = inboundFiles.map(file => ({ name: file?.name || '<unknown>', error: err?.message || String(err) }));
|
|
4937
4941
|
}
|
|
4938
4942
|
}
|
|
4939
4943
|
// Surface partial / total upload failures to the user. We don't abort
|