@hybridaione/hybridclaw 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +47 -15
- package/container/package-lock.json +2 -2
- package/container/package.json +1 -1
- package/container/src/browser-tools.ts +1 -1
- package/container/src/index.ts +243 -14
- package/container/src/token-usage.ts +18 -2
- package/container/src/tools.ts +339 -1
- package/container/src/types.ts +28 -2
- package/dist/agent.d.ts +2 -2
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +2 -2
- package/dist/agent.js.map +1 -1
- package/dist/channels/discord/attachments.d.ts +9 -0
- package/dist/channels/discord/attachments.d.ts.map +1 -0
- package/dist/channels/discord/attachments.js +245 -0
- package/dist/channels/discord/attachments.js.map +1 -0
- package/dist/channels/discord/delivery.d.ts +31 -0
- package/dist/channels/discord/delivery.d.ts.map +1 -0
- package/dist/channels/discord/delivery.js +60 -0
- package/dist/channels/discord/delivery.js.map +1 -0
- package/dist/channels/discord/inbound.d.ts +20 -0
- package/dist/channels/discord/inbound.d.ts.map +1 -0
- package/dist/channels/discord/inbound.js +44 -0
- package/dist/channels/discord/inbound.js.map +1 -0
- package/dist/channels/discord/mentions.d.ts +14 -0
- package/dist/channels/discord/mentions.d.ts.map +1 -0
- package/dist/channels/discord/mentions.js +118 -0
- package/dist/channels/discord/mentions.js.map +1 -0
- package/dist/channels/discord/runtime.d.ts +22 -0
- package/dist/channels/discord/runtime.d.ts.map +1 -0
- package/dist/channels/discord/runtime.js +972 -0
- package/dist/channels/discord/runtime.js.map +1 -0
- package/dist/channels/discord/stream.d.ts +32 -0
- package/dist/channels/discord/stream.d.ts.map +1 -0
- package/dist/channels/discord/stream.js +196 -0
- package/dist/channels/discord/stream.js.map +1 -0
- package/dist/channels/discord/tool-actions.d.ts +31 -0
- package/dist/channels/discord/tool-actions.d.ts.map +1 -0
- package/dist/channels/discord/tool-actions.js +268 -0
- package/dist/channels/discord/tool-actions.js.map +1 -0
- package/dist/container-runner.d.ts +2 -2
- package/dist/container-runner.d.ts.map +1 -1
- package/dist/container-runner.js +12 -2
- package/dist/container-runner.js.map +1 -1
- package/dist/discord.basic.test.d.ts +2 -0
- package/dist/discord.basic.test.d.ts.map +1 -0
- package/dist/discord.basic.test.js +38 -0
- package/dist/discord.basic.test.js.map +1 -0
- package/dist/discord.d.ts +5 -44
- package/dist/discord.d.ts.map +1 -1
- package/dist/discord.js +3 -1468
- package/dist/discord.js.map +1 -1
- package/dist/gateway-service.d.ts +7 -1
- package/dist/gateway-service.d.ts.map +1 -1
- package/dist/gateway-service.js +111 -2
- package/dist/gateway-service.js.map +1 -1
- package/dist/gateway-service.media-routing.test.d.ts +2 -0
- package/dist/gateway-service.media-routing.test.d.ts.map +1 -0
- package/dist/gateway-service.media-routing.test.js +29 -0
- package/dist/gateway-service.media-routing.test.js.map +1 -0
- package/dist/gateway-types.d.ts +8 -0
- package/dist/gateway-types.d.ts.map +1 -1
- package/dist/gateway-types.js.map +1 -1
- package/dist/gateway.js +5 -2
- package/dist/gateway.js.map +1 -1
- package/dist/health.d.ts.map +1 -1
- package/dist/health.js +1 -1
- package/dist/health.js.map +1 -1
- package/dist/heartbeat.d.ts.map +1 -1
- package/dist/heartbeat.js +2 -0
- package/dist/heartbeat.js.map +1 -1
- package/dist/token-efficiency.basic.test.d.ts +2 -0
- package/dist/token-efficiency.basic.test.d.ts.map +1 -0
- package/dist/token-efficiency.basic.test.js +29 -0
- package/dist/token-efficiency.basic.test.js.map +1 -0
- package/dist/token-efficiency.d.ts.map +1 -1
- package/dist/token-efficiency.js +18 -1
- package/dist/token-efficiency.js.map +1 -1
- package/dist/types.d.ts +23 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +10 -2
- package/src/agent.ts +11 -1
- package/src/channels/discord/attachments.ts +282 -0
- package/src/channels/discord/delivery.ts +99 -0
- package/src/channels/discord/inbound.ts +72 -0
- package/src/channels/discord/mentions.ts +130 -0
- package/src/{discord.ts → channels/discord/runtime.ts} +77 -615
- package/src/{discord-stream.ts → channels/discord/stream.ts} +2 -2
- package/src/channels/discord/tool-actions.ts +332 -0
- package/src/container-runner.ts +24 -1
- package/src/gateway-service.ts +125 -1
- package/src/gateway-types.ts +8 -0
- package/src/gateway.ts +5 -5
- package/src/health.ts +2 -1
- package/src/heartbeat.ts +2 -0
- package/src/token-efficiency.ts +17 -1
- package/src/types.ts +27 -1
- package/tests/discord.basic.test.ts +43 -0
- package/tests/gateway-service.media-routing.test.ts +33 -0
- package/tests/token-efficiency.basic.test.ts +32 -0
- package/vitest.e2e.config.ts +15 -0
- package/vitest.integration.config.ts +15 -0
- package/vitest.live.config.ts +16 -0
- package/vitest.unit.config.ts +15 -0
package/container/src/tools.ts
CHANGED
|
@@ -3,7 +3,13 @@ import fs from 'fs';
|
|
|
3
3
|
import path from 'path';
|
|
4
4
|
|
|
5
5
|
import { BROWSER_TOOL_DEFINITIONS, executeBrowserTool, setBrowserModelContext } from './browser-tools.js';
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
DelegationSideEffect,
|
|
8
|
+
DelegationTaskSpec,
|
|
9
|
+
MediaContextItem,
|
|
10
|
+
ScheduleSideEffect,
|
|
11
|
+
ToolDefinition,
|
|
12
|
+
} from './types.js';
|
|
7
13
|
import { webFetch } from './web-fetch.js';
|
|
8
14
|
|
|
9
15
|
// --- Exec safety deny-list (defense-in-depth, adapted from PicoClaw) ---
|
|
@@ -50,8 +56,22 @@ let currentSessionId = '';
|
|
|
50
56
|
let gatewayBaseUrl = '';
|
|
51
57
|
let gatewayApiToken = '';
|
|
52
58
|
let gatewayChannelId = '';
|
|
59
|
+
let currentModelBaseUrl = '';
|
|
60
|
+
let currentModelApiKey = '';
|
|
61
|
+
let currentModelName = '';
|
|
62
|
+
let currentChatbotId = '';
|
|
63
|
+
let currentMediaContext: MediaContextItem[] = [];
|
|
53
64
|
const MAX_PENDING_DELEGATIONS = 3;
|
|
54
65
|
const MAX_DELEGATION_BATCH_ITEMS = 6;
|
|
66
|
+
const DISCORD_MEDIA_CACHE_ROOT = '/discord-media-cache';
|
|
67
|
+
const VISION_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
68
|
+
const VISION_FETCH_TIMEOUT_MS = 12_000;
|
|
69
|
+
const DISCORD_CDN_HOST_PATTERNS: RegExp[] = [
|
|
70
|
+
/^cdn\.discordapp\.com$/i,
|
|
71
|
+
/^media\.discordapp\.net$/i,
|
|
72
|
+
/^cdn\.discordapp\.net$/i,
|
|
73
|
+
/^images-ext-\d+\.discordapp\.net$/i,
|
|
74
|
+
];
|
|
55
75
|
|
|
56
76
|
type DiscordMessageToolAction = 'read' | 'member-info' | 'channel-info';
|
|
57
77
|
|
|
@@ -91,9 +111,17 @@ export function setModelContext(
|
|
|
91
111
|
model: string,
|
|
92
112
|
chatbotId: string,
|
|
93
113
|
): void {
|
|
114
|
+
currentModelBaseUrl = String(baseUrl || '').trim();
|
|
115
|
+
currentModelApiKey = String(apiKey || '').trim();
|
|
116
|
+
currentModelName = String(model || '').trim();
|
|
117
|
+
currentChatbotId = String(chatbotId || '').trim();
|
|
94
118
|
setBrowserModelContext(baseUrl, apiKey, model, chatbotId);
|
|
95
119
|
}
|
|
96
120
|
|
|
121
|
+
export function setMediaContext(media?: MediaContextItem[]): void {
|
|
122
|
+
currentMediaContext = Array.isArray(media) ? media : [];
|
|
123
|
+
}
|
|
124
|
+
|
|
97
125
|
function readStringValue(value: unknown): string | undefined {
|
|
98
126
|
if (typeof value !== 'string') return undefined;
|
|
99
127
|
const trimmed = value.trim();
|
|
@@ -216,6 +244,251 @@ function normalizeDelegationTaskList(params: {
|
|
|
216
244
|
return { tasks };
|
|
217
245
|
}
|
|
218
246
|
|
|
247
|
+
function asRecord(value: unknown): Record<string, unknown> | null {
|
|
248
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
|
|
249
|
+
return value as Record<string, unknown>;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function extractVisionTextContent(content: unknown): string {
|
|
253
|
+
if (typeof content === 'string') return content.trim();
|
|
254
|
+
if (!Array.isArray(content)) return '';
|
|
255
|
+
const chunks: string[] = [];
|
|
256
|
+
for (const part of content) {
|
|
257
|
+
if (typeof part === 'string') {
|
|
258
|
+
if (part.trim()) chunks.push(part.trim());
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
const obj = asRecord(part);
|
|
262
|
+
if (!obj) continue;
|
|
263
|
+
const text = typeof obj.text === 'string' ? obj.text : '';
|
|
264
|
+
if (text.trim()) chunks.push(text.trim());
|
|
265
|
+
}
|
|
266
|
+
return chunks.join('\n').trim();
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function isSafeDiscordCdnUrl(raw: string): boolean {
|
|
270
|
+
let parsed: URL;
|
|
271
|
+
try {
|
|
272
|
+
parsed = new URL(raw);
|
|
273
|
+
} catch {
|
|
274
|
+
return false;
|
|
275
|
+
}
|
|
276
|
+
if (parsed.protocol !== 'https:') return false;
|
|
277
|
+
return DISCORD_CDN_HOST_PATTERNS.some((pattern) => pattern.test(parsed.hostname));
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function normalizeVisionLocalPath(rawPath: string): string | null {
|
|
281
|
+
const trimmed = rawPath.trim();
|
|
282
|
+
if (!trimmed) return null;
|
|
283
|
+
|
|
284
|
+
const normalizedInput = trimmed.replace(/\\/g, '/');
|
|
285
|
+
const candidate = normalizedInput.startsWith('/')
|
|
286
|
+
? path.posix.normalize(normalizedInput)
|
|
287
|
+
: path.posix.normalize(path.posix.join(WORKSPACE_ROOT, normalizedInput));
|
|
288
|
+
if (
|
|
289
|
+
!(candidate === WORKSPACE_ROOT || candidate.startsWith(`${WORKSPACE_ROOT}/`))
|
|
290
|
+
&& !(candidate === DISCORD_MEDIA_CACHE_ROOT || candidate.startsWith(`${DISCORD_MEDIA_CACHE_ROOT}/`))
|
|
291
|
+
) {
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
294
|
+
return candidate;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function isKnownDiscordMediaPath(localPath: string): boolean {
|
|
298
|
+
const knownPaths = currentMediaContext
|
|
299
|
+
.map((entry) => (typeof entry.path === 'string' ? entry.path.trim() : ''))
|
|
300
|
+
.filter(Boolean)
|
|
301
|
+
.map((entryPath) => normalizeVisionLocalPath(entryPath))
|
|
302
|
+
.filter((value): value is string => Boolean(value));
|
|
303
|
+
if (knownPaths.length === 0) return true;
|
|
304
|
+
return knownPaths.includes(localPath);
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function inferImageMimeTypeFromPath(localPath: string, fallbackMime?: string | null): string {
|
|
308
|
+
const normalizedFallback = String(fallbackMime || '').trim().toLowerCase();
|
|
309
|
+
if (normalizedFallback.startsWith('image/')) return normalizedFallback;
|
|
310
|
+
const ext = path.posix.extname(localPath).toLowerCase();
|
|
311
|
+
if (ext === '.png') return 'image/png';
|
|
312
|
+
if (ext === '.jpg' || ext === '.jpeg') return 'image/jpeg';
|
|
313
|
+
if (ext === '.gif') return 'image/gif';
|
|
314
|
+
if (ext === '.webp') return 'image/webp';
|
|
315
|
+
if (ext === '.bmp') return 'image/bmp';
|
|
316
|
+
if (ext === '.svg') return 'image/svg+xml';
|
|
317
|
+
if (ext === '.tif' || ext === '.tiff') return 'image/tiff';
|
|
318
|
+
return 'image/png';
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function readVisionImageFromLocalPath(localPath: string): Promise<{ buffer: Buffer; mimeType: string; source: string }> {
|
|
322
|
+
const normalizedPath = normalizeVisionLocalPath(localPath);
|
|
323
|
+
if (!normalizedPath) {
|
|
324
|
+
throw new Error('local image path must be under /workspace or /discord-media-cache');
|
|
325
|
+
}
|
|
326
|
+
if (normalizedPath.startsWith(`${DISCORD_MEDIA_CACHE_ROOT}/`) && !isKnownDiscordMediaPath(normalizedPath)) {
|
|
327
|
+
throw new Error('requested local image is not part of current media context');
|
|
328
|
+
}
|
|
329
|
+
if (!fs.existsSync(normalizedPath)) {
|
|
330
|
+
throw new Error(`local image not found: ${normalizedPath}`);
|
|
331
|
+
}
|
|
332
|
+
const stat = fs.statSync(normalizedPath);
|
|
333
|
+
if (!stat.isFile()) {
|
|
334
|
+
throw new Error(`local image path is not a file: ${normalizedPath}`);
|
|
335
|
+
}
|
|
336
|
+
if (stat.size <= 0) {
|
|
337
|
+
throw new Error(`local image is empty: ${normalizedPath}`);
|
|
338
|
+
}
|
|
339
|
+
if (stat.size > VISION_IMAGE_MAX_BYTES) {
|
|
340
|
+
throw new Error(`local image exceeds max size (${VISION_IMAGE_MAX_BYTES} bytes)`);
|
|
341
|
+
}
|
|
342
|
+
const buffer = fs.readFileSync(normalizedPath);
|
|
343
|
+
const mediaHint = currentMediaContext.find((entry) => {
|
|
344
|
+
const normalizedEntryPath = entry.path ? normalizeVisionLocalPath(entry.path) : null;
|
|
345
|
+
return normalizedEntryPath === normalizedPath;
|
|
346
|
+
});
|
|
347
|
+
const mimeType = inferImageMimeTypeFromPath(normalizedPath, mediaHint?.mimeType);
|
|
348
|
+
if (!mimeType.startsWith('image/')) {
|
|
349
|
+
throw new Error(`unsupported local image type: ${mimeType}`);
|
|
350
|
+
}
|
|
351
|
+
return {
|
|
352
|
+
buffer,
|
|
353
|
+
mimeType,
|
|
354
|
+
source: normalizedPath,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
async function readVisionImageFromUrl(rawUrl: string): Promise<{ buffer: Buffer; mimeType: string; source: string }> {
|
|
359
|
+
if (!isSafeDiscordCdnUrl(rawUrl)) {
|
|
360
|
+
throw new Error('remote image URL is blocked (only Discord CDN HTTPS URLs are allowed)');
|
|
361
|
+
}
|
|
362
|
+
const controller = new AbortController();
|
|
363
|
+
const timer = setTimeout(() => controller.abort(), VISION_FETCH_TIMEOUT_MS);
|
|
364
|
+
try {
|
|
365
|
+
const response = await fetch(rawUrl, { signal: controller.signal });
|
|
366
|
+
if (!response.ok) {
|
|
367
|
+
throw new Error(`image download failed (${response.status})`);
|
|
368
|
+
}
|
|
369
|
+
const mimeType = String(response.headers.get('content-type') || '')
|
|
370
|
+
.split(';')[0]
|
|
371
|
+
.trim()
|
|
372
|
+
.toLowerCase();
|
|
373
|
+
if (!mimeType.startsWith('image/')) {
|
|
374
|
+
throw new Error(`remote URL is not an image (${mimeType || 'unknown'})`);
|
|
375
|
+
}
|
|
376
|
+
const contentLength = Number.parseInt(response.headers.get('content-length') || '', 10);
|
|
377
|
+
if (Number.isFinite(contentLength) && contentLength > VISION_IMAGE_MAX_BYTES) {
|
|
378
|
+
throw new Error(`remote image exceeds max size (${VISION_IMAGE_MAX_BYTES} bytes)`);
|
|
379
|
+
}
|
|
380
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
381
|
+
if (buffer.length > VISION_IMAGE_MAX_BYTES) {
|
|
382
|
+
throw new Error(`remote image exceeds max size (${VISION_IMAGE_MAX_BYTES} bytes)`);
|
|
383
|
+
}
|
|
384
|
+
return {
|
|
385
|
+
buffer,
|
|
386
|
+
mimeType,
|
|
387
|
+
source: rawUrl,
|
|
388
|
+
};
|
|
389
|
+
} finally {
|
|
390
|
+
clearTimeout(timer);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function visionModelContextError(): string | null {
|
|
395
|
+
if (!currentModelApiKey) return 'vision_analyze is not configured: missing API key context.';
|
|
396
|
+
if (!currentModelBaseUrl) return 'vision_analyze is not configured: missing base URL context.';
|
|
397
|
+
if (!currentModelName) return 'vision_analyze is not configured: missing model context.';
|
|
398
|
+
if (!currentChatbotId) return 'vision_analyze is not configured: missing chatbot_id context.';
|
|
399
|
+
return null;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function callVisionModel(question: string, imageDataUrl: string): Promise<{ model: string; analysis: string }> {
|
|
403
|
+
const contextError = visionModelContextError();
|
|
404
|
+
if (contextError) throw new Error(contextError);
|
|
405
|
+
|
|
406
|
+
const response = await fetch(`${currentModelBaseUrl}/v1/chat/completions`, {
|
|
407
|
+
method: 'POST',
|
|
408
|
+
headers: {
|
|
409
|
+
'Content-Type': 'application/json',
|
|
410
|
+
Authorization: `Bearer ${currentModelApiKey}`,
|
|
411
|
+
},
|
|
412
|
+
body: JSON.stringify({
|
|
413
|
+
model: currentModelName,
|
|
414
|
+
chatbot_id: currentChatbotId,
|
|
415
|
+
enable_rag: false,
|
|
416
|
+
messages: [
|
|
417
|
+
{
|
|
418
|
+
role: 'user',
|
|
419
|
+
content: [
|
|
420
|
+
{ type: 'text', text: question },
|
|
421
|
+
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
|
422
|
+
],
|
|
423
|
+
},
|
|
424
|
+
],
|
|
425
|
+
}),
|
|
426
|
+
});
|
|
427
|
+
|
|
428
|
+
const rawText = await response.text();
|
|
429
|
+
if (!response.ok) {
|
|
430
|
+
const detail = rawText.length > 600 ? `${rawText.slice(0, 600)}...` : rawText;
|
|
431
|
+
throw new Error(`vision API request failed (${response.status}): ${detail}`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
let parsed: unknown;
|
|
435
|
+
try {
|
|
436
|
+
parsed = JSON.parse(rawText);
|
|
437
|
+
} catch {
|
|
438
|
+
throw new Error('vision API returned non-JSON response');
|
|
439
|
+
}
|
|
440
|
+
const record = asRecord(parsed);
|
|
441
|
+
const choices = record?.choices;
|
|
442
|
+
if (!Array.isArray(choices) || choices.length === 0) {
|
|
443
|
+
throw new Error('vision API response did not include choices');
|
|
444
|
+
}
|
|
445
|
+
const firstChoice = asRecord(choices[0]);
|
|
446
|
+
const message = asRecord(firstChoice?.message);
|
|
447
|
+
const analysis = extractVisionTextContent(message?.content);
|
|
448
|
+
if (!analysis) {
|
|
449
|
+
throw new Error('vision API returned empty analysis');
|
|
450
|
+
}
|
|
451
|
+
return {
|
|
452
|
+
model: currentModelName,
|
|
453
|
+
analysis,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
async function runVisionAnalyze(args: Record<string, unknown>): Promise<string> {
|
|
458
|
+
const question = readStringValue(args.question);
|
|
459
|
+
if (!question) return 'Error: question is required';
|
|
460
|
+
|
|
461
|
+
const imageRef = readStringValue(args.image_url) || readStringValue(args.imageUrl) || readStringValue(args.path);
|
|
462
|
+
const fallbackUrl = readStringValue(args.fallback_url) || readStringValue(args.fallbackUrl) || readStringValue(args.original_url);
|
|
463
|
+
if (!imageRef) return 'Error: image_url is required';
|
|
464
|
+
|
|
465
|
+
const candidates = [imageRef, fallbackUrl].filter((value): value is string => Boolean(value));
|
|
466
|
+
const errors: string[] = [];
|
|
467
|
+
for (const candidate of candidates) {
|
|
468
|
+
try {
|
|
469
|
+
const isRemote = /^https?:\/\//i.test(candidate);
|
|
470
|
+
const image = isRemote
|
|
471
|
+
? await readVisionImageFromUrl(candidate)
|
|
472
|
+
: await readVisionImageFromLocalPath(candidate);
|
|
473
|
+
const dataUrl = `data:${image.mimeType};base64,${image.buffer.toString('base64')}`;
|
|
474
|
+
const vision = await callVisionModel(question, dataUrl);
|
|
475
|
+
return JSON.stringify({
|
|
476
|
+
success: true,
|
|
477
|
+
model: vision.model,
|
|
478
|
+
analysis: vision.analysis,
|
|
479
|
+
source: image.source,
|
|
480
|
+
mime_type: image.mimeType,
|
|
481
|
+
size_bytes: image.buffer.length,
|
|
482
|
+
}, null, 2);
|
|
483
|
+
} catch (err) {
|
|
484
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
485
|
+
errors.push(`${candidate}: ${detail}`);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
return `Error: vision_analyze failed. ${errors.join(' | ') || 'No candidate image sources succeeded.'}`;
|
|
490
|
+
}
|
|
491
|
+
|
|
219
492
|
const PREVIEW_MAX_OUTPUT_LINES = 6;
|
|
220
493
|
const PREVIEW_MAX_LINE_LENGTH = 200;
|
|
221
494
|
const BASH_MAX_OUTPUT_LINES = 400;
|
|
@@ -1029,6 +1302,11 @@ export async function executeTool(name: string, argsJson: string): Promise<strin
|
|
|
1029
1302
|
return `${lines.join('\n')}\n\n${header}${result.text}`;
|
|
1030
1303
|
}
|
|
1031
1304
|
|
|
1305
|
+
case 'vision_analyze':
|
|
1306
|
+
case 'image': {
|
|
1307
|
+
return await runVisionAnalyze(args);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1032
1310
|
case 'browser_navigate':
|
|
1033
1311
|
case 'browser_snapshot':
|
|
1034
1312
|
case 'browser_click':
|
|
@@ -1424,6 +1702,66 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
|
|
|
1424
1702
|
},
|
|
1425
1703
|
},
|
|
1426
1704
|
},
|
|
1705
|
+
{
|
|
1706
|
+
type: 'function',
|
|
1707
|
+
function: {
|
|
1708
|
+
name: 'vision_analyze',
|
|
1709
|
+
description:
|
|
1710
|
+
'Analyze an image attachment using vision. Use for Discord-uploaded files (local /discord-media-cache paths first, Discord CDN URLs as fallback).',
|
|
1711
|
+
parameters: {
|
|
1712
|
+
type: 'object',
|
|
1713
|
+
properties: {
|
|
1714
|
+
image_url: {
|
|
1715
|
+
type: 'string',
|
|
1716
|
+
description: 'Local image path (preferred) or Discord CDN HTTPS URL.',
|
|
1717
|
+
},
|
|
1718
|
+
question: {
|
|
1719
|
+
type: 'string',
|
|
1720
|
+
description: 'Question to ask about the image.',
|
|
1721
|
+
},
|
|
1722
|
+
fallback_url: {
|
|
1723
|
+
type: 'string',
|
|
1724
|
+
description: 'Optional fallback Discord CDN URL if image_url cannot be read.',
|
|
1725
|
+
},
|
|
1726
|
+
original_url: {
|
|
1727
|
+
type: 'string',
|
|
1728
|
+
description: 'Optional original URL alias for fallback_url.',
|
|
1729
|
+
},
|
|
1730
|
+
},
|
|
1731
|
+
required: ['image_url', 'question'],
|
|
1732
|
+
},
|
|
1733
|
+
},
|
|
1734
|
+
},
|
|
1735
|
+
{
|
|
1736
|
+
type: 'function',
|
|
1737
|
+
function: {
|
|
1738
|
+
name: 'image',
|
|
1739
|
+
description:
|
|
1740
|
+
'Alias of vision_analyze for image analysis.',
|
|
1741
|
+
parameters: {
|
|
1742
|
+
type: 'object',
|
|
1743
|
+
properties: {
|
|
1744
|
+
image_url: {
|
|
1745
|
+
type: 'string',
|
|
1746
|
+
description: 'Local image path (preferred) or Discord CDN HTTPS URL.',
|
|
1747
|
+
},
|
|
1748
|
+
question: {
|
|
1749
|
+
type: 'string',
|
|
1750
|
+
description: 'Question to ask about the image.',
|
|
1751
|
+
},
|
|
1752
|
+
fallback_url: {
|
|
1753
|
+
type: 'string',
|
|
1754
|
+
description: 'Optional fallback Discord CDN URL if image_url cannot be read.',
|
|
1755
|
+
},
|
|
1756
|
+
original_url: {
|
|
1757
|
+
type: 'string',
|
|
1758
|
+
description: 'Optional original URL alias for fallback_url.',
|
|
1759
|
+
},
|
|
1760
|
+
},
|
|
1761
|
+
required: ['image_url', 'question'],
|
|
1762
|
+
},
|
|
1763
|
+
},
|
|
1764
|
+
},
|
|
1427
1765
|
...BROWSER_TOOL_DEFINITIONS,
|
|
1428
1766
|
{
|
|
1429
1767
|
type: 'function',
|
package/container/src/types.ts
CHANGED
|
@@ -1,6 +1,21 @@
|
|
|
1
|
+
export interface ChatContentTextPart {
|
|
2
|
+
type: 'text';
|
|
3
|
+
text: string;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export interface ChatContentImageUrlPart {
|
|
7
|
+
type: 'image_url';
|
|
8
|
+
image_url: {
|
|
9
|
+
url: string;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export type ChatContentPart = ChatContentTextPart | ChatContentImageUrlPart;
|
|
14
|
+
export type ChatMessageContent = string | ChatContentPart[] | null;
|
|
15
|
+
|
|
1
16
|
export interface ChatMessage {
|
|
2
17
|
role: 'system' | 'user' | 'assistant' | 'tool';
|
|
3
|
-
content:
|
|
18
|
+
content: ChatMessageContent;
|
|
4
19
|
tool_calls?: ToolCall[];
|
|
5
20
|
tool_call_id?: string;
|
|
6
21
|
}
|
|
@@ -19,7 +34,7 @@ export interface ChatCompletionResponse {
|
|
|
19
34
|
choices: Array<{
|
|
20
35
|
message: {
|
|
21
36
|
role: string;
|
|
22
|
-
content:
|
|
37
|
+
content: ChatMessageContent;
|
|
23
38
|
tool_calls?: ToolCall[];
|
|
24
39
|
};
|
|
25
40
|
finish_reason: string;
|
|
@@ -73,6 +88,17 @@ export interface ContainerInput {
|
|
|
73
88
|
channelId: string;
|
|
74
89
|
scheduledTasks?: { id: number; cronExpr: string; runAt: string | null; everyMs: number | null; prompt: string; enabled: number; lastRun: string | null; createdAt: string }[];
|
|
75
90
|
allowedTools?: string[];
|
|
91
|
+
blockedTools?: string[];
|
|
92
|
+
media?: MediaContextItem[];
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface MediaContextItem {
|
|
96
|
+
path: string | null;
|
|
97
|
+
url: string;
|
|
98
|
+
originalUrl: string;
|
|
99
|
+
mimeType: string | null;
|
|
100
|
+
sizeBytes: number;
|
|
101
|
+
filename: string;
|
|
76
102
|
}
|
|
77
103
|
|
|
78
104
|
export interface ToolExecution {
|
package/dist/agent.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type { ChatMessage, ContainerOutput, ScheduledTask, ToolProgressEvent } from './types.js';
|
|
2
|
-
export declare function runAgent(sessionId: string, messages: ChatMessage[], chatbotId: string, enableRag: boolean, model: string, agentId: string, channelId: string, scheduledTasks?: ScheduledTask[], allowedTools?: string[], onTextDelta?: (delta: string) => void, onToolProgress?: (event: ToolProgressEvent) => void, abortSignal?: AbortSignal): Promise<ContainerOutput>;
|
|
1
|
+
import type { ChatMessage, ContainerOutput, MediaContextItem, ScheduledTask, ToolProgressEvent } from './types.js';
|
|
2
|
+
export declare function runAgent(sessionId: string, messages: ChatMessage[], chatbotId: string, enableRag: boolean, model: string, agentId: string, channelId: string, scheduledTasks?: ScheduledTask[], allowedTools?: string[], blockedTools?: string[], onTextDelta?: (delta: string) => void, onToolProgress?: (event: ToolProgressEvent) => void, abortSignal?: AbortSignal, media?: MediaContextItem[]): Promise<ContainerOutput>;
|
|
3
3
|
//# sourceMappingURL=agent.d.ts.map
|
package/dist/agent.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAWpB,wBAAsB,QAAQ,CAC5B,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,WAAW,EAAE,EACvB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,OAAO,EAClB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,cAAc,CAAC,EAAE,aAAa,EAAE,EAChC,YAAY,CAAC,EAAE,MAAM,EAAE,EACvB,YAAY,CAAC,EAAE,MAAM,EAAE,EACvB,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,EACrC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,EACnD,WAAW,CAAC,EAAE,WAAW,EACzB,KAAK,CAAC,EAAE,gBAAgB,EAAE,GACzB,OAAO,CAAC,eAAe,CAAC,CAkB1B"}
|
package/dist/agent.js
CHANGED
|
@@ -14,8 +14,8 @@ function dumpPrompt(sessionId, messages, model, chatbotId) {
|
|
|
14
14
|
}
|
|
15
15
|
catch { /* best-effort */ }
|
|
16
16
|
}
|
|
17
|
-
export async function runAgent(sessionId, messages, chatbotId, enableRag, model, agentId, channelId, scheduledTasks, allowedTools, onTextDelta, onToolProgress, abortSignal) {
|
|
17
|
+
export async function runAgent(sessionId, messages, chatbotId, enableRag, model, agentId, channelId, scheduledTasks, allowedTools, blockedTools, onTextDelta, onToolProgress, abortSignal, media) {
|
|
18
18
|
dumpPrompt(sessionId, messages, model, chatbotId);
|
|
19
|
-
return runContainer(sessionId, messages, chatbotId, enableRag, model, agentId, channelId, scheduledTasks, allowedTools, onTextDelta, onToolProgress, abortSignal);
|
|
19
|
+
return runContainer(sessionId, messages, chatbotId, enableRag, model, agentId, channelId, scheduledTasks, allowedTools, blockedTools, onTextDelta, onToolProgress, abortSignal, media);
|
|
20
20
|
}
|
|
21
21
|
//# sourceMappingURL=agent.js.map
|
package/dist/agent.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.js","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"agent.js","sourceRoot":"","sources":["../src/agent.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AASrD,yFAAyF;AACzF,SAAS,UAAU,CAAC,SAAiB,EAAE,QAAuB,EAAE,KAAa,EAAE,SAAiB;IAC9F,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAC;QAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,SAAiB,EACjB,QAAuB,EACvB,SAAiB,EACjB,SAAkB,EAClB,KAAa,EACb,OAAe,EACf,SAAiB,EACjB,cAAgC,EAChC,YAAuB,EACvB,YAAuB,EACvB,WAAqC,EACrC,cAAmD,EACnD,WAAyB,EACzB,KAA0B;IAE1B,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAClD,OAAO,YAAY,CACjB,SAAS,EACT,QAAQ,EACR,SAAS,EACT,SAAS,EACT,KAAK,EACL,OAAO,EACP,SAAS,EACT,cAAc,EACd,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,WAAW,EACX,KAAK,CACN,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Message as DiscordMessage } from 'discord.js';
|
|
2
|
+
import type { MediaContextItem } from '../../types.js';
|
|
3
|
+
export interface AttachmentContextResult {
|
|
4
|
+
context: string;
|
|
5
|
+
media: MediaContextItem[];
|
|
6
|
+
}
|
|
7
|
+
export declare function looksLikeTextAttachment(name: string, contentType: string): boolean;
|
|
8
|
+
export declare function buildAttachmentContext(messages: DiscordMessage[]): Promise<AttachmentContextResult>;
|
|
9
|
+
//# sourceMappingURL=attachments.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"attachments.d.ts","sourceRoot":"","sources":["../../../src/channels/discord/attachments.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAmC,OAAO,IAAI,cAAc,EAAE,MAAM,YAAY,CAAC;AAI7F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAevD,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAIlF;AAwID,wBAAsB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAiHzG"}
|