@ai-zen/agents-sdk 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +10 -7
- package/README.zh.md +10 -7
- package/dist/capabilities/discovery/builtin.d.ts +3 -3
- package/dist/capabilities/discovery/builtin.js +4 -12
- package/dist/capabilities/implements/builtin/GenerateImageTool.d.ts +8 -8
- package/dist/capabilities/implements/builtin/GenerateImageTool.js +26 -27
- package/dist/capabilities/implements/builtin/ViewImageTool.d.ts +24 -0
- package/dist/capabilities/implements/builtin/ViewImageTool.js +109 -0
- package/dist/capabilities/implements/builtin/index.d.ts +6 -4
- package/dist/capabilities/implements/builtin/index.js +8 -4
- package/dist/capabilities/implements/skillTools.js +5 -5
- package/dist/capabilities/implements/subAgentTools.js +17 -8
- package/dist/config/constants.js +12 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/plugin/AutoMigratePlugin.d.ts +1 -1
- package/dist/plugin/AutoMigratePlugin.js +1 -1
- package/dist/plugin/AutoRefreshToolsPlugin.d.ts +1 -1
- package/dist/plugin/AutoRefreshToolsPlugin.js +2 -2
- package/dist/plugin/ContextGuardPlugin.d.ts +2 -2
- package/dist/plugin/ContextGuardPlugin.js +1 -1
- package/dist/plugin/UnknownToolHintPlugin.d.ts +21 -0
- package/dist/plugin/UnknownToolHintPlugin.js +30 -0
- package/dist/runtime/Provider.d.ts +8 -7
- package/dist/runtime/Provider.js +14 -4
- package/dist/runtime/SdkAgent.d.ts +14 -107
- package/dist/runtime/SdkAgent.js +8 -120
- package/dist/runtime/SdkCallbackTool.d.ts +12 -4
- package/dist/runtime/SdkCallbackTool.js +5 -2
- package/dist/runtime/createAgent.js +4 -3
- package/dist/runtime/createModel.d.ts +17 -4
- package/dist/runtime/createModel.js +15 -11
- package/dist/types/index.d.ts +2 -0
- package/package.json +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.7.0] - 2026-08-14
|
|
4
|
+
|
|
5
|
+
### ⚠️ Breaking
|
|
6
|
+
|
|
7
|
+
- **`@ai-zen/agents-core` upgraded to 4.0.0** — the SDK now runs on the **official OpenAI SDK**: `createModel(provider, modelId)` returns `{ client, model, modelConfig }` (an `openai` client + model name + params spread into the request) instead of a `ChatCompletionModel`; `SdkAgent` / `createAgent` construct with `{ client, model, modelConfig }`; `GenerateImageTool` calls `client.images.generate`
|
|
8
|
+
- **Plugin mechanism promoted to core** — `AgentPlugin` / `SendContext` / `HookResult` now live in `@ai-zen/agents-core` (re-exported by the SDK for compatibility). `SdkAgent` no longer implements its own `use` / `init` / hook iteration, and the `onUnknownTool` property hack is gone — it inherits everything from Core `Agent`
|
|
9
|
+
- **`SdkAgent.defaultUnknownTool` override removed** — the MCP-aware unknown-tool hint moved out of the class into a standalone plugin (see `UnknownToolHintPlugin` below); Core `Agent` now keeps only a simple built-in hint
|
|
10
|
+
- **`SdkAgent.permissions` field removed** — the Agent no longer carries its own `permissions` copy; read `agent.definition.permissions` instead (single source of truth). `AutoRefreshToolsPlugin` and `call_skill_sub_agent` were updated accordingly
|
|
11
|
+
- **`ContextGuardPlugin` default `ratio` changed from 1.2 to 1.5** — the guard now trips at 50% over `maxTokens` (user setting) instead of 20%
|
|
12
|
+
|
|
13
|
+
### 🚀 New Features
|
|
14
|
+
|
|
15
|
+
- **`UnknownToolHintPlugin`** — new standalone plugin providing the MCP-aware unknown-tool hint (replaces the removed `SdkAgent.defaultUnknownTool` override). Register explicitly: `agent.use(new UnknownToolHintPlugin({ provider }))`; without it, unknown tools fall back to Core's simple built-in hint
|
|
16
|
+
- **`Model.vision` field + self-declared tool availability** — models can be marked `vision: true`. `SdkCallbackTool` gains `isAvailable(config, definition)`: tools **self-declare** availability by directly inspecting the full app config and agent definition (e.g. `GenerateImageTool` requires `defaultImageModel`, `ViewImageTool` requires a vision model). All **19 built-in tools** live in `BUILTIN_TOOL_CLASSES` (discovery does no filtering); `Provider.filter`/`buildTools` take the `AgentDefinition` and call `isAvailable` at build time (when the model is already known) to filter the final tool set — no hard-coded tool names, and no runtime duplication (vision checks live only in `ViewImageTool.isAvailable`)
|
|
17
|
+
- **`ViewImageTool`** — let the Agent view/analyze an image (vision models only): a network URL returns an `image_url` content block; a local path is auto-uploaded via the Files API (`client.files.create`, `purpose: "user_data"`) and returned as a `file` content block (no base64). Always registered as a built-in candidate; filtered out by `buildTools` for non-vision models (a runtime check remains as a defensive guard)
|
|
18
|
+
- **`GenerateImageTool` returns a plain string** — it always returns JSON text with the image URLs plus `viewImage` / `downloadFile` hints; how the generated image is consumed (view / download / forward) is left to the model and the caller, instead of forcing image content blocks on vision models
|
|
19
|
+
- **`SdkCallbackTool.exec` passes content-section arrays through** — aligned with Core's `Tool.exec` returning `AgentNS.MessageContent`; other values are still JSON-serialized
|
|
20
|
+
|
|
21
|
+
### 🎯 Optimized
|
|
22
|
+
|
|
23
|
+
- `createAgent` / `subAgentTools` / `skillTools` adapted to the new Agent constructor (`{ client, model, modelConfig }`)
|
|
24
|
+
- The plugins (`AutoMigratePlugin` / `AutoRefreshToolsPlugin` / `ContextGuardPlugin` / `UnknownToolHintPlugin`) now type their hooks from core's `AgentPlugin` / `HookResult`
|
|
25
|
+
|
|
26
|
+
### ✅ Tests
|
|
27
|
+
|
|
28
|
+
- Adapted `createModel` / `SdkAgent` / `Provider` / `createAgent` / `AutoRefreshToolsPlugin` / `ContextGuardPlugin` / `skillTools` / e2e tests to the core 4.0.0 API — all green (incl. real DeepSeek API e2e)
|
|
29
|
+
|
|
3
30
|
## [0.6.0] - 2026-08-14
|
|
4
31
|
|
|
5
32
|
### ⚠️ Breaking
|
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ config ← read/write config.json + migration + in-memory cache + atomic w
|
|
|
23
23
|
crud ← capability-entity CRUD (Agent definitions, etc.; conversations/drafts are persisted by each consumer)
|
|
24
24
|
capabilities ← capability discovery & assembly (built-in + user + MCP + Skill + SubAgent)
|
|
25
25
|
runtime ← Provider + model factory + Agent assembly + MCP connection management + task migration + SdkCallbackTool base
|
|
26
|
-
plugin ← Agent plugins (autoMigrate, autoRefreshTools)
|
|
26
|
+
plugin ← Agent plugins (autoMigrate, autoRefreshTools, contextGuard, unknownToolHint)
|
|
27
27
|
shared ← logging, errors
|
|
28
28
|
```
|
|
29
29
|
|
|
@@ -78,9 +78,9 @@ await agent.send("Hello");
|
|
|
78
78
|
| `crud` | ✅ Implemented — capability-entity CRUD for Agents, etc. (conversations/drafts persisted by each consumer) |
|
|
79
79
|
| `capabilities` | ✅ Implemented — discovery + permission filtering + safe pre-filtering + enumeration disclosure |
|
|
80
80
|
| `runtime` | ✅ Implemented — Provider, Capabilities, createAgent, MCP connection management, task migration |
|
|
81
|
-
| `plugin` | ✅ Implemented — AutoMigratePlugin / AutoRefreshToolsPlugin |
|
|
81
|
+
| `plugin` | ✅ Implemented — AutoMigratePlugin / AutoRefreshToolsPlugin / ContextGuardPlugin / UnknownToolHintPlugin |
|
|
82
82
|
| `shared` | ✅ Implemented — SdkError + injectable Logger |
|
|
83
|
-
| Tests | ✅
|
|
83
|
+
| Tests | ✅ 445 passing, 51 files, all green (incl. real-API chat & viewImage e2e) |
|
|
84
84
|
|
|
85
85
|
## Built-in Tools
|
|
86
86
|
|
|
@@ -106,11 +106,12 @@ All built-in tools are classes (extending `SdkCallbackTool`), instantiated by th
|
|
|
106
106
|
| `edit` | Edit text in a file |
|
|
107
107
|
| `sleep` | Wait for a specified number of milliseconds |
|
|
108
108
|
|
|
109
|
-
Conditionally injected
|
|
109
|
+
Conditionally injected based on the active model / config:
|
|
110
110
|
|
|
111
|
-
| Tool | Description |
|
|
112
|
-
|
|
113
|
-
| `generateImage` | Generate an image from a text description |
|
|
111
|
+
| Tool | Injection Condition | Description |
|
|
112
|
+
|------|---------------------|-------------|
|
|
113
|
+
| `generateImage` | Only when `defaultImageModel` is configured | Generate an image from a text description |
|
|
114
|
+
| `viewImage` | Only for vision models (the agent's `modelId` resolves to a model with `vision: true`) | View / analyze an image: local images are auto-uploaded via the Files API, network URLs are referenced directly |
|
|
114
115
|
|
|
115
116
|
## Built-in Plugins
|
|
116
117
|
|
|
@@ -118,6 +119,8 @@ Conditionally injected (registered only when `defaultImageModel` is configured):
|
|
|
118
119
|
|--------|-------------|
|
|
119
120
|
| `AutoMigratePlugin` | Automatically triggers task migration when the context overflows, generates a handoff document, and transparently replaces the Agent |
|
|
120
121
|
| `AutoRefreshToolsPlugin` | Re-scans the file system before each `send()` to refresh the tool list |
|
|
122
|
+
| `ContextGuardPlugin` | Context safety guard — before each request, throws `ContextOverflowError` (interrupting the conversation) when the previous round's `usage.prompt_tokens` exceeds `maxTokens × ratio` (default 1.5), preventing context runaway from reading oversized files |
|
|
123
|
+
| `UnknownToolHintPlugin` | Smarter unknown-tool hints — when the LLM calls a nonexistent tool, guides it to `call_mcp_tool` / points out permission issues based on MCP config (opt-in via `agent.use`) |
|
|
121
124
|
|
|
122
125
|
## Design Principles
|
|
123
126
|
|
package/README.zh.md
CHANGED
|
@@ -22,7 +22,7 @@ config ← 读写 config.json + 迁移 + 内存缓存 + 原子写入
|
|
|
22
22
|
crud ← 能力实体 CRUD(Agent 定义等;会话/草稿已下放给各端自行持久化)
|
|
23
23
|
capabilities ← 能力发现与装配(内置 + 用户 + MCP + Skill + SubAgent)
|
|
24
24
|
runtime ← Provider + 模型工厂 + Agent 组装 + MCP 连接管理 + 任务迁移 + SdkCallbackTool 工具基类
|
|
25
|
-
plugin ← Agent 插件(autoMigrate、autoRefreshTools)
|
|
25
|
+
plugin ← Agent 插件(autoMigrate、autoRefreshTools、contextGuard、unknownToolHint)
|
|
26
26
|
shared ← 日志、错误
|
|
27
27
|
```
|
|
28
28
|
|
|
@@ -77,9 +77,9 @@ await agent.send("你好");
|
|
|
77
77
|
| `crud` | ✅ 已实现 — Agent 等能力实体 CRUD(会话/草稿由各端自行持久化) |
|
|
78
78
|
| `capabilities` | ✅ 已实现 — 发现 + 权限过滤 + 安全预过滤 + 枚举披露 |
|
|
79
79
|
| `runtime` | ✅ 已实现 — Provider、Capabilities、createAgent、MCP 连接管理、任务迁移 |
|
|
80
|
-
| `plugin` | ✅ 已实现 — AutoMigratePlugin / AutoRefreshToolsPlugin |
|
|
80
|
+
| `plugin` | ✅ 已实现 — AutoMigratePlugin / AutoRefreshToolsPlugin / ContextGuardPlugin / UnknownToolHintPlugin |
|
|
81
81
|
| `shared` | ✅ 已实现 — SdkError + 可注入 Logger |
|
|
82
|
-
| 测试 | ✅
|
|
82
|
+
| 测试 | ✅ 445 通过,51 个文件,全绿(含真实 API 聊天与 viewImage e2e) |
|
|
83
83
|
|
|
84
84
|
## 内置工具
|
|
85
85
|
|
|
@@ -105,11 +105,12 @@ await agent.send("你好");
|
|
|
105
105
|
| `edit` | 编辑文件中的文本 |
|
|
106
106
|
| `sleep` | 等待指定毫秒数后继续 |
|
|
107
107
|
|
|
108
|
-
|
|
108
|
+
条件注入(按当前模型 / 配置决定是否注册):
|
|
109
109
|
|
|
110
|
-
| 工具 | 说明 |
|
|
111
|
-
|
|
112
|
-
| `generateImage` | 根据文字描述生成图片 |
|
|
110
|
+
| 工具 | 注入条件 | 说明 |
|
|
111
|
+
|------|----------|------|
|
|
112
|
+
| `generateImage` | 配置了 `defaultImageModel` 才注册 | 根据文字描述生成图片 |
|
|
113
|
+
| `viewImage` | 仅视觉模型可用(Agent 的 `modelId` 解析为 `vision: true` 的模型) | 查看/分析图片:本地图片自动经 Files API 上传,网络 URL 直接引用 |
|
|
113
114
|
|
|
114
115
|
## 内置插件
|
|
115
116
|
|
|
@@ -117,6 +118,8 @@ await agent.send("你好");
|
|
|
117
118
|
|------|------|
|
|
118
119
|
| `AutoMigratePlugin` | 上下文超限时自动触发任务迁移,生成交接文档,透明替换 Agent |
|
|
119
120
|
| `AutoRefreshToolsPlugin` | 每次 `send()` 前重新扫描文件系统,刷新工具列表 |
|
|
121
|
+
| `ContextGuardPlugin` | 上下文安全护栏 — 每轮发请求前检测上一轮 `usage.prompt_tokens`,超过 `maxTokens × ratio`(默认 1.5,即 +50%)时抛 `ContextOverflowError` 中断对话,防止读入超大文件导致上下文失控 |
|
|
122
|
+
| `UnknownToolHintPlugin` | 未知工具智能提示 — LLM 调用不存在的工具时,根据 MCP 配置引导使用 `call_mcp_tool` / 提示权限问题(调用方显式 `agent.use` 注册) |
|
|
120
123
|
|
|
121
124
|
## 设计原则
|
|
122
125
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Tool } from "@ai-zen/agents-core";
|
|
2
2
|
import type { ToolEnv } from "../../types/index.js";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* 发现内置工具:按静态注册表实例化全部内置工具(无任何过滤)。
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* 工具的可用性由各工具自行声明(SdkCallbackTool.isAvailable,入参含完整 config
|
|
7
|
+
* + 模型信息),在 buildTools/filter 阶段(模型已知)按声明过滤。
|
|
8
8
|
*/
|
|
9
9
|
export declare function discoverBuiltinTools(env: ToolEnv): Tool[];
|
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
import { BUILTIN_TOOL_CLASSES } from "../implements/builtin/index.js";
|
|
2
|
-
import { GenerateImageTool } from "../implements/builtin/GenerateImageTool.js";
|
|
3
2
|
/**
|
|
4
|
-
*
|
|
3
|
+
* 发现内置工具:按静态注册表实例化全部内置工具(无任何过滤)。
|
|
5
4
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* 工具的可用性由各工具自行声明(SdkCallbackTool.isAvailable,入参含完整 config
|
|
6
|
+
* + 模型信息),在 buildTools/filter 阶段(模型已知)按声明过滤。
|
|
8
7
|
*/
|
|
9
8
|
export function discoverBuiltinTools(env) {
|
|
10
|
-
|
|
11
|
-
if (env.config.defaultImageModel) {
|
|
12
|
-
const existingNames = new Set(tools.map((t) => t.function.name));
|
|
13
|
-
if (!existingNames.has("generateImage")) {
|
|
14
|
-
tools.push(new GenerateImageTool(env));
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
return tools;
|
|
9
|
+
return BUILTIN_TOOL_CLASSES.map((Cls) => new Cls(env));
|
|
18
10
|
}
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import type { AgentNS, ToolCallContext } from "@ai-zen/agents-core";
|
|
2
2
|
import { SdkCallbackTool } from "../../../runtime/SdkCallbackTool.js";
|
|
3
|
-
import type { ToolEnv } from "../../../types/index.js";
|
|
3
|
+
import type { AgentDefinition, AppConfig, ToolEnv } from "../../../types/index.js";
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* 根据文字描述生成图片。**统一返回字符串**(JSON,含图片 URL 列表):
|
|
6
|
+
* 生成结果如何处理(查看/下载/转发)由模型与上层自由决定——
|
|
7
|
+
* 需要让模型看图时,模型可主动调用 viewImage(URL);需要保存时用 downloadFile。
|
|
6
8
|
*
|
|
7
|
-
* 依赖 AppConfig 中的 imageModels 和 defaultImageModel
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* 注意:与其它内置工具不同,generateImage 依赖图片模型配置,
|
|
11
|
-
* 因此由 discoverBuiltinTools 按 env.config.defaultImageModel 条件实例化,
|
|
12
|
-
* 不加入 BUILTIN_TOOL_CLASSES 静态注册表。
|
|
9
|
+
* 依赖 AppConfig 中的 imageModels 和 defaultImageModel 配置(isAvailable 声明),
|
|
10
|
+
* 未配置时由 buildTools 过滤,不暴露给 Agent。
|
|
13
11
|
*/
|
|
14
12
|
export declare class GenerateImageTool extends SdkCallbackTool {
|
|
13
|
+
/** 依赖图片模型配置(defaultImageModel),未配置时由 buildTools 按 isAvailable 过滤 */
|
|
14
|
+
isAvailable(config: AppConfig, _definition: AgentDefinition): boolean;
|
|
15
15
|
function: AgentNS.FunctionDefine;
|
|
16
16
|
constructor(env: ToolEnv);
|
|
17
17
|
call(input: {
|
|
@@ -1,19 +1,21 @@
|
|
|
1
|
-
import
|
|
1
|
+
import OpenAI from "openai";
|
|
2
2
|
import { SdkCallbackTool } from "../../../runtime/SdkCallbackTool.js";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* 根据文字描述生成图片。**统一返回字符串**(JSON,含图片 URL 列表):
|
|
5
|
+
* 生成结果如何处理(查看/下载/转发)由模型与上层自由决定——
|
|
6
|
+
* 需要让模型看图时,模型可主动调用 viewImage(URL);需要保存时用 downloadFile。
|
|
5
7
|
*
|
|
6
|
-
* 依赖 AppConfig 中的 imageModels 和 defaultImageModel
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* 注意:与其它内置工具不同,generateImage 依赖图片模型配置,
|
|
10
|
-
* 因此由 discoverBuiltinTools 按 env.config.defaultImageModel 条件实例化,
|
|
11
|
-
* 不加入 BUILTIN_TOOL_CLASSES 静态注册表。
|
|
8
|
+
* 依赖 AppConfig 中的 imageModels 和 defaultImageModel 配置(isAvailable 声明),
|
|
9
|
+
* 未配置时由 buildTools 过滤,不暴露给 Agent。
|
|
12
10
|
*/
|
|
13
11
|
export class GenerateImageTool extends SdkCallbackTool {
|
|
12
|
+
/** 依赖图片模型配置(defaultImageModel),未配置时由 buildTools 按 isAvailable 过滤 */
|
|
13
|
+
isAvailable(config, _definition) {
|
|
14
|
+
return !!config.defaultImageModel;
|
|
15
|
+
}
|
|
14
16
|
function = {
|
|
15
17
|
name: "generateImage",
|
|
16
|
-
description: "
|
|
18
|
+
description: "根据文字描述生成图片,返回图片 URL 列表(JSON)。如需查看图片内容,请用 viewImage 工具(传入图片 URL);如需保存到本地,用 downloadFile 下载。",
|
|
17
19
|
parameters: {
|
|
18
20
|
type: "object",
|
|
19
21
|
properties: {
|
|
@@ -81,34 +83,31 @@ export class GenerateImageTool extends SdkCallbackTool {
|
|
|
81
83
|
error: `端点 "${endpoint.name}" 的 API Key 未设置`,
|
|
82
84
|
});
|
|
83
85
|
}
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
const client = new OpenAI({
|
|
87
|
+
apiKey: endpoint.apiKey,
|
|
88
|
+
baseURL: endpoint.baseUrl,
|
|
87
89
|
});
|
|
88
90
|
const modelName = input.model || imageModel.modelName;
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
});
|
|
92
|
-
const result = await model.generate({
|
|
93
|
-
prompt: prompt.trim(),
|
|
91
|
+
// 通过官方 SDK 生成图片(厂商特有字段如 size/quality 用 any 透传)
|
|
92
|
+
const result = await client.images.generate({
|
|
94
93
|
model: modelName,
|
|
94
|
+
prompt: prompt.trim(),
|
|
95
95
|
size: input.size || imageModel.defaultSize || undefined,
|
|
96
96
|
quality: input.quality || imageModel.defaultQuality || undefined,
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
97
|
+
}, { signal });
|
|
98
|
+
const urls = (result.data ?? [])
|
|
99
|
+
.map((img) => img.url)
|
|
100
|
+
.filter((url) => !!url);
|
|
101
|
+
// 统一返回字符串(JSON):生成结果如何处理(查看/下载/转发)由模型与上层自由决定,
|
|
102
|
+
// 不替模型做「看或不看」的假设。需要让模型看图 → 模型可主动调用 viewImage(URL);
|
|
103
|
+
// 需要保存 → downloadFile 下载。
|
|
104
104
|
return JSON.stringify({
|
|
105
105
|
success: true,
|
|
106
106
|
model: imageModel.name,
|
|
107
107
|
modelId: imageModel.id,
|
|
108
108
|
created: result.created,
|
|
109
|
-
images,
|
|
110
|
-
|
|
111
|
-
note: "图片临时链接有效期为30天,请及时转存图片。",
|
|
109
|
+
images: urls.map((url, i) => ({ index: i, url })),
|
|
110
|
+
note: "图片临时链接有效期为30天。如需查看图片内容,可用 viewImage 工具(传入图片 URL);如需保存,可用 downloadFile 下载。",
|
|
112
111
|
}, null, 2);
|
|
113
112
|
}
|
|
114
113
|
catch (error) {
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { AgentNS, ToolCallContext } from "@ai-zen/agents-core";
|
|
2
|
+
import { SdkCallbackTool } from "../../../runtime/SdkCallbackTool.js";
|
|
3
|
+
import type { AgentDefinition, AppConfig, ToolEnv } from "../../../types/index.js";
|
|
4
|
+
/**
|
|
5
|
+
* 查看图片工具 — 让 Agent 主动查看/分析一张图片。**仅对视觉模型可用**:
|
|
6
|
+
* 由 `isAvailable(config, definition)` 声明(从 definition.modelId 解析模型,检查 Model.vision),
|
|
7
|
+
* buildTools 阶段按声明过滤——非视觉模型不会拿到该工具,故无需运行时重复校验。
|
|
8
|
+
*
|
|
9
|
+
* 输入 `path_or_url`:
|
|
10
|
+
* - http(s) URL → 直接以 `image_url` 内容块返回,模型直接查看;
|
|
11
|
+
* - 本地路径 → 自动通过 Files API(DeepSeek)上传,以 `file` 内容块(file_id)返回。
|
|
12
|
+
*
|
|
13
|
+
* 不使用 base64 内联(本地图片走 Files API,符合 DeepSeek 推荐与 48 MiB 请求体限制)。
|
|
14
|
+
* 依赖配置中的 deepseek 端点(Files API 能力);URL 场景不依赖上传。
|
|
15
|
+
*/
|
|
16
|
+
export declare class ViewImageTool extends SdkCallbackTool {
|
|
17
|
+
/** 仅视觉模型可用:从 definition.modelId 解析当前模型,非视觉模型由 buildTools 过滤 */
|
|
18
|
+
isAvailable(config: AppConfig, definition: AgentDefinition): boolean;
|
|
19
|
+
function: AgentNS.FunctionDefine;
|
|
20
|
+
constructor(env: ToolEnv);
|
|
21
|
+
call(input: {
|
|
22
|
+
path_or_url?: string;
|
|
23
|
+
}, ctx?: ToolCallContext): Promise<AgentNS.MessageContent>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import OpenAI, { toFile } from "openai";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { SdkCallbackTool } from "../../../runtime/SdkCallbackTool.js";
|
|
5
|
+
const SUPPORTED_EXT = [".jpg", ".jpeg", ".png", ".gif", ".webp"];
|
|
6
|
+
/**
|
|
7
|
+
* 查看图片工具 — 让 Agent 主动查看/分析一张图片。**仅对视觉模型可用**:
|
|
8
|
+
* 由 `isAvailable(config, definition)` 声明(从 definition.modelId 解析模型,检查 Model.vision),
|
|
9
|
+
* buildTools 阶段按声明过滤——非视觉模型不会拿到该工具,故无需运行时重复校验。
|
|
10
|
+
*
|
|
11
|
+
* 输入 `path_or_url`:
|
|
12
|
+
* - http(s) URL → 直接以 `image_url` 内容块返回,模型直接查看;
|
|
13
|
+
* - 本地路径 → 自动通过 Files API(DeepSeek)上传,以 `file` 内容块(file_id)返回。
|
|
14
|
+
*
|
|
15
|
+
* 不使用 base64 内联(本地图片走 Files API,符合 DeepSeek 推荐与 48 MiB 请求体限制)。
|
|
16
|
+
* 依赖配置中的 deepseek 端点(Files API 能力);URL 场景不依赖上传。
|
|
17
|
+
*/
|
|
18
|
+
export class ViewImageTool extends SdkCallbackTool {
|
|
19
|
+
/** 仅视觉模型可用:从 definition.modelId 解析当前模型,非视觉模型由 buildTools 过滤 */
|
|
20
|
+
isAvailable(config, definition) {
|
|
21
|
+
const modelId = definition.modelId ?? config.defaultModel;
|
|
22
|
+
const model = (config.models ?? []).find((m) => m.id === modelId);
|
|
23
|
+
return model?.vision === true;
|
|
24
|
+
}
|
|
25
|
+
function = {
|
|
26
|
+
name: "viewImage",
|
|
27
|
+
description: "查看一张图片并分析其内容。传入本地图片路径或 http(s) 图片 URL。本地图片会自动上传(Files API)后引用,网络图片直接引用。适合识别图片文字、描述图片内容、分析图表等。",
|
|
28
|
+
parameters: {
|
|
29
|
+
type: "object",
|
|
30
|
+
properties: {
|
|
31
|
+
path_or_url: {
|
|
32
|
+
type: "string",
|
|
33
|
+
description: "本地图片路径(相对/绝对)或 http(s) 图片 URL。支持的格式:JPEG、PNG、GIF、WebP。",
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
required: ["path_or_url"],
|
|
37
|
+
additionalProperties: false,
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
constructor(env) {
|
|
41
|
+
super({ env });
|
|
42
|
+
}
|
|
43
|
+
async call(input, ctx) {
|
|
44
|
+
const signal = ctx?.signal;
|
|
45
|
+
const target = input?.path_or_url;
|
|
46
|
+
if (!target || !target.trim()) {
|
|
47
|
+
return "viewImage 需要提供 path_or_url 参数(本地图片路径或 http(s) URL)。";
|
|
48
|
+
}
|
|
49
|
+
const value = target.trim();
|
|
50
|
+
try {
|
|
51
|
+
// 网络图片:直接返回 image_url 内容块(无需上传)
|
|
52
|
+
if (/^https?:\/\//i.test(value)) {
|
|
53
|
+
return [
|
|
54
|
+
{
|
|
55
|
+
type: "text",
|
|
56
|
+
text: `已获取网络图片:${value}`,
|
|
57
|
+
},
|
|
58
|
+
{ type: "image_url", image_url: { url: value } },
|
|
59
|
+
];
|
|
60
|
+
}
|
|
61
|
+
// 本地图片:解析路径 + 校验 + Files API 上传
|
|
62
|
+
const absPath = path.isAbsolute(value)
|
|
63
|
+
? value
|
|
64
|
+
: path.join(this.env.cwd, value);
|
|
65
|
+
let stat;
|
|
66
|
+
try {
|
|
67
|
+
stat = await fs.stat(absPath);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return `本地图片不存在: ${absPath}`;
|
|
71
|
+
}
|
|
72
|
+
if (!stat.isFile()) {
|
|
73
|
+
return `"${absPath}" 不是文件。`;
|
|
74
|
+
}
|
|
75
|
+
const ext = path.extname(absPath).toLowerCase();
|
|
76
|
+
if (!SUPPORTED_EXT.includes(ext)) {
|
|
77
|
+
return `不支持的图片格式 "${ext || "(无扩展名)"}"。支持的格式:JPEG、PNG、GIF、WebP。`;
|
|
78
|
+
}
|
|
79
|
+
// 上传端点:deepseek(Files API 能力)
|
|
80
|
+
const endpoint = (this.env.config.endpoints ?? []).find((e) => e.id === "deepseek");
|
|
81
|
+
if (!endpoint) {
|
|
82
|
+
return "未配置 deepseek 端点,无法上传本地图片(Files API)。如需查看本地图片,请先配置 deepseek 端点;网络图片 URL 可直接使用。";
|
|
83
|
+
}
|
|
84
|
+
if (!endpoint.apiKey) {
|
|
85
|
+
return `端点 "${endpoint.name}" 的 API Key 未设置,无法上传本地图片。`;
|
|
86
|
+
}
|
|
87
|
+
const client = new OpenAI({
|
|
88
|
+
apiKey: endpoint.apiKey,
|
|
89
|
+
baseURL: endpoint.baseUrl,
|
|
90
|
+
});
|
|
91
|
+
const file = await toFile(await fs.readFile(absPath), path.basename(absPath));
|
|
92
|
+
const uploaded = await client.files.create({ file, purpose: "user_data" }, { signal });
|
|
93
|
+
return [
|
|
94
|
+
{
|
|
95
|
+
type: "text",
|
|
96
|
+
text: `图片已上传(${path.basename(absPath)}),file_id: ${uploaded.id}`,
|
|
97
|
+
},
|
|
98
|
+
{ type: "file", file_id: uploaded.id },
|
|
99
|
+
];
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
// 中断(abort)时返回明确的中断结果
|
|
103
|
+
if (signal?.aborted) {
|
|
104
|
+
return "图片查看被中断(aborted)";
|
|
105
|
+
}
|
|
106
|
+
return `查看图片失败: ${error instanceof Error ? error.message : String(error)}`;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -17,13 +17,15 @@ import { BatchEditTool } from "./BatchEditTool.js";
|
|
|
17
17
|
import { EditTool } from "./EditTool.js";
|
|
18
18
|
import { ExecAsyncTool } from "./ExecAsyncTool.js";
|
|
19
19
|
import { SleepTool } from "./SleepTool.js";
|
|
20
|
+
import { ViewImageTool } from "./ViewImageTool.js";
|
|
20
21
|
import { GenerateImageTool } from "./GenerateImageTool.js";
|
|
21
22
|
/**
|
|
22
|
-
*
|
|
23
|
+
* 全部内置工具类(19 个)。
|
|
23
24
|
* 工具类由 Provider 用 ToolEnv 实例化(每个 Provider 一套实例,注入其 cwd)。
|
|
24
25
|
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
26
|
+
* 发现层不做任何过滤——工具的可用性由各工具自行声明(SdkCallbackTool.isAvailable,
|
|
27
|
+
* 入参含完整 config + 模型信息),在 buildTools/filter 阶段按声明过滤。
|
|
28
|
+
* 如 GenerateImageTool 依赖 defaultImageModel、ViewImageTool 仅视觉模型可用。
|
|
27
29
|
*/
|
|
28
30
|
export declare const BUILTIN_TOOL_CLASSES: Array<new (env: ToolEnv) => SdkCallbackTool>;
|
|
29
|
-
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, GenerateImageTool, };
|
|
31
|
+
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, ViewImageTool, GenerateImageTool, };
|
|
@@ -15,13 +15,15 @@ import { BatchEditTool } from "./BatchEditTool.js";
|
|
|
15
15
|
import { EditTool } from "./EditTool.js";
|
|
16
16
|
import { ExecAsyncTool } from "./ExecAsyncTool.js";
|
|
17
17
|
import { SleepTool } from "./SleepTool.js";
|
|
18
|
+
import { ViewImageTool } from "./ViewImageTool.js";
|
|
18
19
|
import { GenerateImageTool } from "./GenerateImageTool.js";
|
|
19
20
|
/**
|
|
20
|
-
*
|
|
21
|
+
* 全部内置工具类(19 个)。
|
|
21
22
|
* 工具类由 Provider 用 ToolEnv 实例化(每个 Provider 一套实例,注入其 cwd)。
|
|
22
23
|
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
24
|
+
* 发现层不做任何过滤——工具的可用性由各工具自行声明(SdkCallbackTool.isAvailable,
|
|
25
|
+
* 入参含完整 config + 模型信息),在 buildTools/filter 阶段按声明过滤。
|
|
26
|
+
* 如 GenerateImageTool 依赖 defaultImageModel、ViewImageTool 仅视觉模型可用。
|
|
25
27
|
*/
|
|
26
28
|
export const BUILTIN_TOOL_CLASSES = [
|
|
27
29
|
CwdTool,
|
|
@@ -41,5 +43,7 @@ export const BUILTIN_TOOL_CLASSES = [
|
|
|
41
43
|
EditTool,
|
|
42
44
|
ExecAsyncTool,
|
|
43
45
|
SleepTool,
|
|
46
|
+
ViewImageTool,
|
|
47
|
+
GenerateImageTool,
|
|
44
48
|
];
|
|
45
|
-
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, GenerateImageTool, };
|
|
49
|
+
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, ViewImageTool, GenerateImageTool, };
|
|
@@ -113,17 +113,17 @@ export function createCallSkillSubAgentTool(skillDirs, filteredSkills, provider)
|
|
|
113
113
|
if (!skill.subAgent) {
|
|
114
114
|
return `Skill "${skillId}" 不支持子 Agent 模式,请使用 load_skill 加载指导后自行处理`;
|
|
115
115
|
}
|
|
116
|
+
// 临时 Skill 子 Agent 复用父 Agent 定义(权限 + 模型),排除当前 Skill 防自调用
|
|
116
117
|
const parentAgent = ctx.agent;
|
|
117
|
-
const
|
|
118
|
-
? parentAgent.
|
|
119
|
-
: undefined;
|
|
120
|
-
const skillTools = provider && parentPermissions
|
|
121
|
-
? provider.buildTools(parentPermissions, {
|
|
118
|
+
const skillTools = provider && parentAgent instanceof SdkAgent
|
|
119
|
+
? provider.buildTools(parentAgent.definition, {
|
|
122
120
|
exclude: { skills: [skillId] },
|
|
123
121
|
})
|
|
124
122
|
: [];
|
|
125
123
|
const subAgent = new Agent({
|
|
124
|
+
client: parentAgent.client,
|
|
126
125
|
model: parentAgent.model,
|
|
126
|
+
modelConfig: parentAgent.modelConfig,
|
|
127
127
|
messages: [
|
|
128
128
|
Message.System(skill.content),
|
|
129
129
|
Message.User(task),
|
|
@@ -18,28 +18,37 @@ export function createSubAgentTool(def, provider) {
|
|
|
18
18
|
parameters: def.function.parameters,
|
|
19
19
|
},
|
|
20
20
|
messages: def.messages,
|
|
21
|
-
buildAgent: (
|
|
21
|
+
buildAgent: (_parsedArgs, ctx) => {
|
|
22
22
|
// 模型解析:SubAgent 可指定独立模型,否则复用父 Agent 的模型
|
|
23
|
-
let
|
|
23
|
+
let client;
|
|
24
|
+
let model;
|
|
25
|
+
let modelConfig;
|
|
24
26
|
if (def.modelId) {
|
|
25
|
-
|
|
27
|
+
const m = createModel(provider, def.modelId);
|
|
28
|
+
client = m.client;
|
|
29
|
+
model = m.model;
|
|
30
|
+
modelConfig = m.modelConfig;
|
|
26
31
|
}
|
|
27
32
|
else {
|
|
28
|
-
const
|
|
29
|
-
if (!
|
|
33
|
+
const parent = ctx.agent;
|
|
34
|
+
if (!parent) {
|
|
30
35
|
throw new Error(`SubAgent "${selfName}" 未指定 modelId,且父 Agent 没有可复用的模型`);
|
|
31
36
|
}
|
|
32
|
-
|
|
37
|
+
client = parent.client;
|
|
38
|
+
model = parent.model;
|
|
39
|
+
modelConfig = parent.modelConfig;
|
|
33
40
|
}
|
|
34
41
|
// SubAgent 独立解析自己的工具(排除自身,防止递归)
|
|
35
|
-
const subFiltered = provider.filter(def
|
|
42
|
+
const subFiltered = provider.filter(def, {
|
|
36
43
|
exclude: {
|
|
37
44
|
subagents: def.function?.name ? [def.function.name] : undefined,
|
|
38
45
|
},
|
|
39
46
|
});
|
|
40
47
|
const tools = provider.instantiate(subFiltered);
|
|
41
48
|
const subAgent = new Agent({
|
|
42
|
-
|
|
49
|
+
client,
|
|
50
|
+
model,
|
|
51
|
+
modelConfig,
|
|
43
52
|
tools,
|
|
44
53
|
});
|
|
45
54
|
return subAgent;
|
package/dist/config/constants.js
CHANGED
|
@@ -119,7 +119,8 @@ export const DEFAULT_APP_CONFIG = {
|
|
|
119
119
|
modelName: "glm-5v-turbo",
|
|
120
120
|
maxContextTokens: 250_000,
|
|
121
121
|
defaultParams: {},
|
|
122
|
-
|
|
122
|
+
vision: true,
|
|
123
|
+
description: "智谱AI 多模态Coding基座(支持图片输入)",
|
|
123
124
|
},
|
|
124
125
|
{
|
|
125
126
|
id: "glm-4.7-flash",
|
|
@@ -148,6 +149,16 @@ export const DEFAULT_APP_CONFIG = {
|
|
|
148
149
|
defaultParams: { thinking: { type: "enabled" } },
|
|
149
150
|
description: "DeepSeek 经济高效模型",
|
|
150
151
|
},
|
|
152
|
+
{
|
|
153
|
+
id: "deepseek-v4-flash-vision-exp",
|
|
154
|
+
name: "DeepSeek-V4-Flash-Vision-Exp",
|
|
155
|
+
endpointId: "deepseek",
|
|
156
|
+
modelName: "deepseek-v4-flash-vision-exp",
|
|
157
|
+
maxContextTokens: 250_000,
|
|
158
|
+
defaultParams: { thinking: { type: "enabled" } },
|
|
159
|
+
vision: true,
|
|
160
|
+
description: "DeepSeek 视觉实验模型(支持图片/文件输入)",
|
|
161
|
+
},
|
|
151
162
|
],
|
|
152
163
|
imageModels: [
|
|
153
164
|
{
|
package/dist/index.d.ts
CHANGED
|
@@ -29,9 +29,11 @@ export type { AutoMigrateOptions } from "./plugin/AutoMigratePlugin.js";
|
|
|
29
29
|
export { AutoRefreshToolsPlugin } from "./plugin/AutoRefreshToolsPlugin.js";
|
|
30
30
|
export { ContextGuardPlugin } from "./plugin/ContextGuardPlugin.js";
|
|
31
31
|
export type { ContextGuardOptions } from "./plugin/ContextGuardPlugin.js";
|
|
32
|
+
export { UnknownToolHintPlugin } from "./plugin/UnknownToolHintPlugin.js";
|
|
32
33
|
export { BUILTIN_TOOL_CLASSES } from "./capabilities/implements/builtin/index.js";
|
|
33
34
|
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, } from "./capabilities/implements/builtin/index.js";
|
|
34
35
|
export { GenerateImageTool } from "./capabilities/implements/builtin/GenerateImageTool.js";
|
|
36
|
+
export { ViewImageTool } from "./capabilities/implements/builtin/ViewImageTool.js";
|
|
35
37
|
export { createLoadSkillTool, createCallSkillSubAgentTool } from "./capabilities/implements/skillTools.js";
|
|
36
38
|
export { createLoadMcpTool, createCallMcpTool, createReadMcpResourceTool } from "./capabilities/implements/mcpTools.js";
|
|
37
39
|
export { createSubAgentTool } from "./capabilities/implements/subAgentTools.js";
|
package/dist/index.js
CHANGED
|
@@ -24,10 +24,12 @@ export { TaskMigrationService } from "./runtime/TaskMigrationService.js";
|
|
|
24
24
|
export { AutoMigratePlugin } from "./plugin/AutoMigratePlugin.js";
|
|
25
25
|
export { AutoRefreshToolsPlugin } from "./plugin/AutoRefreshToolsPlugin.js";
|
|
26
26
|
export { ContextGuardPlugin } from "./plugin/ContextGuardPlugin.js";
|
|
27
|
+
export { UnknownToolHintPlugin } from "./plugin/UnknownToolHintPlugin.js";
|
|
27
28
|
// 工具 — 内置工具类 + 动态工具工厂
|
|
28
29
|
export { BUILTIN_TOOL_CLASSES } from "./capabilities/implements/builtin/index.js";
|
|
29
30
|
export { CwdTool, ReadFileTool, WriteFileTool, ExecTool, MkdirTool, RmTool, GlobTool, LsTool, ExistTool, FindTextTool, DownloadFileTool, RenameTool, CopyTool, BatchEditTool, EditTool, ExecAsyncTool, SleepTool, } from "./capabilities/implements/builtin/index.js";
|
|
30
31
|
export { GenerateImageTool } from "./capabilities/implements/builtin/GenerateImageTool.js";
|
|
32
|
+
export { ViewImageTool } from "./capabilities/implements/builtin/ViewImageTool.js";
|
|
31
33
|
export { createLoadSkillTool, createCallSkillSubAgentTool } from "./capabilities/implements/skillTools.js";
|
|
32
34
|
export { createLoadMcpTool, createCallMcpTool, createReadMcpResourceTool } from "./capabilities/implements/mcpTools.js";
|
|
33
35
|
export { createSubAgentTool } from "./capabilities/implements/subAgentTools.js";
|
|
@@ -22,7 +22,7 @@ export class AutoMigratePlugin {
|
|
|
22
22
|
this.options = options;
|
|
23
23
|
}
|
|
24
24
|
async onAfterSend(ctx) {
|
|
25
|
-
const
|
|
25
|
+
const agent = ctx.agent;
|
|
26
26
|
const { maxTokens, migrationAgent, onBeforeMigrate, onMigrated } = this.options;
|
|
27
27
|
const promptTokens = agent.lastUsage?.prompt_tokens;
|
|
28
28
|
if (promptTokens == null)
|
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export class AutoRefreshToolsPlugin {
|
|
10
10
|
async onBeforeSend(ctx) {
|
|
11
|
-
const
|
|
11
|
+
const agent = ctx.agent;
|
|
12
12
|
await agent.provider.refresh({ silent: true });
|
|
13
|
-
agent.tools = agent.provider.buildTools(agent.
|
|
13
|
+
agent.tools = agent.provider.buildTools(agent.definition, {
|
|
14
14
|
exclude: {
|
|
15
15
|
subagents: agent.definition.function?.name
|
|
16
16
|
? [agent.definition.function.name]
|