@swarmclawai/swarmclaw 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -139
- package/package.json +1 -1
- package/src/app/api/agents/[id]/thread/route.ts +1 -2
- package/src/app/api/agents/route.ts +1 -1
- package/src/app/api/{sessions → chats}/[id]/checkpoints/route.ts +1 -1
- package/src/app/api/{sessions → chats}/[id]/main-loop/route.ts +2 -2
- package/src/app/api/{sessions → chats}/[id]/restore/route.ts +1 -1
- package/src/app/api/{sessions → chats}/[id]/route.ts +4 -52
- package/src/app/api/{sessions → chats}/route.ts +5 -7
- package/src/app/api/plugins/route.ts +3 -0
- package/src/app/api/plugins/settings/route.ts +35 -0
- package/src/app/api/usage/route.ts +30 -0
- package/src/cli/index.js +35 -33
- package/src/cli/index.ts +40 -39
- package/src/cli/spec.js +29 -27
- package/src/components/agents/agent-card.tsx +1 -1
- package/src/components/agents/agent-chat-list.tsx +3 -3
- package/src/components/agents/agent-list.tsx +8 -13
- package/src/components/agents/agent-sheet.tsx +2 -2
- package/src/components/agents/cron-job-form.tsx +3 -3
- package/src/components/agents/inspector-panel.tsx +2 -2
- package/src/components/auth/setup-wizard.tsx +5 -38
- package/src/components/chat/chat-area.tsx +10 -14
- package/src/components/{sessions/session-card.tsx → chat/chat-card.tsx} +3 -3
- package/src/components/chat/chat-header.tsx +156 -73
- package/src/components/{sessions/session-list.tsx → chat/chat-list.tsx} +4 -5
- package/src/components/chat/chat-tool-toggles.tsx +26 -17
- package/src/components/chat/checkpoint-timeline.tsx +4 -4
- package/src/components/chat/message-bubble.tsx +4 -1
- package/src/components/chat/message-list.tsx +2 -2
- package/src/components/{sessions/new-session-sheet.tsx → chat/new-chat-sheet.tsx} +6 -6
- package/src/components/chat/session-debug-panel.tsx +1 -1
- package/src/components/chat/tool-request-banner.tsx +3 -3
- package/src/components/chatrooms/agent-hover-card.tsx +3 -3
- package/src/components/chatrooms/chatroom-tool-request-banner.tsx +2 -2
- package/src/components/connectors/connector-sheet.tsx +1 -1
- package/src/components/home/home-view.tsx +1 -1
- package/src/components/layout/app-layout.tsx +23 -2
- package/src/components/plugins/plugin-list.tsx +475 -254
- package/src/components/plugins/plugin-sheet.tsx +124 -10
- package/src/components/settings/gateway-connection-panel.tsx +1 -1
- package/src/components/shared/command-palette.tsx +0 -1
- package/src/components/shared/settings/section-heartbeat.tsx +1 -1
- package/src/components/shared/settings/section-providers.tsx +1 -1
- package/src/components/shared/settings/settings-page.tsx +1 -12
- package/src/components/usage/metrics-dashboard.tsx +73 -0
- package/src/components/webhooks/webhook-sheet.tsx +1 -1
- package/src/lib/chat.ts +1 -1
- package/src/lib/{sessions.ts → chats.ts} +28 -18
- package/src/lib/providers/claude-cli.ts +1 -1
- package/src/lib/server/approvals.ts +4 -4
- package/src/lib/server/capability-router.ts +10 -8
- package/src/lib/server/chat-execution.ts +36 -105
- package/src/lib/server/chatroom-helpers.ts +3 -3
- package/src/lib/server/connectors/manager.ts +4 -4
- package/src/lib/server/cost.ts +34 -1
- package/src/lib/server/daemon-state.ts +2 -2
- package/src/lib/server/heartbeat-service.ts +1 -1
- package/src/lib/server/main-agent-loop.ts +25 -160
- package/src/lib/server/main-session.ts +6 -13
- package/src/lib/server/orchestrator-lg.ts +3 -3
- package/src/lib/server/orchestrator.ts +5 -5
- package/src/lib/server/plugins.ts +112 -4
- package/src/lib/server/provider-health.ts +5 -3
- package/src/lib/server/queue.ts +12 -10
- package/src/lib/server/session-run-manager.test.ts +9 -6
- package/src/lib/server/session-run-manager.ts +1 -3
- package/src/lib/server/session-tools/calendar.ts +376 -0
- package/src/lib/server/session-tools/canvas.ts +1 -1
- package/src/lib/server/session-tools/chatroom.ts +4 -2
- package/src/lib/server/session-tools/connector.ts +5 -2
- package/src/lib/server/session-tools/context.ts +7 -3
- package/src/lib/server/session-tools/crud.ts +14 -6
- package/src/lib/server/session-tools/delegate.ts +95 -8
- package/src/lib/server/session-tools/discovery.ts +2 -2
- package/src/lib/server/session-tools/edit_file.ts +4 -2
- package/src/lib/server/session-tools/email.ts +322 -0
- package/src/lib/server/session-tools/file.ts +5 -2
- package/src/lib/server/session-tools/git.ts +1 -1
- package/src/lib/server/session-tools/http.ts +1 -1
- package/src/lib/server/session-tools/image-gen.ts +382 -0
- package/src/lib/server/session-tools/index.ts +74 -49
- package/src/lib/server/session-tools/memory.ts +139 -2
- package/src/lib/server/session-tools/monitor.ts +1 -1
- package/src/lib/server/session-tools/openclaw-nodes.ts +1 -1
- package/src/lib/server/session-tools/openclaw-workspace.ts +1 -1
- package/src/lib/server/session-tools/platform.ts +6 -3
- package/src/lib/server/session-tools/plugin-creator.ts +3 -3
- package/src/lib/server/session-tools/replicate.ts +303 -0
- package/src/lib/server/session-tools/sample-ui.ts +1 -1
- package/src/lib/server/session-tools/sandbox.ts +4 -2
- package/src/lib/server/session-tools/schedule.ts +4 -2
- package/src/lib/server/session-tools/session-info.ts +7 -4
- package/src/lib/server/session-tools/shell.ts +5 -2
- package/src/lib/server/session-tools/subagent.ts +2 -2
- package/src/lib/server/session-tools/wallet.ts +29 -2
- package/src/lib/server/session-tools/web.ts +44 -5
- package/src/lib/server/storage.ts +29 -9
- package/src/lib/server/stream-agent-chat.ts +72 -249
- package/src/lib/server/tool-aliases.ts +26 -15
- package/src/lib/server/tool-capability-policy.test.ts +9 -9
- package/src/lib/server/tool-capability-policy.ts +32 -27
- package/src/lib/tool-definitions.ts +4 -0
- package/src/lib/validation/schemas.ts +3 -1
- package/src/stores/use-app-store.ts +5 -5
- package/src/stores/use-chat-store.ts +7 -7
- package/src/types/index.ts +65 -3
- /package/src/app/api/{sessions → chats}/[id]/browser/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/chat/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/clear/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/deploy/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/devserver/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/edit-resend/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/fork/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/mailbox/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/messages/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/retry/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/[id]/stop/route.ts +0 -0
- /package/src/app/api/{sessions → chats}/heartbeat/route.ts +0 -0
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
import { tool, type StructuredToolInterface } from '@langchain/core/tools'
|
|
3
|
+
import fs from 'fs'
|
|
4
|
+
import path from 'path'
|
|
5
|
+
import type { Plugin, PluginHooks } from '@/types'
|
|
6
|
+
import { getPluginManager } from '../plugins'
|
|
7
|
+
import { normalizeToolInputArgs } from './normalize-tool-args'
|
|
8
|
+
import { loadSettings } from '../storage'
|
|
9
|
+
import { UPLOAD_DIR } from '../storage'
|
|
10
|
+
import type { ToolBuildContext } from './context'
|
|
11
|
+
|
|
12
|
+
type ImageProvider = 'openai' | 'stability' | 'replicate' | 'fal' | 'together' | 'fireworks' | 'bfl' | 'custom'
|
|
13
|
+
|
|
14
|
+
interface PluginConfig {
|
|
15
|
+
provider: ImageProvider
|
|
16
|
+
apiKey: string
|
|
17
|
+
model: string
|
|
18
|
+
defaultSize: string
|
|
19
|
+
customEndpoint: string
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function getConfig(): PluginConfig {
|
|
23
|
+
const settings = loadSettings()
|
|
24
|
+
const ps = (settings.pluginSettings as Record<string, Record<string, unknown>> | undefined)?.image_gen ?? {}
|
|
25
|
+
return {
|
|
26
|
+
provider: (ps.provider as ImageProvider) || 'openai',
|
|
27
|
+
apiKey: (ps.apiKey as string) || '',
|
|
28
|
+
model: (ps.model as string) || '',
|
|
29
|
+
defaultSize: (ps.defaultSize as string) || '1024x1024',
|
|
30
|
+
customEndpoint: (ps.customEndpoint as string) || '',
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
type GenResult = { b64?: string; url?: string; error?: string }
|
|
35
|
+
|
|
36
|
+
// --- Provider Implementations ---
|
|
37
|
+
|
|
38
|
+
async function generateOpenAI(prompt: string, size: string, quality: string, cfg: PluginConfig): Promise<GenResult> {
|
|
39
|
+
const model = cfg.model || 'gpt-image-1'
|
|
40
|
+
const res = await fetch('https://api.openai.com/v1/images/generations', {
|
|
41
|
+
method: 'POST',
|
|
42
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
|
|
43
|
+
body: JSON.stringify({ model, prompt, n: 1, size, quality, response_format: 'b64_json' }),
|
|
44
|
+
signal: AbortSignal.timeout(120_000),
|
|
45
|
+
})
|
|
46
|
+
if (!res.ok) return { error: `OpenAI ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
47
|
+
const data = await res.json()
|
|
48
|
+
return { b64: data?.data?.[0]?.b64_json }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function generateStability(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
52
|
+
// Stability v2beta uses multipart/form-data and returns raw image bytes
|
|
53
|
+
const model = cfg.model || 'sd3'
|
|
54
|
+
const formData = new FormData()
|
|
55
|
+
formData.append('prompt', prompt)
|
|
56
|
+
formData.append('model', model)
|
|
57
|
+
formData.append('output_format', 'png')
|
|
58
|
+
// Map size to aspect ratio
|
|
59
|
+
const [w, h] = size.split('x').map(Number)
|
|
60
|
+
if (w && h) {
|
|
61
|
+
const ratio = w > h ? '16:9' : h > w ? '9:16' : '1:1'
|
|
62
|
+
formData.append('aspect_ratio', ratio)
|
|
63
|
+
}
|
|
64
|
+
const res = await fetch('https://api.stability.ai/v2beta/stable-image/generate/sd3', {
|
|
65
|
+
method: 'POST',
|
|
66
|
+
headers: { Authorization: `Bearer ${cfg.apiKey}`, Accept: 'image/*' },
|
|
67
|
+
body: formData,
|
|
68
|
+
signal: AbortSignal.timeout(120_000),
|
|
69
|
+
})
|
|
70
|
+
if (!res.ok) return { error: `Stability ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
71
|
+
const buf = Buffer.from(await res.arrayBuffer())
|
|
72
|
+
return { b64: buf.toString('base64') }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function generateReplicate(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
76
|
+
const model = cfg.model || 'black-forest-labs/flux-schnell'
|
|
77
|
+
const [w, h] = size.split('x').map(Number)
|
|
78
|
+
// Try sync mode first (Prefer: wait blocks up to 60s)
|
|
79
|
+
const createRes = await fetch('https://api.replicate.com/v1/predictions', {
|
|
80
|
+
method: 'POST',
|
|
81
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}`, Prefer: 'wait' },
|
|
82
|
+
body: JSON.stringify({ model, input: { prompt, width: w || 1024, height: h || 1024 } }),
|
|
83
|
+
signal: AbortSignal.timeout(120_000),
|
|
84
|
+
})
|
|
85
|
+
if (!createRes.ok) return { error: `Replicate ${createRes.status}: ${(await createRes.text().catch(() => '')).slice(0, 300)}` }
|
|
86
|
+
let prediction = await createRes.json()
|
|
87
|
+
|
|
88
|
+
// If sync didn't complete, poll
|
|
89
|
+
if (prediction.status !== 'succeeded' && prediction.status !== 'failed' && prediction.urls?.get) {
|
|
90
|
+
const deadline = Date.now() + 120_000
|
|
91
|
+
while (prediction.status !== 'succeeded' && prediction.status !== 'failed' && Date.now() < deadline) {
|
|
92
|
+
await new Promise((r) => setTimeout(r, 2000))
|
|
93
|
+
const pollRes = await fetch(prediction.urls.get, {
|
|
94
|
+
headers: { Authorization: `Bearer ${cfg.apiKey}` },
|
|
95
|
+
signal: AbortSignal.timeout(10_000),
|
|
96
|
+
})
|
|
97
|
+
prediction = await pollRes.json()
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
if (prediction.status === 'failed') return { error: `Replicate failed: ${prediction.error || 'unknown'}` }
|
|
101
|
+
const output = Array.isArray(prediction.output) ? prediction.output[0] : prediction.output
|
|
102
|
+
if (typeof output === 'string') return { url: output }
|
|
103
|
+
return { error: 'No image in Replicate response.' }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function generateFal(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
107
|
+
const model = cfg.model || 'fal-ai/flux/schnell'
|
|
108
|
+
const [w, h] = size.split('x').map(Number)
|
|
109
|
+
const res = await fetch(`https://fal.run/${model}`, {
|
|
110
|
+
method: 'POST',
|
|
111
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Key ${cfg.apiKey}` },
|
|
112
|
+
body: JSON.stringify({ prompt, image_size: { width: w || 1024, height: h || 1024 }, num_images: 1 }),
|
|
113
|
+
signal: AbortSignal.timeout(120_000),
|
|
114
|
+
})
|
|
115
|
+
if (!res.ok) return { error: `fal.ai ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
116
|
+
const data = await res.json()
|
|
117
|
+
const imageUrl = data?.images?.[0]?.url
|
|
118
|
+
if (imageUrl) return { url: imageUrl }
|
|
119
|
+
return { error: 'No image in fal.ai response.' }
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function generateTogether(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
123
|
+
const model = cfg.model || 'black-forest-labs/FLUX.1-schnell-Free'
|
|
124
|
+
const [w, h] = size.split('x').map(Number)
|
|
125
|
+
const res = await fetch('https://api.together.xyz/v1/images/generations', {
|
|
126
|
+
method: 'POST',
|
|
127
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
|
|
128
|
+
body: JSON.stringify({ model, prompt, width: w || 1024, height: h || 1024, n: 1, response_format: 'b64_json' }),
|
|
129
|
+
signal: AbortSignal.timeout(120_000),
|
|
130
|
+
})
|
|
131
|
+
if (!res.ok) return { error: `Together ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
132
|
+
const data = await res.json()
|
|
133
|
+
const b64 = data?.data?.[0]?.b64_json
|
|
134
|
+
if (b64) return { b64 }
|
|
135
|
+
const url = data?.data?.[0]?.url
|
|
136
|
+
if (url) return { url }
|
|
137
|
+
return { error: 'No image in Together response.' }
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async function generateFireworks(prompt: string, _size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
141
|
+
const model = cfg.model || 'flux-1-schnell-fp8'
|
|
142
|
+
const res = await fetch(`https://api.fireworks.ai/inference/v1/workflows/accounts/fireworks/models/${model}/text_to_image`, {
|
|
143
|
+
method: 'POST',
|
|
144
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}`, Accept: 'image/jpeg' },
|
|
145
|
+
body: JSON.stringify({ prompt }),
|
|
146
|
+
signal: AbortSignal.timeout(120_000),
|
|
147
|
+
})
|
|
148
|
+
if (!res.ok) return { error: `Fireworks ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
149
|
+
// Response may be JSON with base64 array or raw image
|
|
150
|
+
const ct = res.headers.get('content-type') || ''
|
|
151
|
+
if (ct.includes('application/json')) {
|
|
152
|
+
const data = await res.json()
|
|
153
|
+
const b64 = data?.base64?.[0] ?? data?.data?.[0]?.b64_json
|
|
154
|
+
if (b64) return { b64 }
|
|
155
|
+
return { error: 'No image in Fireworks JSON response.' }
|
|
156
|
+
}
|
|
157
|
+
// Raw image bytes
|
|
158
|
+
const buf = Buffer.from(await res.arrayBuffer())
|
|
159
|
+
return { b64: buf.toString('base64') }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function generateBFL(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
|
|
163
|
+
const [w, h] = size.split('x').map(Number)
|
|
164
|
+
const model = cfg.model || 'flux-pro-1.1'
|
|
165
|
+
const createRes = await fetch(`https://api.bfl.ai/v1/${model}`, {
|
|
166
|
+
method: 'POST',
|
|
167
|
+
headers: { 'Content-Type': 'application/json', 'x-key': cfg.apiKey },
|
|
168
|
+
body: JSON.stringify({ prompt, width: w || 1024, height: h || 1024 }),
|
|
169
|
+
signal: AbortSignal.timeout(15_000),
|
|
170
|
+
})
|
|
171
|
+
if (!createRes.ok) return { error: `BFL ${createRes.status}: ${(await createRes.text().catch(() => '')).slice(0, 300)}` }
|
|
172
|
+
const task = await createRes.json()
|
|
173
|
+
const pollingUrl = task?.polling_url || (task?.id ? `https://api.bfl.ai/v1/get_result?id=${task.id}` : null)
|
|
174
|
+
if (!pollingUrl) return { error: 'No polling URL from BFL.' }
|
|
175
|
+
|
|
176
|
+
// Poll for result
|
|
177
|
+
const deadline = Date.now() + 120_000
|
|
178
|
+
while (Date.now() < deadline) {
|
|
179
|
+
await new Promise((r) => setTimeout(r, 2000))
|
|
180
|
+
const pollRes = await fetch(pollingUrl, {
|
|
181
|
+
headers: { 'x-key': cfg.apiKey },
|
|
182
|
+
signal: AbortSignal.timeout(10_000),
|
|
183
|
+
})
|
|
184
|
+
const result = await pollRes.json()
|
|
185
|
+
if (result.status === 'Ready' && result.result?.sample) return { url: result.result.sample }
|
|
186
|
+
if (result.status === 'Error') return { error: `BFL error: ${result.result || 'unknown'}` }
|
|
187
|
+
}
|
|
188
|
+
return { error: 'BFL generation timed out.' }
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async function generateCustom(prompt: string, size: string, quality: string, cfg: PluginConfig): Promise<GenResult> {
|
|
192
|
+
if (!cfg.customEndpoint) return { error: 'Custom endpoint URL not configured.' }
|
|
193
|
+
// Assumes OpenAI-compatible image generation API
|
|
194
|
+
const [w, h] = size.split('x').map(Number)
|
|
195
|
+
const res = await fetch(cfg.customEndpoint, {
|
|
196
|
+
method: 'POST',
|
|
197
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
|
|
198
|
+
body: JSON.stringify({ model: cfg.model || 'default', prompt, n: 1, size, width: w || 1024, height: h || 1024, quality, response_format: 'b64_json' }),
|
|
199
|
+
signal: AbortSignal.timeout(120_000),
|
|
200
|
+
})
|
|
201
|
+
if (!res.ok) return { error: `Custom API ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
|
|
202
|
+
const data = await res.json()
|
|
203
|
+
const b64 = data?.data?.[0]?.b64_json ?? data?.images?.[0]?.b64_json ?? data?.b64_json ?? data?.artifacts?.[0]?.base64
|
|
204
|
+
if (b64) return { b64 }
|
|
205
|
+
const url = data?.data?.[0]?.url ?? data?.images?.[0]?.url ?? data?.url
|
|
206
|
+
if (url) return { url }
|
|
207
|
+
return { error: 'No image found in custom API response.' }
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// --- Dispatcher ---
|
|
211
|
+
|
|
212
|
+
const PROVIDERS: Record<ImageProvider, (prompt: string, size: string, quality: string, cfg: PluginConfig) => Promise<GenResult>> = {
|
|
213
|
+
openai: generateOpenAI,
|
|
214
|
+
stability: (p, s, _q, c) => generateStability(p, s, c),
|
|
215
|
+
replicate: (p, s, _q, c) => generateReplicate(p, s, c),
|
|
216
|
+
fal: (p, s, _q, c) => generateFal(p, s, c),
|
|
217
|
+
together: (p, s, _q, c) => generateTogether(p, s, c),
|
|
218
|
+
fireworks: (p, s, _q, c) => generateFireworks(p, s, c),
|
|
219
|
+
bfl: (p, s, _q, c) => generateBFL(p, s, c),
|
|
220
|
+
custom: generateCustom,
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function saveImageResult(result: GenResult, prompt: string, filename: string | undefined): Promise<string> {
|
|
224
|
+
if (result.error) return `Error: ${result.error}`
|
|
225
|
+
|
|
226
|
+
if (result.b64) {
|
|
227
|
+
const buf = Buffer.from(result.b64, 'base64')
|
|
228
|
+
const baseName = filename || `img-${Date.now()}.png`
|
|
229
|
+
const safeName = baseName.replace(/[^a-zA-Z0-9._-]/g, '_')
|
|
230
|
+
const dest = path.join(UPLOAD_DIR, safeName)
|
|
231
|
+
if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true })
|
|
232
|
+
fs.writeFileSync(dest, buf)
|
|
233
|
+
return `Image generated (${buf.length} bytes).\n\n\n\n[Download](/api/uploads/${safeName})`
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (result.url) {
|
|
237
|
+
// Download remote URL to uploads
|
|
238
|
+
try {
|
|
239
|
+
const res = await fetch(result.url, { signal: AbortSignal.timeout(30_000) })
|
|
240
|
+
if (res.ok) {
|
|
241
|
+
const buf = Buffer.from(await res.arrayBuffer())
|
|
242
|
+
const baseName = filename || `img-${Date.now()}.png`
|
|
243
|
+
const safeName = baseName.replace(/[^a-zA-Z0-9._-]/g, '_')
|
|
244
|
+
const dest = path.join(UPLOAD_DIR, safeName)
|
|
245
|
+
if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true })
|
|
246
|
+
fs.writeFileSync(dest, buf)
|
|
247
|
+
return `Image generated (${buf.length} bytes).\n\n\n\n[Download](/api/uploads/${safeName})`
|
|
248
|
+
}
|
|
249
|
+
} catch { /* fall through to URL-only response */ }
|
|
250
|
+
return `Image generated: ${result.url}`
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
return 'Error: No image returned.'
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
async function executeImageGen(args: Record<string, unknown>): Promise<string> {
|
|
257
|
+
const normalized = normalizeToolInputArgs(args)
|
|
258
|
+
const prompt = String(normalized.prompt || '').trim()
|
|
259
|
+
if (!prompt) return 'Error: prompt is required.'
|
|
260
|
+
|
|
261
|
+
const cfg = getConfig()
|
|
262
|
+
if (!cfg.apiKey) return 'Error: Image generation API key not configured. Ask the user to add one in Plugin Settings > Image Generation.'
|
|
263
|
+
|
|
264
|
+
const size = String(normalized.size || cfg.defaultSize)
|
|
265
|
+
const quality = String(normalized.quality || 'standard')
|
|
266
|
+
const filename = normalized.filename as string | undefined
|
|
267
|
+
|
|
268
|
+
const generate = PROVIDERS[cfg.provider]
|
|
269
|
+
if (!generate) return `Error: Unknown provider "${cfg.provider}".`
|
|
270
|
+
|
|
271
|
+
try {
|
|
272
|
+
const result = await generate(prompt, size, quality, cfg)
|
|
273
|
+
return saveImageResult(result, prompt, filename)
|
|
274
|
+
} catch (err: unknown) {
|
|
275
|
+
return `Error: ${err instanceof Error ? err.message : String(err)}`
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const ImageGenPlugin: Plugin = {
|
|
280
|
+
name: 'Image Generation',
|
|
281
|
+
enabledByDefault: false,
|
|
282
|
+
description: 'Generate images from text prompts. Supports OpenAI, Stability AI, Replicate, fal.ai, Together AI, Fireworks AI, BFL (Flux), or any OpenAI-compatible API.',
|
|
283
|
+
hooks: {
|
|
284
|
+
getCapabilityDescription: () =>
|
|
285
|
+
'I can generate images from text descriptions using `generate_image`. Supports different sizes, quality levels, and providers.',
|
|
286
|
+
} as PluginHooks,
|
|
287
|
+
tools: [
|
|
288
|
+
{
|
|
289
|
+
name: 'generate_image',
|
|
290
|
+
description: 'Generate an image from a text prompt. The image is saved and a download link is returned. Use descriptive, detailed prompts for best results.',
|
|
291
|
+
parameters: {
|
|
292
|
+
type: 'object',
|
|
293
|
+
properties: {
|
|
294
|
+
prompt: { type: 'string', description: 'Detailed text description of the image to generate' },
|
|
295
|
+
size: { type: 'string', enum: ['1024x1024', '1536x1024', '1024x1536', '512x512', '768x768', '1280x720', '720x1280'], description: 'Image dimensions (default: 1024x1024)' },
|
|
296
|
+
quality: { type: 'string', enum: ['standard', 'hd', 'low', 'medium', 'high'], description: 'Quality level (default: standard). Primarily used by OpenAI.' },
|
|
297
|
+
filename: { type: 'string', description: 'Optional filename for the saved image (e.g. "hero-banner.png")' },
|
|
298
|
+
},
|
|
299
|
+
required: ['prompt'],
|
|
300
|
+
},
|
|
301
|
+
execute: async (args) => executeImageGen(args),
|
|
302
|
+
},
|
|
303
|
+
],
|
|
304
|
+
ui: {
|
|
305
|
+
settingsFields: [
|
|
306
|
+
{
|
|
307
|
+
key: 'provider',
|
|
308
|
+
label: 'Provider',
|
|
309
|
+
type: 'select',
|
|
310
|
+
options: [
|
|
311
|
+
{ value: 'openai', label: 'OpenAI (DALL-E / gpt-image)' },
|
|
312
|
+
{ value: 'stability', label: 'Stability AI' },
|
|
313
|
+
{ value: 'replicate', label: 'Replicate (Flux, SDXL, etc.)' },
|
|
314
|
+
{ value: 'fal', label: 'fal.ai (Flux, SDXL, etc.)' },
|
|
315
|
+
{ value: 'together', label: 'Together AI' },
|
|
316
|
+
{ value: 'fireworks', label: 'Fireworks AI' },
|
|
317
|
+
{ value: 'bfl', label: 'BFL / Black Forest Labs (Flux Pro)' },
|
|
318
|
+
{ value: 'custom', label: 'Custom (OpenAI-compatible endpoint)' },
|
|
319
|
+
],
|
|
320
|
+
defaultValue: 'openai',
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
key: 'apiKey',
|
|
324
|
+
label: 'API Key',
|
|
325
|
+
type: 'secret',
|
|
326
|
+
required: true,
|
|
327
|
+
placeholder: 'sk-... / r8_... / fal-...',
|
|
328
|
+
help: 'API key for the selected provider.',
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
key: 'model',
|
|
332
|
+
label: 'Model',
|
|
333
|
+
type: 'text',
|
|
334
|
+
placeholder: 'gpt-image-1 / black-forest-labs/flux-schnell / ...',
|
|
335
|
+
help: 'Model ID. Each provider has its own default if left blank.',
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
key: 'defaultSize',
|
|
339
|
+
label: 'Default Size',
|
|
340
|
+
type: 'select',
|
|
341
|
+
options: [
|
|
342
|
+
{ value: '1024x1024', label: '1024x1024 (Square)' },
|
|
343
|
+
{ value: '1536x1024', label: '1536x1024 (Landscape)' },
|
|
344
|
+
{ value: '1024x1536', label: '1024x1536 (Portrait)' },
|
|
345
|
+
{ value: '1280x720', label: '1280x720 (16:9)' },
|
|
346
|
+
{ value: '512x512', label: '512x512 (Small)' },
|
|
347
|
+
{ value: '768x768', label: '768x768 (Medium)' },
|
|
348
|
+
],
|
|
349
|
+
defaultValue: '1024x1024',
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
key: 'customEndpoint',
|
|
353
|
+
label: 'Custom Endpoint URL',
|
|
354
|
+
type: 'text',
|
|
355
|
+
placeholder: 'https://your-api.example.com/v1/images/generations',
|
|
356
|
+
help: 'Only used when provider is "Custom". Should accept OpenAI-compatible image generation requests.',
|
|
357
|
+
},
|
|
358
|
+
],
|
|
359
|
+
},
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
getPluginManager().registerBuiltin('image_gen', ImageGenPlugin)
|
|
363
|
+
|
|
364
|
+
export function buildImageGenTools(bctx: ToolBuildContext): StructuredToolInterface[] {
|
|
365
|
+
if (!bctx.hasPlugin('image_gen')) return []
|
|
366
|
+
|
|
367
|
+
return [
|
|
368
|
+
tool(
|
|
369
|
+
async (args) => executeImageGen(args),
|
|
370
|
+
{
|
|
371
|
+
name: 'generate_image',
|
|
372
|
+
description: ImageGenPlugin.tools![0].description,
|
|
373
|
+
schema: z.object({
|
|
374
|
+
prompt: z.string().describe('Detailed text description of the image to generate'),
|
|
375
|
+
size: z.enum(['1024x1024', '1536x1024', '1024x1536', '512x512', '768x768', '1280x720', '720x1280']).optional().describe('Image dimensions (default: 1024x1024)'),
|
|
376
|
+
quality: z.enum(['standard', 'hd', 'low', 'medium', 'high']).optional().describe('Quality level (default: standard)'),
|
|
377
|
+
filename: z.string().optional().describe('Optional filename for the saved image'),
|
|
378
|
+
}),
|
|
379
|
+
},
|
|
380
|
+
),
|
|
381
|
+
]
|
|
382
|
+
}
|
|
@@ -5,7 +5,7 @@ import { loadSettings, loadSessions, saveSessions, loadMcpServers } from '../sto
|
|
|
5
5
|
import { loadRuntimeSettings } from '../runtime-settings'
|
|
6
6
|
import { log } from '../logger'
|
|
7
7
|
import { resolveSessionToolPolicy } from '../tool-capability-policy'
|
|
8
|
-
import {
|
|
8
|
+
import { expandPluginIds } from '../tool-aliases'
|
|
9
9
|
import type { ToolContext, SessionToolsResult, ToolBuildContext } from './context'
|
|
10
10
|
|
|
11
11
|
// Import all tool modules to trigger their builtin registration
|
|
@@ -33,6 +33,10 @@ import { buildDiscoveryTools } from './discovery'
|
|
|
33
33
|
import { buildMonitorTools } from './monitor'
|
|
34
34
|
import { buildSampleUITools } from './sample-ui'
|
|
35
35
|
import { buildPluginCreatorTools } from './plugin-creator'
|
|
36
|
+
import { buildImageGenTools } from './image-gen'
|
|
37
|
+
import { buildEmailTools } from './email'
|
|
38
|
+
import { buildCalendarTools } from './calendar'
|
|
39
|
+
import { buildReplicateTools } from './replicate'
|
|
36
40
|
import { normalizeToolInputArgs } from './normalize-tool-args'
|
|
37
41
|
|
|
38
42
|
import { getPluginManager } from '../plugins'
|
|
@@ -41,34 +45,37 @@ import { jsonSchemaToZod } from '../mcp-client'
|
|
|
41
45
|
export type { ToolContext, SessionToolsResult }
|
|
42
46
|
export { sweepOrphanedBrowsers, cleanupSessionBrowser, getActiveBrowserCount, hasActiveBrowser }
|
|
43
47
|
|
|
44
|
-
export async function buildSessionTools(cwd: string,
|
|
48
|
+
export async function buildSessionTools(cwd: string, enabledPlugins: string[], ctx?: ToolContext): Promise<SessionToolsResult> {
|
|
45
49
|
const tools: StructuredToolInterface[] = []
|
|
46
50
|
const cleanupFns: (() => Promise<void>)[] = []
|
|
47
|
-
|
|
51
|
+
|
|
48
52
|
try {
|
|
49
53
|
const runtime = loadRuntimeSettings()
|
|
50
54
|
const commandTimeoutMs = runtime.shellCommandTimeoutMs
|
|
51
55
|
const claudeTimeoutMs = runtime.claudeCodeTimeoutMs
|
|
52
56
|
const cliProcessTimeoutMs = runtime.cliProcessTimeoutMs
|
|
53
57
|
const appSettings = loadSettings()
|
|
54
|
-
const toolPolicy = resolveSessionToolPolicy(
|
|
55
|
-
const
|
|
56
|
-
const
|
|
57
|
-
const
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
&& !
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
const
|
|
58
|
+
const toolPolicy = resolveSessionToolPolicy(enabledPlugins, appSettings)
|
|
59
|
+
const expandedEnabled = expandPluginIds(toolPolicy.enabledPlugins)
|
|
60
|
+
const expandedBlocked = expandPluginIds(toolPolicy.blockedPlugins.map((entry) => entry.tool))
|
|
61
|
+
const blockedSet = new Set(expandedBlocked)
|
|
62
|
+
const filteredEnabled = expandedEnabled.filter((id) => !blockedSet.has(id))
|
|
63
|
+
const pluginManager = getPluginManager()
|
|
64
|
+
const activePlugins = (filteredEnabled.includes('shell')
|
|
65
|
+
&& !filteredEnabled.includes('process')
|
|
66
|
+
&& !blockedSet.has('process')
|
|
67
|
+
? [...filteredEnabled, 'process']
|
|
68
|
+
: filteredEnabled).filter(t => pluginManager.isEnabled(t))
|
|
69
|
+
const activePluginSet = new Set(activePlugins)
|
|
70
|
+
const hasPlugin = (pluginName: string) => activePluginSet.has(pluginName)
|
|
71
|
+
/** @deprecated Use hasPlugin */
|
|
72
|
+
const hasTool = hasPlugin
|
|
66
73
|
|
|
67
|
-
if (toolPolicy.
|
|
68
|
-
log.info('session-tools', 'Capability policy blocked
|
|
74
|
+
if (toolPolicy.blockedPlugins.length > 0) {
|
|
75
|
+
log.info('session-tools', 'Capability policy blocked plugin families', {
|
|
69
76
|
sessionId: ctx?.sessionId || null,
|
|
70
77
|
agentId: ctx?.agentId || null,
|
|
71
|
-
|
|
78
|
+
blockedPlugins: toolPolicy.blockedPlugins.map((entry) => `${entry.tool}:${entry.reason}`),
|
|
72
79
|
})
|
|
73
80
|
}
|
|
74
81
|
|
|
@@ -78,14 +85,14 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
78
85
|
return sessions[ctx.sessionId] || null
|
|
79
86
|
}
|
|
80
87
|
|
|
81
|
-
const readStoredDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode'): string | null => {
|
|
88
|
+
const readStoredDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode' | 'gemini'): string | null => {
|
|
82
89
|
const session = resolveCurrentSession()
|
|
83
90
|
if (!session?.delegateResumeIds || typeof session.delegateResumeIds !== 'object') return null
|
|
84
91
|
const raw = session.delegateResumeIds[key]
|
|
85
92
|
return typeof raw === 'string' && raw.trim() ? raw.trim() : null
|
|
86
93
|
}
|
|
87
94
|
|
|
88
|
-
const persistDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode', resumeId: string | null | undefined): void => {
|
|
95
|
+
const persistDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode' | 'gemini', resumeId: string | null | undefined): void => {
|
|
89
96
|
const normalized = typeof resumeId === 'string' ? resumeId.trim() : ''
|
|
90
97
|
if (!normalized || !ctx?.sessionId) return
|
|
91
98
|
const sessions = loadSessions()
|
|
@@ -106,6 +113,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
106
113
|
const bctx: ToolBuildContext = {
|
|
107
114
|
cwd,
|
|
108
115
|
ctx,
|
|
116
|
+
hasPlugin,
|
|
109
117
|
hasTool,
|
|
110
118
|
cleanupFns,
|
|
111
119
|
commandTimeoutMs,
|
|
@@ -114,41 +122,54 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
114
122
|
persistDelegateResumeId,
|
|
115
123
|
readStoredDelegateResumeId,
|
|
116
124
|
resolveCurrentSession,
|
|
117
|
-
|
|
125
|
+
activePlugins,
|
|
118
126
|
}
|
|
119
127
|
|
|
120
128
|
// 1. Build Native Bridge Tools (Legacy enablement)
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
129
|
+
const toolToPluginMap: Record<string, string> = {}
|
|
130
|
+
|
|
131
|
+
const nativeBuilders: Array<[string, (ctx: ToolBuildContext) => StructuredToolInterface[]]> = [
|
|
132
|
+
['shell', buildShellTools],
|
|
133
|
+
['files', buildFileTools],
|
|
134
|
+
['edit_file', buildEditFileTools],
|
|
135
|
+
['delegate', buildDelegateTools],
|
|
136
|
+
['web', buildWebTools],
|
|
137
|
+
['memory', buildMemoryTools],
|
|
138
|
+
['manage_platform', buildPlatformTools],
|
|
139
|
+
['sandbox', buildSandboxTools],
|
|
140
|
+
['manage_chatrooms', buildChatroomTools],
|
|
141
|
+
['spawn_subagent', buildSubagentTools],
|
|
142
|
+
['canvas', buildCanvasTools],
|
|
143
|
+
['http', buildHttpTools],
|
|
144
|
+
['git', buildGitTools],
|
|
145
|
+
['wallet', buildWalletTools],
|
|
146
|
+
['openclaw_workspace', buildOpenClawWorkspaceTools],
|
|
147
|
+
['schedule', buildScheduleTools],
|
|
148
|
+
['manage_sessions', buildSessionInfoTools],
|
|
149
|
+
['openclaw_nodes', buildOpenClawNodeTools],
|
|
150
|
+
['context_mgmt', buildContextTools],
|
|
151
|
+
['manage_connectors', buildConnectorTools],
|
|
152
|
+
['discovery', buildDiscoveryTools],
|
|
153
|
+
['monitor', buildMonitorTools],
|
|
154
|
+
['sample_ui', buildSampleUITools],
|
|
155
|
+
['plugin_creator', buildPluginCreatorTools],
|
|
156
|
+
['image_gen', buildImageGenTools],
|
|
157
|
+
['email', buildEmailTools],
|
|
158
|
+
['calendar', buildCalendarTools],
|
|
159
|
+
['replicate', buildReplicateTools],
|
|
160
|
+
]
|
|
161
|
+
|
|
162
|
+
for (const [pluginId, builder] of nativeBuilders) {
|
|
163
|
+
const builtTools = builder(bctx)
|
|
164
|
+
for (const t of builtTools) {
|
|
165
|
+
toolToPluginMap[t.name] = pluginId
|
|
166
|
+
}
|
|
167
|
+
tools.push(...builtTools)
|
|
168
|
+
}
|
|
147
169
|
|
|
148
170
|
// 2. Build Plugin Tools (Built-in + External)
|
|
149
171
|
try {
|
|
150
|
-
const
|
|
151
|
-
const pluginTools = pluginManager.getTools(activeTools)
|
|
172
|
+
const pluginTools = pluginManager.getTools(activePlugins)
|
|
152
173
|
const existingNames = new Set(tools.map((t) => t.name))
|
|
153
174
|
|
|
154
175
|
for (const entry of pluginTools) {
|
|
@@ -161,6 +182,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
161
182
|
continue
|
|
162
183
|
}
|
|
163
184
|
existingNames.add(pt.name)
|
|
185
|
+
toolToPluginMap[pt.name] = entry.pluginId
|
|
164
186
|
|
|
165
187
|
tools.push(
|
|
166
188
|
tool(
|
|
@@ -208,6 +230,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
208
230
|
const mcpLcTools = await mcpToolsToLangChain(conn.client, config.name)
|
|
209
231
|
for (const t of mcpLcTools) {
|
|
210
232
|
if (!disabledMcpToolNames.has(t.name)) {
|
|
233
|
+
toolToPluginMap[t.name] = `mcp:${serverId}`
|
|
211
234
|
tools.push(t)
|
|
212
235
|
}
|
|
213
236
|
}
|
|
@@ -224,6 +247,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
224
247
|
}
|
|
225
248
|
|
|
226
249
|
// 4. Always available: request_tool_access
|
|
250
|
+
toolToPluginMap['request_tool_access'] = '_system'
|
|
227
251
|
tools.push(
|
|
228
252
|
tool(
|
|
229
253
|
async (args) => {
|
|
@@ -255,6 +279,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
|
|
|
255
279
|
try { await fn() } catch { /* ignore */ }
|
|
256
280
|
}
|
|
257
281
|
},
|
|
282
|
+
toolToPluginMap,
|
|
258
283
|
}
|
|
259
284
|
} catch (err: any) {
|
|
260
285
|
console.error('[session-tools] buildSessionTools critical failure:', err.message)
|