@swarmclawai/swarmclaw 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +85 -139
  2. package/package.json +1 -1
  3. package/src/app/api/agents/[id]/thread/route.ts +1 -2
  4. package/src/app/api/agents/route.ts +1 -1
  5. package/src/app/api/{sessions → chats}/[id]/checkpoints/route.ts +1 -1
  6. package/src/app/api/{sessions → chats}/[id]/main-loop/route.ts +2 -2
  7. package/src/app/api/{sessions → chats}/[id]/restore/route.ts +1 -1
  8. package/src/app/api/{sessions → chats}/[id]/route.ts +4 -52
  9. package/src/app/api/{sessions → chats}/route.ts +5 -7
  10. package/src/app/api/plugins/route.ts +3 -0
  11. package/src/app/api/plugins/settings/route.ts +35 -0
  12. package/src/app/api/usage/route.ts +30 -0
  13. package/src/cli/index.js +35 -33
  14. package/src/cli/index.ts +40 -39
  15. package/src/cli/spec.js +29 -27
  16. package/src/components/agents/agent-card.tsx +1 -1
  17. package/src/components/agents/agent-chat-list.tsx +3 -3
  18. package/src/components/agents/agent-list.tsx +8 -13
  19. package/src/components/agents/agent-sheet.tsx +2 -2
  20. package/src/components/agents/cron-job-form.tsx +3 -3
  21. package/src/components/agents/inspector-panel.tsx +2 -2
  22. package/src/components/auth/setup-wizard.tsx +5 -38
  23. package/src/components/chat/chat-area.tsx +10 -14
  24. package/src/components/{sessions/session-card.tsx → chat/chat-card.tsx} +3 -3
  25. package/src/components/chat/chat-header.tsx +156 -73
  26. package/src/components/{sessions/session-list.tsx → chat/chat-list.tsx} +4 -5
  27. package/src/components/chat/chat-tool-toggles.tsx +26 -17
  28. package/src/components/chat/checkpoint-timeline.tsx +4 -4
  29. package/src/components/chat/message-bubble.tsx +4 -1
  30. package/src/components/chat/message-list.tsx +2 -2
  31. package/src/components/{sessions/new-session-sheet.tsx → chat/new-chat-sheet.tsx} +6 -6
  32. package/src/components/chat/session-debug-panel.tsx +1 -1
  33. package/src/components/chat/tool-request-banner.tsx +3 -3
  34. package/src/components/chatrooms/agent-hover-card.tsx +3 -3
  35. package/src/components/chatrooms/chatroom-tool-request-banner.tsx +2 -2
  36. package/src/components/connectors/connector-sheet.tsx +1 -1
  37. package/src/components/home/home-view.tsx +1 -1
  38. package/src/components/layout/app-layout.tsx +23 -2
  39. package/src/components/plugins/plugin-list.tsx +475 -254
  40. package/src/components/plugins/plugin-sheet.tsx +124 -10
  41. package/src/components/settings/gateway-connection-panel.tsx +1 -1
  42. package/src/components/shared/command-palette.tsx +0 -1
  43. package/src/components/shared/settings/section-heartbeat.tsx +1 -1
  44. package/src/components/shared/settings/section-providers.tsx +1 -1
  45. package/src/components/shared/settings/settings-page.tsx +1 -12
  46. package/src/components/usage/metrics-dashboard.tsx +73 -0
  47. package/src/components/webhooks/webhook-sheet.tsx +1 -1
  48. package/src/lib/chat.ts +1 -1
  49. package/src/lib/{sessions.ts → chats.ts} +28 -18
  50. package/src/lib/providers/claude-cli.ts +1 -1
  51. package/src/lib/server/approvals.ts +4 -4
  52. package/src/lib/server/capability-router.ts +10 -8
  53. package/src/lib/server/chat-execution.ts +36 -105
  54. package/src/lib/server/chatroom-helpers.ts +3 -3
  55. package/src/lib/server/connectors/manager.ts +4 -4
  56. package/src/lib/server/cost.ts +34 -1
  57. package/src/lib/server/daemon-state.ts +2 -2
  58. package/src/lib/server/heartbeat-service.ts +1 -1
  59. package/src/lib/server/main-agent-loop.ts +25 -160
  60. package/src/lib/server/main-session.ts +6 -13
  61. package/src/lib/server/orchestrator-lg.ts +3 -3
  62. package/src/lib/server/orchestrator.ts +5 -5
  63. package/src/lib/server/plugins.ts +112 -4
  64. package/src/lib/server/provider-health.ts +5 -3
  65. package/src/lib/server/queue.ts +12 -10
  66. package/src/lib/server/session-run-manager.test.ts +9 -6
  67. package/src/lib/server/session-run-manager.ts +1 -3
  68. package/src/lib/server/session-tools/calendar.ts +376 -0
  69. package/src/lib/server/session-tools/canvas.ts +1 -1
  70. package/src/lib/server/session-tools/chatroom.ts +4 -2
  71. package/src/lib/server/session-tools/connector.ts +5 -2
  72. package/src/lib/server/session-tools/context.ts +7 -3
  73. package/src/lib/server/session-tools/crud.ts +14 -6
  74. package/src/lib/server/session-tools/delegate.ts +95 -8
  75. package/src/lib/server/session-tools/discovery.ts +2 -2
  76. package/src/lib/server/session-tools/edit_file.ts +4 -2
  77. package/src/lib/server/session-tools/email.ts +322 -0
  78. package/src/lib/server/session-tools/file.ts +5 -2
  79. package/src/lib/server/session-tools/git.ts +1 -1
  80. package/src/lib/server/session-tools/http.ts +1 -1
  81. package/src/lib/server/session-tools/image-gen.ts +382 -0
  82. package/src/lib/server/session-tools/index.ts +74 -49
  83. package/src/lib/server/session-tools/memory.ts +139 -2
  84. package/src/lib/server/session-tools/monitor.ts +1 -1
  85. package/src/lib/server/session-tools/openclaw-nodes.ts +1 -1
  86. package/src/lib/server/session-tools/openclaw-workspace.ts +1 -1
  87. package/src/lib/server/session-tools/platform.ts +6 -3
  88. package/src/lib/server/session-tools/plugin-creator.ts +3 -3
  89. package/src/lib/server/session-tools/replicate.ts +303 -0
  90. package/src/lib/server/session-tools/sample-ui.ts +1 -1
  91. package/src/lib/server/session-tools/sandbox.ts +4 -2
  92. package/src/lib/server/session-tools/schedule.ts +4 -2
  93. package/src/lib/server/session-tools/session-info.ts +7 -4
  94. package/src/lib/server/session-tools/shell.ts +5 -2
  95. package/src/lib/server/session-tools/subagent.ts +2 -2
  96. package/src/lib/server/session-tools/wallet.ts +29 -2
  97. package/src/lib/server/session-tools/web.ts +44 -5
  98. package/src/lib/server/storage.ts +29 -9
  99. package/src/lib/server/stream-agent-chat.ts +72 -249
  100. package/src/lib/server/tool-aliases.ts +26 -15
  101. package/src/lib/server/tool-capability-policy.test.ts +9 -9
  102. package/src/lib/server/tool-capability-policy.ts +32 -27
  103. package/src/lib/tool-definitions.ts +4 -0
  104. package/src/lib/validation/schemas.ts +3 -1
  105. package/src/stores/use-app-store.ts +5 -5
  106. package/src/stores/use-chat-store.ts +7 -7
  107. package/src/types/index.ts +65 -3
  108. /package/src/app/api/{sessions → chats}/[id]/browser/route.ts +0 -0
  109. /package/src/app/api/{sessions → chats}/[id]/chat/route.ts +0 -0
  110. /package/src/app/api/{sessions → chats}/[id]/clear/route.ts +0 -0
  111. /package/src/app/api/{sessions → chats}/[id]/deploy/route.ts +0 -0
  112. /package/src/app/api/{sessions → chats}/[id]/devserver/route.ts +0 -0
  113. /package/src/app/api/{sessions → chats}/[id]/edit-resend/route.ts +0 -0
  114. /package/src/app/api/{sessions → chats}/[id]/fork/route.ts +0 -0
  115. /package/src/app/api/{sessions → chats}/[id]/mailbox/route.ts +0 -0
  116. /package/src/app/api/{sessions → chats}/[id]/messages/route.ts +0 -0
  117. /package/src/app/api/{sessions → chats}/[id]/retry/route.ts +0 -0
  118. /package/src/app/api/{sessions → chats}/[id]/stop/route.ts +0 -0
  119. /package/src/app/api/{sessions → chats}/heartbeat/route.ts +0 -0
@@ -0,0 +1,382 @@
1
+ import { z } from 'zod'
2
+ import { tool, type StructuredToolInterface } from '@langchain/core/tools'
3
+ import fs from 'fs'
4
+ import path from 'path'
5
+ import type { Plugin, PluginHooks } from '@/types'
6
+ import { getPluginManager } from '../plugins'
7
+ import { normalizeToolInputArgs } from './normalize-tool-args'
8
+ import { loadSettings } from '../storage'
9
+ import { UPLOAD_DIR } from '../storage'
10
+ import type { ToolBuildContext } from './context'
11
+
12
+ type ImageProvider = 'openai' | 'stability' | 'replicate' | 'fal' | 'together' | 'fireworks' | 'bfl' | 'custom'
13
+
14
+ interface PluginConfig {
15
+ provider: ImageProvider
16
+ apiKey: string
17
+ model: string
18
+ defaultSize: string
19
+ customEndpoint: string
20
+ }
21
+
22
+ function getConfig(): PluginConfig {
23
+ const settings = loadSettings()
24
+ const ps = (settings.pluginSettings as Record<string, Record<string, unknown>> | undefined)?.image_gen ?? {}
25
+ return {
26
+ provider: (ps.provider as ImageProvider) || 'openai',
27
+ apiKey: (ps.apiKey as string) || '',
28
+ model: (ps.model as string) || '',
29
+ defaultSize: (ps.defaultSize as string) || '1024x1024',
30
+ customEndpoint: (ps.customEndpoint as string) || '',
31
+ }
32
+ }
33
+
34
+ type GenResult = { b64?: string; url?: string; error?: string }
35
+
36
+ // --- Provider Implementations ---
37
+
38
+ async function generateOpenAI(prompt: string, size: string, quality: string, cfg: PluginConfig): Promise<GenResult> {
39
+ const model = cfg.model || 'gpt-image-1'
40
+ const res = await fetch('https://api.openai.com/v1/images/generations', {
41
+ method: 'POST',
42
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
43
+ body: JSON.stringify({ model, prompt, n: 1, size, quality, response_format: 'b64_json' }),
44
+ signal: AbortSignal.timeout(120_000),
45
+ })
46
+ if (!res.ok) return { error: `OpenAI ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
47
+ const data = await res.json()
48
+ return { b64: data?.data?.[0]?.b64_json }
49
+ }
50
+
51
+ async function generateStability(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
52
+ // Stability v2beta uses multipart/form-data and returns raw image bytes
53
+ const model = cfg.model || 'sd3'
54
+ const formData = new FormData()
55
+ formData.append('prompt', prompt)
56
+ formData.append('model', model)
57
+ formData.append('output_format', 'png')
58
+ // Map size to aspect ratio
59
+ const [w, h] = size.split('x').map(Number)
60
+ if (w && h) {
61
+ const ratio = w > h ? '16:9' : h > w ? '9:16' : '1:1'
62
+ formData.append('aspect_ratio', ratio)
63
+ }
64
+ const res = await fetch('https://api.stability.ai/v2beta/stable-image/generate/sd3', {
65
+ method: 'POST',
66
+ headers: { Authorization: `Bearer ${cfg.apiKey}`, Accept: 'image/*' },
67
+ body: formData,
68
+ signal: AbortSignal.timeout(120_000),
69
+ })
70
+ if (!res.ok) return { error: `Stability ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
71
+ const buf = Buffer.from(await res.arrayBuffer())
72
+ return { b64: buf.toString('base64') }
73
+ }
74
+
75
+ async function generateReplicate(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
76
+ const model = cfg.model || 'black-forest-labs/flux-schnell'
77
+ const [w, h] = size.split('x').map(Number)
78
+ // Try sync mode first (Prefer: wait blocks up to 60s)
79
+ const createRes = await fetch('https://api.replicate.com/v1/predictions', {
80
+ method: 'POST',
81
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}`, Prefer: 'wait' },
82
+ body: JSON.stringify({ model, input: { prompt, width: w || 1024, height: h || 1024 } }),
83
+ signal: AbortSignal.timeout(120_000),
84
+ })
85
+ if (!createRes.ok) return { error: `Replicate ${createRes.status}: ${(await createRes.text().catch(() => '')).slice(0, 300)}` }
86
+ let prediction = await createRes.json()
87
+
88
+ // If sync didn't complete, poll
89
+ if (prediction.status !== 'succeeded' && prediction.status !== 'failed' && prediction.urls?.get) {
90
+ const deadline = Date.now() + 120_000
91
+ while (prediction.status !== 'succeeded' && prediction.status !== 'failed' && Date.now() < deadline) {
92
+ await new Promise((r) => setTimeout(r, 2000))
93
+ const pollRes = await fetch(prediction.urls.get, {
94
+ headers: { Authorization: `Bearer ${cfg.apiKey}` },
95
+ signal: AbortSignal.timeout(10_000),
96
+ })
97
+ prediction = await pollRes.json()
98
+ }
99
+ }
100
+ if (prediction.status === 'failed') return { error: `Replicate failed: ${prediction.error || 'unknown'}` }
101
+ const output = Array.isArray(prediction.output) ? prediction.output[0] : prediction.output
102
+ if (typeof output === 'string') return { url: output }
103
+ return { error: 'No image in Replicate response.' }
104
+ }
105
+
106
+ async function generateFal(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
107
+ const model = cfg.model || 'fal-ai/flux/schnell'
108
+ const [w, h] = size.split('x').map(Number)
109
+ const res = await fetch(`https://fal.run/${model}`, {
110
+ method: 'POST',
111
+ headers: { 'Content-Type': 'application/json', Authorization: `Key ${cfg.apiKey}` },
112
+ body: JSON.stringify({ prompt, image_size: { width: w || 1024, height: h || 1024 }, num_images: 1 }),
113
+ signal: AbortSignal.timeout(120_000),
114
+ })
115
+ if (!res.ok) return { error: `fal.ai ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
116
+ const data = await res.json()
117
+ const imageUrl = data?.images?.[0]?.url
118
+ if (imageUrl) return { url: imageUrl }
119
+ return { error: 'No image in fal.ai response.' }
120
+ }
121
+
122
+ async function generateTogether(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
123
+ const model = cfg.model || 'black-forest-labs/FLUX.1-schnell-Free'
124
+ const [w, h] = size.split('x').map(Number)
125
+ const res = await fetch('https://api.together.xyz/v1/images/generations', {
126
+ method: 'POST',
127
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
128
+ body: JSON.stringify({ model, prompt, width: w || 1024, height: h || 1024, n: 1, response_format: 'b64_json' }),
129
+ signal: AbortSignal.timeout(120_000),
130
+ })
131
+ if (!res.ok) return { error: `Together ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
132
+ const data = await res.json()
133
+ const b64 = data?.data?.[0]?.b64_json
134
+ if (b64) return { b64 }
135
+ const url = data?.data?.[0]?.url
136
+ if (url) return { url }
137
+ return { error: 'No image in Together response.' }
138
+ }
139
+
140
+ async function generateFireworks(prompt: string, _size: string, cfg: PluginConfig): Promise<GenResult> {
141
+ const model = cfg.model || 'flux-1-schnell-fp8'
142
+ const res = await fetch(`https://api.fireworks.ai/inference/v1/workflows/accounts/fireworks/models/${model}/text_to_image`, {
143
+ method: 'POST',
144
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}`, Accept: 'image/jpeg' },
145
+ body: JSON.stringify({ prompt }),
146
+ signal: AbortSignal.timeout(120_000),
147
+ })
148
+ if (!res.ok) return { error: `Fireworks ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
149
+ // Response may be JSON with base64 array or raw image
150
+ const ct = res.headers.get('content-type') || ''
151
+ if (ct.includes('application/json')) {
152
+ const data = await res.json()
153
+ const b64 = data?.base64?.[0] ?? data?.data?.[0]?.b64_json
154
+ if (b64) return { b64 }
155
+ return { error: 'No image in Fireworks JSON response.' }
156
+ }
157
+ // Raw image bytes
158
+ const buf = Buffer.from(await res.arrayBuffer())
159
+ return { b64: buf.toString('base64') }
160
+ }
161
+
162
+ async function generateBFL(prompt: string, size: string, cfg: PluginConfig): Promise<GenResult> {
163
+ const [w, h] = size.split('x').map(Number)
164
+ const model = cfg.model || 'flux-pro-1.1'
165
+ const createRes = await fetch(`https://api.bfl.ai/v1/${model}`, {
166
+ method: 'POST',
167
+ headers: { 'Content-Type': 'application/json', 'x-key': cfg.apiKey },
168
+ body: JSON.stringify({ prompt, width: w || 1024, height: h || 1024 }),
169
+ signal: AbortSignal.timeout(15_000),
170
+ })
171
+ if (!createRes.ok) return { error: `BFL ${createRes.status}: ${(await createRes.text().catch(() => '')).slice(0, 300)}` }
172
+ const task = await createRes.json()
173
+ const pollingUrl = task?.polling_url || (task?.id ? `https://api.bfl.ai/v1/get_result?id=${task.id}` : null)
174
+ if (!pollingUrl) return { error: 'No polling URL from BFL.' }
175
+
176
+ // Poll for result
177
+ const deadline = Date.now() + 120_000
178
+ while (Date.now() < deadline) {
179
+ await new Promise((r) => setTimeout(r, 2000))
180
+ const pollRes = await fetch(pollingUrl, {
181
+ headers: { 'x-key': cfg.apiKey },
182
+ signal: AbortSignal.timeout(10_000),
183
+ })
184
+ const result = await pollRes.json()
185
+ if (result.status === 'Ready' && result.result?.sample) return { url: result.result.sample }
186
+ if (result.status === 'Error') return { error: `BFL error: ${result.result || 'unknown'}` }
187
+ }
188
+ return { error: 'BFL generation timed out.' }
189
+ }
190
+
191
+ async function generateCustom(prompt: string, size: string, quality: string, cfg: PluginConfig): Promise<GenResult> {
192
+ if (!cfg.customEndpoint) return { error: 'Custom endpoint URL not configured.' }
193
+ // Assumes OpenAI-compatible image generation API
194
+ const [w, h] = size.split('x').map(Number)
195
+ const res = await fetch(cfg.customEndpoint, {
196
+ method: 'POST',
197
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${cfg.apiKey}` },
198
+ body: JSON.stringify({ model: cfg.model || 'default', prompt, n: 1, size, width: w || 1024, height: h || 1024, quality, response_format: 'b64_json' }),
199
+ signal: AbortSignal.timeout(120_000),
200
+ })
201
+ if (!res.ok) return { error: `Custom API ${res.status}: ${(await res.text().catch(() => '')).slice(0, 300)}` }
202
+ const data = await res.json()
203
+ const b64 = data?.data?.[0]?.b64_json ?? data?.images?.[0]?.b64_json ?? data?.b64_json ?? data?.artifacts?.[0]?.base64
204
+ if (b64) return { b64 }
205
+ const url = data?.data?.[0]?.url ?? data?.images?.[0]?.url ?? data?.url
206
+ if (url) return { url }
207
+ return { error: 'No image found in custom API response.' }
208
+ }
209
+
210
+ // --- Dispatcher ---
211
+
212
+ const PROVIDERS: Record<ImageProvider, (prompt: string, size: string, quality: string, cfg: PluginConfig) => Promise<GenResult>> = {
213
+ openai: generateOpenAI,
214
+ stability: (p, s, _q, c) => generateStability(p, s, c),
215
+ replicate: (p, s, _q, c) => generateReplicate(p, s, c),
216
+ fal: (p, s, _q, c) => generateFal(p, s, c),
217
+ together: (p, s, _q, c) => generateTogether(p, s, c),
218
+ fireworks: (p, s, _q, c) => generateFireworks(p, s, c),
219
+ bfl: (p, s, _q, c) => generateBFL(p, s, c),
220
+ custom: generateCustom,
221
+ }
222
+
223
+ async function saveImageResult(result: GenResult, prompt: string, filename: string | undefined): Promise<string> {
224
+ if (result.error) return `Error: ${result.error}`
225
+
226
+ if (result.b64) {
227
+ const buf = Buffer.from(result.b64, 'base64')
228
+ const baseName = filename || `img-${Date.now()}.png`
229
+ const safeName = baseName.replace(/[^a-zA-Z0-9._-]/g, '_')
230
+ const dest = path.join(UPLOAD_DIR, safeName)
231
+ if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true })
232
+ fs.writeFileSync(dest, buf)
233
+ return `Image generated (${buf.length} bytes).\n\n![${prompt.slice(0, 60)}](/api/uploads/${safeName})\n\n[Download](/api/uploads/${safeName})`
234
+ }
235
+
236
+ if (result.url) {
237
+ // Download remote URL to uploads
238
+ try {
239
+ const res = await fetch(result.url, { signal: AbortSignal.timeout(30_000) })
240
+ if (res.ok) {
241
+ const buf = Buffer.from(await res.arrayBuffer())
242
+ const baseName = filename || `img-${Date.now()}.png`
243
+ const safeName = baseName.replace(/[^a-zA-Z0-9._-]/g, '_')
244
+ const dest = path.join(UPLOAD_DIR, safeName)
245
+ if (!fs.existsSync(UPLOAD_DIR)) fs.mkdirSync(UPLOAD_DIR, { recursive: true })
246
+ fs.writeFileSync(dest, buf)
247
+ return `Image generated (${buf.length} bytes).\n\n![${prompt.slice(0, 60)}](/api/uploads/${safeName})\n\n[Download](/api/uploads/${safeName})`
248
+ }
249
+ } catch { /* fall through to URL-only response */ }
250
+ return `Image generated: ${result.url}`
251
+ }
252
+
253
+ return 'Error: No image returned.'
254
+ }
255
+
256
+ async function executeImageGen(args: Record<string, unknown>): Promise<string> {
257
+ const normalized = normalizeToolInputArgs(args)
258
+ const prompt = String(normalized.prompt || '').trim()
259
+ if (!prompt) return 'Error: prompt is required.'
260
+
261
+ const cfg = getConfig()
262
+ if (!cfg.apiKey) return 'Error: Image generation API key not configured. Ask the user to add one in Plugin Settings > Image Generation.'
263
+
264
+ const size = String(normalized.size || cfg.defaultSize)
265
+ const quality = String(normalized.quality || 'standard')
266
+ const filename = normalized.filename as string | undefined
267
+
268
+ const generate = PROVIDERS[cfg.provider]
269
+ if (!generate) return `Error: Unknown provider "${cfg.provider}".`
270
+
271
+ try {
272
+ const result = await generate(prompt, size, quality, cfg)
273
+ return saveImageResult(result, prompt, filename)
274
+ } catch (err: unknown) {
275
+ return `Error: ${err instanceof Error ? err.message : String(err)}`
276
+ }
277
+ }
278
+
279
+ const ImageGenPlugin: Plugin = {
280
+ name: 'Image Generation',
281
+ enabledByDefault: false,
282
+ description: 'Generate images from text prompts. Supports OpenAI, Stability AI, Replicate, fal.ai, Together AI, Fireworks AI, BFL (Flux), or any OpenAI-compatible API.',
283
+ hooks: {
284
+ getCapabilityDescription: () =>
285
+ 'I can generate images from text descriptions using `generate_image`. Supports different sizes, quality levels, and providers.',
286
+ } as PluginHooks,
287
+ tools: [
288
+ {
289
+ name: 'generate_image',
290
+ description: 'Generate an image from a text prompt. The image is saved and a download link is returned. Use descriptive, detailed prompts for best results.',
291
+ parameters: {
292
+ type: 'object',
293
+ properties: {
294
+ prompt: { type: 'string', description: 'Detailed text description of the image to generate' },
295
+ size: { type: 'string', enum: ['1024x1024', '1536x1024', '1024x1536', '512x512', '768x768', '1280x720', '720x1280'], description: 'Image dimensions (default: 1024x1024)' },
296
+ quality: { type: 'string', enum: ['standard', 'hd', 'low', 'medium', 'high'], description: 'Quality level (default: standard). Primarily used by OpenAI.' },
297
+ filename: { type: 'string', description: 'Optional filename for the saved image (e.g. "hero-banner.png")' },
298
+ },
299
+ required: ['prompt'],
300
+ },
301
+ execute: async (args) => executeImageGen(args),
302
+ },
303
+ ],
304
+ ui: {
305
+ settingsFields: [
306
+ {
307
+ key: 'provider',
308
+ label: 'Provider',
309
+ type: 'select',
310
+ options: [
311
+ { value: 'openai', label: 'OpenAI (DALL-E / gpt-image)' },
312
+ { value: 'stability', label: 'Stability AI' },
313
+ { value: 'replicate', label: 'Replicate (Flux, SDXL, etc.)' },
314
+ { value: 'fal', label: 'fal.ai (Flux, SDXL, etc.)' },
315
+ { value: 'together', label: 'Together AI' },
316
+ { value: 'fireworks', label: 'Fireworks AI' },
317
+ { value: 'bfl', label: 'BFL / Black Forest Labs (Flux Pro)' },
318
+ { value: 'custom', label: 'Custom (OpenAI-compatible endpoint)' },
319
+ ],
320
+ defaultValue: 'openai',
321
+ },
322
+ {
323
+ key: 'apiKey',
324
+ label: 'API Key',
325
+ type: 'secret',
326
+ required: true,
327
+ placeholder: 'sk-... / r8_... / fal-...',
328
+ help: 'API key for the selected provider.',
329
+ },
330
+ {
331
+ key: 'model',
332
+ label: 'Model',
333
+ type: 'text',
334
+ placeholder: 'gpt-image-1 / black-forest-labs/flux-schnell / ...',
335
+ help: 'Model ID. Each provider has its own default if left blank.',
336
+ },
337
+ {
338
+ key: 'defaultSize',
339
+ label: 'Default Size',
340
+ type: 'select',
341
+ options: [
342
+ { value: '1024x1024', label: '1024x1024 (Square)' },
343
+ { value: '1536x1024', label: '1536x1024 (Landscape)' },
344
+ { value: '1024x1536', label: '1024x1536 (Portrait)' },
345
+ { value: '1280x720', label: '1280x720 (16:9)' },
346
+ { value: '512x512', label: '512x512 (Small)' },
347
+ { value: '768x768', label: '768x768 (Medium)' },
348
+ ],
349
+ defaultValue: '1024x1024',
350
+ },
351
+ {
352
+ key: 'customEndpoint',
353
+ label: 'Custom Endpoint URL',
354
+ type: 'text',
355
+ placeholder: 'https://your-api.example.com/v1/images/generations',
356
+ help: 'Only used when provider is "Custom". Should accept OpenAI-compatible image generation requests.',
357
+ },
358
+ ],
359
+ },
360
+ }
361
+
362
+ getPluginManager().registerBuiltin('image_gen', ImageGenPlugin)
363
+
364
+ export function buildImageGenTools(bctx: ToolBuildContext): StructuredToolInterface[] {
365
+ if (!bctx.hasPlugin('image_gen')) return []
366
+
367
+ return [
368
+ tool(
369
+ async (args) => executeImageGen(args),
370
+ {
371
+ name: 'generate_image',
372
+ description: ImageGenPlugin.tools![0].description,
373
+ schema: z.object({
374
+ prompt: z.string().describe('Detailed text description of the image to generate'),
375
+ size: z.enum(['1024x1024', '1536x1024', '1024x1536', '512x512', '768x768', '1280x720', '720x1280']).optional().describe('Image dimensions (default: 1024x1024)'),
376
+ quality: z.enum(['standard', 'hd', 'low', 'medium', 'high']).optional().describe('Quality level (default: standard)'),
377
+ filename: z.string().optional().describe('Optional filename for the saved image'),
378
+ }),
379
+ },
380
+ ),
381
+ ]
382
+ }
@@ -5,7 +5,7 @@ import { loadSettings, loadSessions, saveSessions, loadMcpServers } from '../sto
5
5
  import { loadRuntimeSettings } from '../runtime-settings'
6
6
  import { log } from '../logger'
7
7
  import { resolveSessionToolPolicy } from '../tool-capability-policy'
8
- import { expandToolIds } from '../tool-aliases'
8
+ import { expandPluginIds } from '../tool-aliases'
9
9
  import type { ToolContext, SessionToolsResult, ToolBuildContext } from './context'
10
10
 
11
11
  // Import all tool modules to trigger their builtin registration
@@ -33,6 +33,10 @@ import { buildDiscoveryTools } from './discovery'
33
33
  import { buildMonitorTools } from './monitor'
34
34
  import { buildSampleUITools } from './sample-ui'
35
35
  import { buildPluginCreatorTools } from './plugin-creator'
36
+ import { buildImageGenTools } from './image-gen'
37
+ import { buildEmailTools } from './email'
38
+ import { buildCalendarTools } from './calendar'
39
+ import { buildReplicateTools } from './replicate'
36
40
  import { normalizeToolInputArgs } from './normalize-tool-args'
37
41
 
38
42
  import { getPluginManager } from '../plugins'
@@ -41,34 +45,37 @@ import { jsonSchemaToZod } from '../mcp-client'
41
45
  export type { ToolContext, SessionToolsResult }
42
46
  export { sweepOrphanedBrowsers, cleanupSessionBrowser, getActiveBrowserCount, hasActiveBrowser }
43
47
 
44
- export async function buildSessionTools(cwd: string, enabledTools: string[], ctx?: ToolContext): Promise<SessionToolsResult> {
48
+ export async function buildSessionTools(cwd: string, enabledPlugins: string[], ctx?: ToolContext): Promise<SessionToolsResult> {
45
49
  const tools: StructuredToolInterface[] = []
46
50
  const cleanupFns: (() => Promise<void>)[] = []
47
-
51
+
48
52
  try {
49
53
  const runtime = loadRuntimeSettings()
50
54
  const commandTimeoutMs = runtime.shellCommandTimeoutMs
51
55
  const claudeTimeoutMs = runtime.claudeCodeTimeoutMs
52
56
  const cliProcessTimeoutMs = runtime.cliProcessTimeoutMs
53
57
  const appSettings = loadSettings()
54
- const toolPolicy = resolveSessionToolPolicy(enabledTools, appSettings)
55
- const expandedEnabledTools = expandToolIds(toolPolicy.enabledTools)
56
- const expandedBlockedTools = expandToolIds(toolPolicy.blockedTools.map((entry) => entry.tool))
57
- const blockedToolSet = new Set(expandedBlockedTools)
58
- const filteredEnabledTools = expandedEnabledTools.filter((toolId) => !blockedToolSet.has(toolId))
59
- const activeTools = filteredEnabledTools.includes('shell')
60
- && !filteredEnabledTools.includes('process')
61
- && !blockedToolSet.has('process')
62
- ? [...filteredEnabledTools, 'process']
63
- : filteredEnabledTools
64
- const activeToolSet = new Set(activeTools)
65
- const hasTool = (toolName: string) => activeToolSet.has(toolName)
58
+ const toolPolicy = resolveSessionToolPolicy(enabledPlugins, appSettings)
59
+ const expandedEnabled = expandPluginIds(toolPolicy.enabledPlugins)
60
+ const expandedBlocked = expandPluginIds(toolPolicy.blockedPlugins.map((entry) => entry.tool))
61
+ const blockedSet = new Set(expandedBlocked)
62
+ const filteredEnabled = expandedEnabled.filter((id) => !blockedSet.has(id))
63
+ const pluginManager = getPluginManager()
64
+ const activePlugins = (filteredEnabled.includes('shell')
65
+ && !filteredEnabled.includes('process')
66
+ && !blockedSet.has('process')
67
+ ? [...filteredEnabled, 'process']
68
+ : filteredEnabled).filter(t => pluginManager.isEnabled(t))
69
+ const activePluginSet = new Set(activePlugins)
70
+ const hasPlugin = (pluginName: string) => activePluginSet.has(pluginName)
71
+ /** @deprecated Use hasPlugin */
72
+ const hasTool = hasPlugin
66
73
 
67
- if (toolPolicy.blockedTools.length > 0) {
68
- log.info('session-tools', 'Capability policy blocked tool families', {
74
+ if (toolPolicy.blockedPlugins.length > 0) {
75
+ log.info('session-tools', 'Capability policy blocked plugin families', {
69
76
  sessionId: ctx?.sessionId || null,
70
77
  agentId: ctx?.agentId || null,
71
- blockedTools: toolPolicy.blockedTools.map((entry) => `${entry.tool}:${entry.reason}`),
78
+ blockedPlugins: toolPolicy.blockedPlugins.map((entry) => `${entry.tool}:${entry.reason}`),
72
79
  })
73
80
  }
74
81
 
@@ -78,14 +85,14 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
78
85
  return sessions[ctx.sessionId] || null
79
86
  }
80
87
 
81
- const readStoredDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode'): string | null => {
88
+ const readStoredDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode' | 'gemini'): string | null => {
82
89
  const session = resolveCurrentSession()
83
90
  if (!session?.delegateResumeIds || typeof session.delegateResumeIds !== 'object') return null
84
91
  const raw = session.delegateResumeIds[key]
85
92
  return typeof raw === 'string' && raw.trim() ? raw.trim() : null
86
93
  }
87
94
 
88
- const persistDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode', resumeId: string | null | undefined): void => {
95
+ const persistDelegateResumeId = (key: 'claudeCode' | 'codex' | 'opencode' | 'gemini', resumeId: string | null | undefined): void => {
89
96
  const normalized = typeof resumeId === 'string' ? resumeId.trim() : ''
90
97
  if (!normalized || !ctx?.sessionId) return
91
98
  const sessions = loadSessions()
@@ -106,6 +113,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
106
113
  const bctx: ToolBuildContext = {
107
114
  cwd,
108
115
  ctx,
116
+ hasPlugin,
109
117
  hasTool,
110
118
  cleanupFns,
111
119
  commandTimeoutMs,
@@ -114,41 +122,54 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
114
122
  persistDelegateResumeId,
115
123
  readStoredDelegateResumeId,
116
124
  resolveCurrentSession,
117
- activeTools,
125
+ activePlugins,
118
126
  }
119
127
 
120
128
  // 1. Build Native Bridge Tools (Legacy enablement)
121
- tools.push(
122
- ...buildShellTools(bctx),
123
- ...buildFileTools(bctx),
124
- ...buildEditFileTools(bctx),
125
- ...buildDelegateTools(bctx),
126
- ...buildWebTools(bctx),
127
- ...buildMemoryTools(bctx),
128
- ...buildPlatformTools(bctx),
129
- ...buildSandboxTools(bctx),
130
- ...buildChatroomTools(bctx),
131
- ...buildSubagentTools(bctx),
132
- ...buildCanvasTools(bctx),
133
- ...buildHttpTools(bctx),
134
- ...buildGitTools(bctx),
135
- ...buildWalletTools(bctx),
136
- ...buildOpenClawWorkspaceTools(bctx),
137
- ...buildScheduleTools(bctx),
138
- ...buildSessionInfoTools(bctx),
139
- ...buildOpenClawNodeTools(bctx),
140
- ...buildContextTools(bctx),
141
- ...buildConnectorTools(bctx),
142
- ...buildDiscoveryTools(bctx),
143
- ...buildMonitorTools(bctx),
144
- ...buildSampleUITools(bctx),
145
- ...buildPluginCreatorTools(bctx),
146
- )
129
+ const toolToPluginMap: Record<string, string> = {}
130
+
131
+ const nativeBuilders: Array<[string, (ctx: ToolBuildContext) => StructuredToolInterface[]]> = [
132
+ ['shell', buildShellTools],
133
+ ['files', buildFileTools],
134
+ ['edit_file', buildEditFileTools],
135
+ ['delegate', buildDelegateTools],
136
+ ['web', buildWebTools],
137
+ ['memory', buildMemoryTools],
138
+ ['manage_platform', buildPlatformTools],
139
+ ['sandbox', buildSandboxTools],
140
+ ['manage_chatrooms', buildChatroomTools],
141
+ ['spawn_subagent', buildSubagentTools],
142
+ ['canvas', buildCanvasTools],
143
+ ['http', buildHttpTools],
144
+ ['git', buildGitTools],
145
+ ['wallet', buildWalletTools],
146
+ ['openclaw_workspace', buildOpenClawWorkspaceTools],
147
+ ['schedule', buildScheduleTools],
148
+ ['manage_sessions', buildSessionInfoTools],
149
+ ['openclaw_nodes', buildOpenClawNodeTools],
150
+ ['context_mgmt', buildContextTools],
151
+ ['manage_connectors', buildConnectorTools],
152
+ ['discovery', buildDiscoveryTools],
153
+ ['monitor', buildMonitorTools],
154
+ ['sample_ui', buildSampleUITools],
155
+ ['plugin_creator', buildPluginCreatorTools],
156
+ ['image_gen', buildImageGenTools],
157
+ ['email', buildEmailTools],
158
+ ['calendar', buildCalendarTools],
159
+ ['replicate', buildReplicateTools],
160
+ ]
161
+
162
+ for (const [pluginId, builder] of nativeBuilders) {
163
+ const builtTools = builder(bctx)
164
+ for (const t of builtTools) {
165
+ toolToPluginMap[t.name] = pluginId
166
+ }
167
+ tools.push(...builtTools)
168
+ }
147
169
 
148
170
  // 2. Build Plugin Tools (Built-in + External)
149
171
  try {
150
- const pluginManager = getPluginManager()
151
- const pluginTools = pluginManager.getTools(activeTools)
172
+ const pluginTools = pluginManager.getTools(activePlugins)
152
173
  const existingNames = new Set(tools.map((t) => t.name))
153
174
 
154
175
  for (const entry of pluginTools) {
@@ -161,6 +182,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
161
182
  continue
162
183
  }
163
184
  existingNames.add(pt.name)
185
+ toolToPluginMap[pt.name] = entry.pluginId
164
186
 
165
187
  tools.push(
166
188
  tool(
@@ -208,6 +230,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
208
230
  const mcpLcTools = await mcpToolsToLangChain(conn.client, config.name)
209
231
  for (const t of mcpLcTools) {
210
232
  if (!disabledMcpToolNames.has(t.name)) {
233
+ toolToPluginMap[t.name] = `mcp:${serverId}`
211
234
  tools.push(t)
212
235
  }
213
236
  }
@@ -224,6 +247,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
224
247
  }
225
248
 
226
249
  // 4. Always available: request_tool_access
250
+ toolToPluginMap['request_tool_access'] = '_system'
227
251
  tools.push(
228
252
  tool(
229
253
  async (args) => {
@@ -255,6 +279,7 @@ export async function buildSessionTools(cwd: string, enabledTools: string[], ctx
255
279
  try { await fn() } catch { /* ignore */ }
256
280
  }
257
281
  },
282
+ toolToPluginMap,
258
283
  }
259
284
  } catch (err: any) {
260
285
  console.error('[session-tools] buildSessionTools critical failure:', err.message)