osborn 0.1.6 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +8 -1
- package/dist/bridge-llm.d.ts +22 -0
- package/dist/bridge-llm.js +39 -0
- package/dist/claude-handler.d.ts +6 -0
- package/dist/claude-handler.js +43 -1
- package/dist/claude-llm.d.ts +128 -0
- package/dist/claude-llm.js +623 -0
- package/dist/codex-llm.d.ts +40 -0
- package/dist/codex-llm.js +144 -0
- package/dist/config.d.ts +227 -1
- package/dist/config.js +775 -8
- package/dist/conversation-brain.d.ts +92 -0
- package/dist/conversation-brain.js +360 -0
- package/dist/fast-brain.d.ts +122 -0
- package/dist/fast-brain.js +1404 -0
- package/dist/index.js +1997 -322
- package/dist/prompts.d.ts +19 -0
- package/dist/prompts.js +610 -0
- package/dist/session-access.d.ts +399 -0
- package/dist/session-access.js +775 -0
- package/dist/smithery-proxy.d.ts +57 -0
- package/dist/smithery-proxy.js +195 -0
- package/dist/status-manager.d.ts +90 -0
- package/dist/status-manager.js +187 -0
- package/dist/voice-io.d.ts +70 -0
- package/dist/voice-io.js +152 -0
- package/package.json +17 -6
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claude LLM Wrapper for LiveKit Agents
|
|
3
|
+
*
|
|
4
|
+
* Wraps the Claude Agent SDK (@anthropic-ai/claude-agent-sdk) to work
|
|
5
|
+
* with LiveKit's AgentSession as an LLM provider.
|
|
6
|
+
*
|
|
7
|
+
* Flow: User speaks → STT → ClaudeLLM (Agent SDK) → TTS → User hears
|
|
8
|
+
*/
|
|
9
|
+
import { llm, shortuuid, DEFAULT_API_CONNECT_OPTIONS } from '@livekit/agents';
|
|
10
|
+
import { query } from '@anthropic-ai/claude-agent-sdk';
|
|
11
|
+
import { EventEmitter } from 'events';
|
|
12
|
+
import { saveSessionMetadata } from './config.js';
|
|
13
|
+
import { getResearchSystemPrompt } from './prompts.js';
|
|
14
|
+
/**
|
|
15
|
+
* Strip markdown formatting for TTS (text-to-speech)
|
|
16
|
+
* Removes **bold**, ##headers, ```code```, etc. so TTS doesn't read them literally
|
|
17
|
+
*/
|
|
18
|
+
function stripMarkdownForTTS(text) {
|
|
19
|
+
return text
|
|
20
|
+
// Remove code blocks (``` ... ```)
|
|
21
|
+
.replace(/```[\s\S]*?```/g, ' [code block] ')
|
|
22
|
+
// Remove inline code (` ... `)
|
|
23
|
+
.replace(/`([^`]+)`/g, '$1')
|
|
24
|
+
// Remove bold (**text** or __text__)
|
|
25
|
+
.replace(/\*\*([^*]+)\*\*/g, '$1')
|
|
26
|
+
.replace(/__([^_]+)__/g, '$1')
|
|
27
|
+
// Remove italic (*text* or _text_) - be careful not to match bullet points
|
|
28
|
+
.replace(/(?<!\*)\*([^*]+)\*(?!\*)/g, '$1')
|
|
29
|
+
.replace(/(?<!_)_([^_]+)_(?!_)/g, '$1')
|
|
30
|
+
// Remove headers (# ## ### etc)
|
|
31
|
+
.replace(/^#{1,6}\s+/gm, '')
|
|
32
|
+
// Remove bullet points but keep content
|
|
33
|
+
.replace(/^[\s]*[-*+]\s+/gm, '')
|
|
34
|
+
// Remove numbered lists but keep content
|
|
35
|
+
.replace(/^[\s]*\d+\.\s+/gm, '')
|
|
36
|
+
// Remove horizontal rules
|
|
37
|
+
.replace(/^[-*_]{3,}$/gm, '')
|
|
38
|
+
// Remove links [text](url) -> text
|
|
39
|
+
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
|
|
40
|
+
// Remove images 
|
|
41
|
+
.replace(/!\[([^\]]*)\]\([^)]+\)/g, '$1')
|
|
42
|
+
// Remove blockquotes
|
|
43
|
+
.replace(/^>\s+/gm, '')
|
|
44
|
+
// Clean up multiple spaces/newlines
|
|
45
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
46
|
+
.replace(/ +/g, ' ')
|
|
47
|
+
.trim();
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Summarize text for TTS - create short spoken summaries
|
|
51
|
+
* Full output goes to frontend, this condensed version is spoken
|
|
52
|
+
*/
|
|
53
|
+
function summarizeForTTS(text, maxLength = 500) {
|
|
54
|
+
// First strip markdown
|
|
55
|
+
let summary = stripMarkdownForTTS(text);
|
|
56
|
+
// Remove file paths (keep just filename)
|
|
57
|
+
summary = summary.replace(/\/[\w\-\.\/]+\/([\w\-\.]+)/g, '$1');
|
|
58
|
+
// Remove code block placeholders if too many
|
|
59
|
+
const codeBlockCount = (summary.match(/\[code block\]/g) || []).length;
|
|
60
|
+
if (codeBlockCount > 1) {
|
|
61
|
+
summary = summary.replace(/\[code block\]/g, '').replace(/\s+/g, ' ');
|
|
62
|
+
summary = summary.trim() + ` I've included ${codeBlockCount} code examples.`;
|
|
63
|
+
}
|
|
64
|
+
// If still too long, take first sentence(s) up to maxLength
|
|
65
|
+
if (summary.length > maxLength) {
|
|
66
|
+
// Try to break at sentence boundaries
|
|
67
|
+
const sentences = summary.match(/[^.!?]+[.!?]+/g) || [summary];
|
|
68
|
+
let result = '';
|
|
69
|
+
for (const sentence of sentences) {
|
|
70
|
+
if ((result + sentence).length <= maxLength) {
|
|
71
|
+
result += sentence;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// If no complete sentence fits, truncate with ellipsis
|
|
78
|
+
if (!result) {
|
|
79
|
+
result = summary.substring(0, maxLength - 3) + '...';
|
|
80
|
+
}
|
|
81
|
+
summary = result.trim();
|
|
82
|
+
}
|
|
83
|
+
return summary || 'Done.';
|
|
84
|
+
}
|
|
85
|
+
// Research mode tools — full research capabilities
|
|
86
|
+
const RESEARCH_TOOLS = [
|
|
87
|
+
'Read', 'Write', 'Edit', 'Glob', 'Grep',
|
|
88
|
+
'Bash', 'WebSearch', 'WebFetch',
|
|
89
|
+
'LSP', 'Task', 'TodoWrite',
|
|
90
|
+
];
|
|
91
|
+
/**
|
|
92
|
+
* Claude LLM - Wraps Claude Agent SDK for LiveKit
|
|
93
|
+
* Research mode: reads anything, writes only to session workspace
|
|
94
|
+
*/
|
|
95
|
+
export class ClaudeLLM extends llm.LLM {
|
|
96
|
+
#opts;
|
|
97
|
+
#sessionId = null;
|
|
98
|
+
#eventEmitter;
|
|
99
|
+
#resumeSessionId = null;
|
|
100
|
+
#continueSession = false;
|
|
101
|
+
#mcpServers = {};
|
|
102
|
+
// File checkpointing - stores checkpoint UUIDs for rewinding file changes
|
|
103
|
+
#checkpoints = [];
|
|
104
|
+
#latestCheckpoint = null;
|
|
105
|
+
// Pending permission request (for voice approval flow)
|
|
106
|
+
#pendingPermission = null;
|
|
107
|
+
constructor(opts = {}) {
|
|
108
|
+
super();
|
|
109
|
+
// Session resume/continue options
|
|
110
|
+
this.#resumeSessionId = opts.resumeSessionId || null;
|
|
111
|
+
this.#continueSession = opts.continueSession || false;
|
|
112
|
+
// MCP servers
|
|
113
|
+
this.#mcpServers = opts.mcpServers || {};
|
|
114
|
+
this.#opts = {
|
|
115
|
+
workingDirectory: opts.workingDirectory || process.cwd(),
|
|
116
|
+
permissionMode: opts.permissionMode || 'default',
|
|
117
|
+
allowedTools: opts.allowedTools || RESEARCH_TOOLS,
|
|
118
|
+
resumeSessionId: this.#resumeSessionId || undefined,
|
|
119
|
+
continueSession: this.#continueSession,
|
|
120
|
+
mcpServers: this.#mcpServers,
|
|
121
|
+
};
|
|
122
|
+
this.#eventEmitter = opts.eventEmitter || new EventEmitter();
|
|
123
|
+
console.log('🟠 ClaudeLLM initialized (Research Mode)');
|
|
124
|
+
console.log(` 📁 Working dir: ${this.#opts.workingDirectory}`);
|
|
125
|
+
console.log(` 🔧 Allowed tools: ${this.#opts.allowedTools?.join(', ')}`);
|
|
126
|
+
const mcpCount = Object.keys(this.#mcpServers).length;
|
|
127
|
+
if (mcpCount > 0) {
|
|
128
|
+
console.log(` 🔌 MCP servers: ${Object.keys(this.#mcpServers).join(', ')}`);
|
|
129
|
+
}
|
|
130
|
+
if (this.#resumeSessionId) {
|
|
131
|
+
console.log(` 🔄 Resuming session: ${this.#resumeSessionId}`);
|
|
132
|
+
}
|
|
133
|
+
else if (this.#continueSession) {
|
|
134
|
+
console.log(` 🔄 Continuing most recent session`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Respond to a pending permission request
|
|
139
|
+
* Call this after receiving 'permission_request' event
|
|
140
|
+
*/
|
|
141
|
+
respondToPermission(allow, message) {
|
|
142
|
+
if (this.#pendingPermission) {
|
|
143
|
+
const input = this.#pendingPermission.input;
|
|
144
|
+
if (allow) {
|
|
145
|
+
this.#pendingPermission.resolve({
|
|
146
|
+
behavior: 'allow',
|
|
147
|
+
updatedInput: input, // Pass through original input
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
this.#pendingPermission.resolve({
|
|
152
|
+
behavior: 'deny',
|
|
153
|
+
message: message || 'User denied permission',
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
this.#pendingPermission = null;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Check if there's a pending permission request
|
|
161
|
+
*/
|
|
162
|
+
hasPendingPermission() {
|
|
163
|
+
return this.#pendingPermission !== null;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Get pending permission details
|
|
167
|
+
*/
|
|
168
|
+
getPendingPermission() {
|
|
169
|
+
if (this.#pendingPermission) {
|
|
170
|
+
return { toolName: this.#pendingPermission.toolName, input: this.#pendingPermission.input };
|
|
171
|
+
}
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
// ============================================================
|
|
175
|
+
// MCP SERVER MANAGEMENT - Runtime enable/disable MCP servers
|
|
176
|
+
// ============================================================
|
|
177
|
+
/**
|
|
178
|
+
* Get all currently enabled MCP servers
|
|
179
|
+
*/
|
|
180
|
+
getMcpServers() {
|
|
181
|
+
return { ...this.#mcpServers };
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Get list of enabled MCP server keys
|
|
185
|
+
*/
|
|
186
|
+
getEnabledMcpServerKeys() {
|
|
187
|
+
return Object.keys(this.#mcpServers);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Replace all MCP servers at once
|
|
191
|
+
*/
|
|
192
|
+
setMcpServers(servers) {
|
|
193
|
+
this.#mcpServers = { ...servers };
|
|
194
|
+
this.#opts.mcpServers = this.#mcpServers;
|
|
195
|
+
console.log(`🔌 MCP servers updated: ${Object.keys(servers).join(', ') || 'none'}`);
|
|
196
|
+
this.#eventEmitter.emit('mcp_servers_changed', {
|
|
197
|
+
enabledKeys: Object.keys(this.#mcpServers),
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Enable a single MCP server
|
|
202
|
+
*/
|
|
203
|
+
enableMcpServer(key, config) {
|
|
204
|
+
this.#mcpServers[key] = config;
|
|
205
|
+
this.#opts.mcpServers = this.#mcpServers;
|
|
206
|
+
console.log(`🔌 MCP server enabled: ${key}`);
|
|
207
|
+
this.#eventEmitter.emit('mcp_servers_changed', {
|
|
208
|
+
enabledKeys: Object.keys(this.#mcpServers),
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Disable a single MCP server
|
|
213
|
+
*/
|
|
214
|
+
disableMcpServer(key) {
|
|
215
|
+
delete this.#mcpServers[key];
|
|
216
|
+
this.#opts.mcpServers = this.#mcpServers;
|
|
217
|
+
console.log(`🔌 MCP server disabled: ${key}`);
|
|
218
|
+
this.#eventEmitter.emit('mcp_servers_changed', {
|
|
219
|
+
enabledKeys: Object.keys(this.#mcpServers),
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
label() {
|
|
223
|
+
return 'claude.agent-sdk';
|
|
224
|
+
}
|
|
225
|
+
get model() {
|
|
226
|
+
return this.#opts.model || 'claude-sonnet-4-6';
|
|
227
|
+
}
|
|
228
|
+
get sessionId() {
|
|
229
|
+
return this.#sessionId;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Set session ID to resume a specific conversation
|
|
233
|
+
* Call this before sending the first message to resume from a previous session
|
|
234
|
+
*/
|
|
235
|
+
setResumeSessionId(sessionId) {
|
|
236
|
+
this.#resumeSessionId = sessionId;
|
|
237
|
+
// CRITICAL: Sync to opts so ClaudeLLMStream.run() picks up the resume ID
|
|
238
|
+
this.#opts.resumeSessionId = sessionId || undefined;
|
|
239
|
+
if (sessionId) {
|
|
240
|
+
console.log(`🔄 Will resume session: ${sessionId}`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Reset state for mid-conversation session switch
|
|
245
|
+
* Clears pending permissions and resets conversation tracking
|
|
246
|
+
*/
|
|
247
|
+
resetForSessionSwitch() {
|
|
248
|
+
// Clear any pending permission request from previous session
|
|
249
|
+
if (this.#pendingPermission) {
|
|
250
|
+
// Deny the pending permission to clean up
|
|
251
|
+
this.#pendingPermission.resolve({
|
|
252
|
+
behavior: 'deny',
|
|
253
|
+
message: 'Session switched - permission request cancelled',
|
|
254
|
+
});
|
|
255
|
+
this.#pendingPermission = null;
|
|
256
|
+
}
|
|
257
|
+
// Clear session resume state so new resume can take effect
|
|
258
|
+
this.#resumeSessionId = null;
|
|
259
|
+
this.#continueSession = false;
|
|
260
|
+
this.#opts.resumeSessionId = undefined;
|
|
261
|
+
this.#opts.continueSession = false;
|
|
262
|
+
this.#sessionId = null;
|
|
263
|
+
// Clear checkpoints from previous session
|
|
264
|
+
this.#checkpoints = [];
|
|
265
|
+
this.#latestCheckpoint = null;
|
|
266
|
+
// Emit event for listeners
|
|
267
|
+
this.#eventEmitter.emit('session_reset');
|
|
268
|
+
console.log('🔄 LLM state reset for session switch');
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Enable "continue" mode - resumes most recent session
|
|
272
|
+
*/
|
|
273
|
+
setContinueSession(enabled) {
|
|
274
|
+
this.#continueSession = enabled;
|
|
275
|
+
this.#opts.continueSession = enabled;
|
|
276
|
+
if (enabled) {
|
|
277
|
+
console.log(`🔄 Will continue most recent session`);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Check if this instance is configured to resume a session
|
|
282
|
+
*/
|
|
283
|
+
get isResumingSession() {
|
|
284
|
+
return !!(this.#resumeSessionId || this.#continueSession);
|
|
285
|
+
}
|
|
286
|
+
get events() {
|
|
287
|
+
return this.#eventEmitter;
|
|
288
|
+
}
|
|
289
|
+
// ============================================================
|
|
290
|
+
// FILE CHECKPOINTING - Track and rewind file changes
|
|
291
|
+
// ============================================================
|
|
292
|
+
/**
|
|
293
|
+
* Capture a checkpoint UUID for potential file rewind
|
|
294
|
+
* Called internally when receiving user message UUIDs from the SDK
|
|
295
|
+
*/
|
|
296
|
+
captureCheckpoint(checkpointId) {
|
|
297
|
+
this.#checkpoints.push(checkpointId);
|
|
298
|
+
this.#latestCheckpoint = checkpointId;
|
|
299
|
+
console.log(`📍 Checkpoint captured: ${checkpointId.substring(0, 8)}...`);
|
|
300
|
+
this.#eventEmitter.emit('checkpoint_captured', { checkpointId });
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Get the most recent checkpoint UUID
|
|
304
|
+
* Use this to rewind all file changes back to the beginning
|
|
305
|
+
*/
|
|
306
|
+
getLatestCheckpoint() {
|
|
307
|
+
return this.#latestCheckpoint;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Get the first checkpoint UUID (initial state)
|
|
311
|
+
* Rewinding to this restores all files to their original state
|
|
312
|
+
*/
|
|
313
|
+
getFirstCheckpoint() {
|
|
314
|
+
return this.#checkpoints.length > 0 ? this.#checkpoints[0] : null;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Get all captured checkpoint UUIDs
|
|
318
|
+
* Ordered from oldest to newest
|
|
319
|
+
*/
|
|
320
|
+
getCheckpoints() {
|
|
321
|
+
return [...this.#checkpoints];
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Clear all captured checkpoints
|
|
325
|
+
* Call this when starting a new session
|
|
326
|
+
*/
|
|
327
|
+
clearCheckpoints() {
|
|
328
|
+
this.#checkpoints = [];
|
|
329
|
+
this.#latestCheckpoint = null;
|
|
330
|
+
console.log('🧹 Checkpoints cleared');
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Check if checkpoints are available
|
|
334
|
+
*/
|
|
335
|
+
hasCheckpoints() {
|
|
336
|
+
return this.#checkpoints.length > 0;
|
|
337
|
+
}
|
|
338
|
+
chat({ chatCtx, toolCtx, connOptions = DEFAULT_API_CONNECT_OPTIONS, }) {
|
|
339
|
+
return new ClaudeLLMStream(this, {
|
|
340
|
+
chatCtx,
|
|
341
|
+
toolCtx,
|
|
342
|
+
connOptions,
|
|
343
|
+
opts: this.#opts,
|
|
344
|
+
sessionId: this.#sessionId,
|
|
345
|
+
onSessionId: (id) => {
|
|
346
|
+
const isFirst = !this.#sessionId;
|
|
347
|
+
this.#sessionId = id;
|
|
348
|
+
if (isFirst) {
|
|
349
|
+
this.#eventEmitter.emit('session_id', { sessionId: id });
|
|
350
|
+
}
|
|
351
|
+
},
|
|
352
|
+
eventEmitter: this.#eventEmitter,
|
|
353
|
+
// Pass checkpoint capture handler
|
|
354
|
+
onCheckpoint: (checkpointId) => {
|
|
355
|
+
this.captureCheckpoint(checkpointId);
|
|
356
|
+
},
|
|
357
|
+
// Pass permission handler for canUseTool callback
|
|
358
|
+
onPermissionRequest: (toolName, input) => {
|
|
359
|
+
return new Promise((resolve) => {
|
|
360
|
+
this.#pendingPermission = { toolName, input, resolve };
|
|
361
|
+
console.log(`⚠️ Permission request: ${toolName}`);
|
|
362
|
+
this.#eventEmitter.emit('permission_request', { toolName, input });
|
|
363
|
+
});
|
|
364
|
+
},
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* Claude LLM Stream - Runs Claude Agent SDK query() and streams results
|
|
370
|
+
*/
|
|
371
|
+
class ClaudeLLMStream extends llm.LLMStream {
|
|
372
|
+
#opts;
|
|
373
|
+
#sessionId;
|
|
374
|
+
#onSessionId;
|
|
375
|
+
#eventEmitter;
|
|
376
|
+
#onPermissionRequest;
|
|
377
|
+
#onCheckpoint;
|
|
378
|
+
constructor(llmInstance, { chatCtx, toolCtx, connOptions, opts, sessionId, onSessionId, eventEmitter, onCheckpoint, onPermissionRequest, }) {
|
|
379
|
+
super(llmInstance, { chatCtx, toolCtx, connOptions });
|
|
380
|
+
this.#opts = opts;
|
|
381
|
+
this.#sessionId = sessionId;
|
|
382
|
+
this.#onSessionId = onSessionId;
|
|
383
|
+
this.#eventEmitter = eventEmitter;
|
|
384
|
+
this.#onCheckpoint = onCheckpoint;
|
|
385
|
+
this.#onPermissionRequest = onPermissionRequest;
|
|
386
|
+
}
|
|
387
|
+
async run() {
|
|
388
|
+
const requestId = `claude_${shortuuid()}`;
|
|
389
|
+
try {
|
|
390
|
+
// Extract user's message from chat context
|
|
391
|
+
// ChatContext has .items which are ChatItem[] (ChatMessage | FunctionCall | FunctionCallOutput)
|
|
392
|
+
const items = this.chatCtx.items;
|
|
393
|
+
// Find the last user message
|
|
394
|
+
let userText = '';
|
|
395
|
+
for (let i = items.length - 1; i >= 0; i--) {
|
|
396
|
+
const item = items[i];
|
|
397
|
+
if (item.type === 'message' && item.role === 'user') {
|
|
398
|
+
// Content is ChatContent[] = (ImageContent | AudioContent | string)[]
|
|
399
|
+
if (Array.isArray(item.content)) {
|
|
400
|
+
userText = item.content
|
|
401
|
+
.filter((c) => typeof c === 'string')
|
|
402
|
+
.join('\n');
|
|
403
|
+
}
|
|
404
|
+
break;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
if (!userText.trim()) {
|
|
408
|
+
this.queue.put({
|
|
409
|
+
id: requestId,
|
|
410
|
+
delta: { role: 'assistant', content: "I didn't catch that. Could you repeat?" },
|
|
411
|
+
});
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
console.log(`🎤 User: "${userText.substring(0, 100)}${userText.length > 100 ? '...' : ''}"`);
|
|
415
|
+
// Build Claude Agent SDK options
|
|
416
|
+
const resumeSessionId = this.#opts.resumeSessionId;
|
|
417
|
+
const continueSession = this.#opts.continueSession;
|
|
418
|
+
// Session workspace path for system prompt — only available after SDK assigns a real session ID
|
|
419
|
+
const sessionId = this.#sessionId || this.#opts.resumeSessionId || null;
|
|
420
|
+
const workspacePath = sessionId
|
|
421
|
+
? (this.#opts.workingDirectory
|
|
422
|
+
? `${this.#opts.workingDirectory}/.osborn/sessions/${sessionId}/`
|
|
423
|
+
: `.osborn/sessions/${sessionId}/`)
|
|
424
|
+
: null;
|
|
425
|
+
// Build allowedTools with MCP wildcard patterns
|
|
426
|
+
const mcpKeys = Object.keys(this.#opts.mcpServers || {});
|
|
427
|
+
const mcpPatterns = mcpKeys.map(key => `mcp__${key}__*`);
|
|
428
|
+
const allowedTools = [
|
|
429
|
+
...(this.#opts.allowedTools || []),
|
|
430
|
+
...mcpPatterns,
|
|
431
|
+
];
|
|
432
|
+
const sdkOptions = {
|
|
433
|
+
cwd: this.#opts.workingDirectory,
|
|
434
|
+
permissionMode: this.#opts.permissionMode,
|
|
435
|
+
allowedTools,
|
|
436
|
+
model: this.#opts.model || 'claude-sonnet-4-6',
|
|
437
|
+
enableFileCheckpointing: true,
|
|
438
|
+
extraArgs: { 'replay-user-messages': null },
|
|
439
|
+
...(resumeSessionId && { resume: resumeSessionId }),
|
|
440
|
+
...(continueSession && !resumeSessionId && { continue: true }),
|
|
441
|
+
...(this.#sessionId && !resumeSessionId && !continueSession && { resume: this.#sessionId }),
|
|
442
|
+
...(mcpKeys.length > 0 && {
|
|
443
|
+
mcpServers: this.#opts.mcpServers,
|
|
444
|
+
}),
|
|
445
|
+
...(mcpKeys.length > 0 && (() => {
|
|
446
|
+
for (const [key, cfg] of Object.entries(this.#opts.mcpServers || {})) {
|
|
447
|
+
const cfgType = cfg.type || 'stdio';
|
|
448
|
+
console.log(`🔌 SDK query MCP: ${key} [type=${cfgType}]`);
|
|
449
|
+
}
|
|
450
|
+
return {};
|
|
451
|
+
})()),
|
|
452
|
+
// Research mode system prompt — always injected
|
|
453
|
+
systemPrompt: getResearchSystemPrompt(workspacePath),
|
|
454
|
+
canUseTool: async (toolName, input, _options) => {
|
|
455
|
+
// Auto-approve writes to session workspace (but block spec.md and library/ — fast brain manages those)
|
|
456
|
+
if (toolName === 'Write' || toolName === 'Edit') {
|
|
457
|
+
const filePath = String(input?.file_path || '');
|
|
458
|
+
if (filePath.includes('.osborn/sessions/') || filePath.includes('.osborn/research/')) {
|
|
459
|
+
// Block writes to spec.md and library/ — the fast brain manages these
|
|
460
|
+
const fileName = filePath.split('/').pop() || '';
|
|
461
|
+
if (fileName === 'spec.md' || filePath.includes('/library/')) {
|
|
462
|
+
console.log(`🚫 Blocked research agent write to managed file: ${filePath} (fast brain handles spec.md and library/)`);
|
|
463
|
+
return { behavior: 'deny', message: 'spec.md and library/ are managed by the fast brain sub-agent. Do NOT write to them. Return your findings in your response text — the fast brain will organize them into spec.md and library/ automatically.' };
|
|
464
|
+
}
|
|
465
|
+
console.log(`✅ Auto-approved ${toolName} to workspace: ${filePath}`);
|
|
466
|
+
return { behavior: 'allow', updatedInput: input };
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
// Auto-approve AskUserQuestion — research agent should freely ask clarifying questions
|
|
470
|
+
if (toolName === 'AskUserQuestion') {
|
|
471
|
+
console.log(`✅ Auto-approved ${toolName}`);
|
|
472
|
+
return { behavior: 'allow', updatedInput: input };
|
|
473
|
+
}
|
|
474
|
+
// Auto-deny tools the research agent should never use
|
|
475
|
+
if (toolName === 'EnterPlanMode' || toolName === 'ExitPlanMode') {
|
|
476
|
+
console.log(`🚫 Auto-denied ${toolName} (not used in research mode)`);
|
|
477
|
+
return { behavior: 'deny', message: 'Research mode does not use plan mode. Just proceed with the research directly.' };
|
|
478
|
+
}
|
|
479
|
+
console.log(`⚠️ Permission needed: ${toolName}`);
|
|
480
|
+
return this.#onPermissionRequest(toolName, input);
|
|
481
|
+
},
|
|
482
|
+
hooks: {
|
|
483
|
+
PreToolUse: [{
|
|
484
|
+
matcher: '.*',
|
|
485
|
+
hooks: [async (input) => {
|
|
486
|
+
const toolName = input?.tool_name || 'unknown';
|
|
487
|
+
const toolInput = input?.tool_input || {};
|
|
488
|
+
// Safety: block Write/Edit outside session workspace
|
|
489
|
+
if (toolName === 'Write' || toolName === 'Edit') {
|
|
490
|
+
const filePath = String(toolInput.file_path || '');
|
|
491
|
+
if (filePath && !filePath.includes('.osborn/sessions/') && !filePath.includes('.osborn/research/')) {
|
|
492
|
+
console.log(`🚫 Research mode: blocked write to ${filePath}`);
|
|
493
|
+
this.#eventEmitter.emit('tool_blocked', { name: toolName, reason: 'Research mode: writes restricted to session workspace' });
|
|
494
|
+
return { decision: 'block', reason: 'Research mode: write to .osborn/sessions/ only.' };
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
console.log(`🔧 Claude: ${toolName}`);
|
|
498
|
+
this.#eventEmitter.emit('tool_use', { name: toolName, input: toolInput });
|
|
499
|
+
return {};
|
|
500
|
+
}]
|
|
501
|
+
}],
|
|
502
|
+
PostToolUse: [{
|
|
503
|
+
matcher: '.*',
|
|
504
|
+
hooks: [async (input) => {
|
|
505
|
+
const toolName = input?.tool_name || 'unknown';
|
|
506
|
+
const toolInput = input?.tool_input || {};
|
|
507
|
+
const toolResponse = input?.tool_response; // Capture actual tool output for fast brain processing
|
|
508
|
+
console.log(`✅ Done: ${toolName}`);
|
|
509
|
+
this.#eventEmitter.emit('tool_result', { name: toolName, input: toolInput, response: toolResponse });
|
|
510
|
+
return {};
|
|
511
|
+
}]
|
|
512
|
+
}]
|
|
513
|
+
}
|
|
514
|
+
};
|
|
515
|
+
// Run Claude Agent SDK query() and stream results
|
|
516
|
+
let hasOutput = false;
|
|
517
|
+
let fullResponse = ''; // Collect full response for frontend
|
|
518
|
+
for await (const message of query({ prompt: userText, options: sdkOptions })) {
|
|
519
|
+
// Capture session ID for context continuity
|
|
520
|
+
if (message.type === 'system' && message.subtype === 'init') {
|
|
521
|
+
// Log MCP server connection status
|
|
522
|
+
const mcpServers = message.mcp_servers;
|
|
523
|
+
if (mcpServers && Array.isArray(mcpServers)) {
|
|
524
|
+
for (const s of mcpServers) {
|
|
525
|
+
const status = s.status === 'connected' ? '✅' : '❌';
|
|
526
|
+
console.log(`${status} MCP server ${s.name}: ${s.status}`);
|
|
527
|
+
if (s.status !== 'connected') {
|
|
528
|
+
console.log(` 🔍 MCP error:`, JSON.stringify(s));
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
const newSessionId = message.session_id;
|
|
533
|
+
if (newSessionId) {
|
|
534
|
+
this.#onSessionId(newSessionId);
|
|
535
|
+
const isNewSession = !this.#sessionId;
|
|
536
|
+
if (isNewSession) {
|
|
537
|
+
console.log(`📋 New session: ${newSessionId}`);
|
|
538
|
+
}
|
|
539
|
+
this.#sessionId = newSessionId;
|
|
540
|
+
// Save session metadata for new sessions
|
|
541
|
+
if (isNewSession && this.#opts.workingDirectory) {
|
|
542
|
+
saveSessionMetadata(this.#opts.workingDirectory, {
|
|
543
|
+
sessionId: newSessionId,
|
|
544
|
+
lastUpdated: new Date().toISOString(),
|
|
545
|
+
projectPath: this.#opts.workingDirectory,
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
// Verify session resume succeeded (if we requested a specific session)
|
|
549
|
+
const requestedResumeId = this.#opts.resumeSessionId;
|
|
550
|
+
if (requestedResumeId && newSessionId !== requestedResumeId) {
|
|
551
|
+
console.error(`❌ Session resume FAILED: Expected ${requestedResumeId.substring(0, 8)}..., got ${newSessionId.substring(0, 8)}...`);
|
|
552
|
+
this.#eventEmitter.emit('session_resume_failed', {
|
|
553
|
+
requestedSessionId: requestedResumeId,
|
|
554
|
+
actualSessionId: newSessionId,
|
|
555
|
+
});
|
|
556
|
+
}
|
|
557
|
+
else if (requestedResumeId && newSessionId === requestedResumeId) {
|
|
558
|
+
console.log(`✅ Session resumed successfully: ${newSessionId.substring(0, 8)}...`);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
// Capture checkpoint UUIDs from user messages (for file rewind capability)
|
|
563
|
+
// Per SDK docs: user messages include a UUID that can be used as a restore point
|
|
564
|
+
if (message.type === 'user' && message.uuid) {
|
|
565
|
+
const checkpointId = message.uuid;
|
|
566
|
+
this.#onCheckpoint(checkpointId);
|
|
567
|
+
}
|
|
568
|
+
// Stream text chunks
|
|
569
|
+
if (message.type === 'assistant' && message.message?.content) {
|
|
570
|
+
for (const block of message.message.content) {
|
|
571
|
+
if (block.type === 'text' && block.text) {
|
|
572
|
+
hasOutput = true;
|
|
573
|
+
const rawText = block.text;
|
|
574
|
+
// Emit RAW text to frontend (for chat bubbles with full formatting)
|
|
575
|
+
this.#eventEmitter.emit('assistant_text', { text: rawText });
|
|
576
|
+
// Collect for final TTS summary
|
|
577
|
+
fullResponse += rawText + ' ';
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
// Final result
|
|
582
|
+
if (message.type === 'result' && message.result) {
|
|
583
|
+
const rawResult = message.result;
|
|
584
|
+
// Emit RAW result to frontend
|
|
585
|
+
this.#eventEmitter.emit('assistant_result', { text: rawResult });
|
|
586
|
+
if (!hasOutput) {
|
|
587
|
+
fullResponse = rawResult;
|
|
588
|
+
hasOutput = true;
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
// Send SUMMARIZED output to TTS (spoken)
|
|
593
|
+
if (hasOutput && fullResponse.trim()) {
|
|
594
|
+
const ttsText = summarizeForTTS(fullResponse.trim());
|
|
595
|
+
console.log(`🔊 TTS (summarized ${fullResponse.length} → ${ttsText.length} chars): "${ttsText.substring(0, 80)}..."`);
|
|
596
|
+
this.queue.put({
|
|
597
|
+
id: requestId,
|
|
598
|
+
delta: { role: 'assistant', content: ttsText },
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
else {
|
|
602
|
+
this.queue.put({
|
|
603
|
+
id: requestId,
|
|
604
|
+
delta: { role: 'assistant', content: 'Done.' },
|
|
605
|
+
});
|
|
606
|
+
}
|
|
607
|
+
console.log('✅ Claude response complete');
|
|
608
|
+
}
|
|
609
|
+
catch (error) {
|
|
610
|
+
console.error('❌ Claude Agent SDK error:', error);
|
|
611
|
+
this.queue.put({
|
|
612
|
+
id: requestId,
|
|
613
|
+
delta: { role: 'assistant', content: 'Sorry, I encountered an error.' },
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Create a ClaudeLLM instance
|
|
620
|
+
*/
|
|
621
|
+
export function createClaudeLLM(opts) {
|
|
622
|
+
return new ClaudeLLM(opts);
|
|
623
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codex LLM Wrapper for LiveKit Agents
|
|
3
|
+
*
|
|
4
|
+
* Wraps the Codex Agent SDK (@openai/codex-sdk) to work
|
|
5
|
+
* with LiveKit's AgentSession as an LLM provider.
|
|
6
|
+
*
|
|
7
|
+
* Flow: User speaks → STT → CodexLLM (Agent SDK) → TTS → User hears
|
|
8
|
+
*/
|
|
9
|
+
import { llm, type APIConnectOptions } from '@livekit/agents';
|
|
10
|
+
import { Codex } from '@openai/codex-sdk';
|
|
11
|
+
import { EventEmitter } from 'events';
|
|
12
|
+
export interface CodexLLMOptions {
|
|
13
|
+
workingDirectory?: string;
|
|
14
|
+
skipGitRepoCheck?: boolean;
|
|
15
|
+
/** Event emitter for tool_use, progress updates */
|
|
16
|
+
eventEmitter?: EventEmitter;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Codex LLM - Wraps Codex Agent SDK for LiveKit
|
|
20
|
+
*/
|
|
21
|
+
export declare class CodexLLM extends llm.LLM {
|
|
22
|
+
#private;
|
|
23
|
+
constructor(opts?: CodexLLMOptions);
|
|
24
|
+
label(): string;
|
|
25
|
+
get model(): string;
|
|
26
|
+
get events(): EventEmitter;
|
|
27
|
+
get thread(): ReturnType<Codex['startThread']> | null;
|
|
28
|
+
chat({ chatCtx, toolCtx, connOptions, }: {
|
|
29
|
+
chatCtx: llm.ChatContext;
|
|
30
|
+
toolCtx?: llm.ToolContext;
|
|
31
|
+
connOptions?: APIConnectOptions;
|
|
32
|
+
parallelToolCalls?: boolean;
|
|
33
|
+
toolChoice?: llm.ToolChoice;
|
|
34
|
+
extraKwargs?: Record<string, unknown>;
|
|
35
|
+
}): llm.LLMStream;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Create a CodexLLM instance
|
|
39
|
+
*/
|
|
40
|
+
export declare function createCodexLLM(opts?: CodexLLMOptions): CodexLLM;
|