@phuetz/code-buddy 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/agent/codebuddy-agent.d.ts +6 -0
- package/dist/agent/codebuddy-agent.js +8 -0
- package/dist/agent/tool-handler.d.ts +6 -0
- package/dist/agent/tool-handler.js +9 -0
- package/dist/channels/core.d.ts +3 -0
- package/dist/channels/telegram/client.d.ts +14 -0
- package/dist/channels/telegram/client.js +60 -12
- package/dist/codebuddy/tools.js +6 -2
- package/dist/commands/council.d.ts +29 -0
- package/dist/commands/council.js +299 -0
- package/dist/commands/handlers/channel-handlers.d.ts +9 -6
- package/dist/commands/handlers/channel-handlers.js +216 -41
- package/dist/fleet/model-scoreboard.d.ts +62 -0
- package/dist/fleet/model-scoreboard.js +131 -0
- package/dist/index.js +21 -0
- package/dist/mcp/client.js +10 -2
- package/dist/memory/persistent-memory.d.ts +7 -1
- package/dist/memory/persistent-memory.js +26 -5
- package/dist/security/approval-modes.js +4 -0
- package/dist/tools/registry/memory-tools.d.ts +5 -5
- package/dist/tools/registry/memory-tools.js +24 -8
- package/dist/tools/registry/types.d.ts +3 -0
- package/dist/voice/local-tts.d.ts +6 -0
- package/dist/voice/local-tts.js +32 -1
- package/package.json +4 -1
|
@@ -313,14 +313,115 @@ let aiHandlerRegistered = false;
|
|
|
313
313
|
export function __resetChannelAIHandlerForTests() {
|
|
314
314
|
aiHandlerRegistered = false;
|
|
315
315
|
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
316
|
+
const channelAgentCache = new Map();
|
|
317
|
+
const CHANNEL_AGENT_IDLE_MS = 2 * 60 * 60 * 1000; // evict after 2h idle
|
|
318
|
+
const CHANNEL_AGENT_MAX = 50;
|
|
319
|
+
const channelBotPersonas = new Map();
|
|
320
|
+
export function registerChannelBotPersona(botId, persona) {
|
|
321
|
+
if (botId)
|
|
322
|
+
channelBotPersonas.set(botId, persona);
|
|
323
|
+
}
|
|
324
|
+
/** Reload a chat's prior history from the disk session store into a cold agent. */
|
|
325
|
+
async function restoreChannelSession(agent, sessionKey) {
|
|
326
|
+
try {
|
|
327
|
+
const store = agent.getSessionStore();
|
|
328
|
+
const session = await store.loadSession(sessionKey);
|
|
329
|
+
if (!session?.messages?.length)
|
|
330
|
+
return;
|
|
331
|
+
const restorer = agent;
|
|
332
|
+
restorer.historyManager.setChatHistory(store.convertMessagesToChatEntries(session.messages));
|
|
333
|
+
restorer.historyManager.setMessages(session.messages
|
|
334
|
+
.filter((m) => m.type === 'user' || m.type === 'assistant')
|
|
335
|
+
.map((m) => ({
|
|
336
|
+
role: m.type === 'user' ? 'user' : 'assistant',
|
|
337
|
+
content: m.content,
|
|
338
|
+
})));
|
|
339
|
+
}
|
|
340
|
+
catch (err) {
|
|
341
|
+
logger.warn(`channel session restore failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
/** Persist the agent's current conversation to disk so it survives restarts/eviction. */
|
|
345
|
+
async function persistChannelSession(agent, sessionKey) {
|
|
346
|
+
try {
|
|
347
|
+
const store = agent.getSessionStore();
|
|
348
|
+
const messages = agent
|
|
349
|
+
.getChatHistory()
|
|
350
|
+
.filter((e) => e.type === 'user' || e.type === 'assistant' || e.type === 'tool_result')
|
|
351
|
+
.map((e) => ({
|
|
352
|
+
type: e.type,
|
|
353
|
+
content: String(e.content ?? ''),
|
|
354
|
+
timestamp: (e.timestamp instanceof Date ? e.timestamp : new Date()).toISOString(),
|
|
355
|
+
}));
|
|
356
|
+
const existing = await store.loadSession(sessionKey);
|
|
357
|
+
await store.saveSession({
|
|
358
|
+
id: sessionKey,
|
|
359
|
+
name: existing?.name || `Channel ${sessionKey}`,
|
|
360
|
+
model: existing?.model || 'channel',
|
|
361
|
+
createdAt: existing?.createdAt || new Date(),
|
|
362
|
+
lastAccessedAt: new Date(),
|
|
363
|
+
messages,
|
|
364
|
+
workingDirectory: existing?.workingDirectory || process.cwd(),
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
logger.warn(`channel session persist failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
async function getOrCreateChannelAgent(sessionKey, resolved, agentConfig, botId) {
|
|
372
|
+
const now = Date.now();
|
|
373
|
+
// Evict idle agents first.
|
|
374
|
+
for (const [key, cached] of channelAgentCache) {
|
|
375
|
+
if (now - cached.lastUsed > CHANNEL_AGENT_IDLE_MS)
|
|
376
|
+
channelAgentCache.delete(key);
|
|
377
|
+
}
|
|
378
|
+
const hit = channelAgentCache.get(sessionKey);
|
|
379
|
+
if (hit) {
|
|
380
|
+
hit.lastUsed = now;
|
|
381
|
+
return hit.agent;
|
|
382
|
+
}
|
|
383
|
+
// Per-bot persona (multi-bot): a bot may define its own model + system prompt.
|
|
384
|
+
const persona = botId ? channelBotPersonas.get(botId) : undefined;
|
|
385
|
+
// Opt-in Code Explorer nudge (set CODE_EXPLORER_BIN): some models won't reach
|
|
386
|
+
// for the code-graph MCP tools on their own and just say "I can't" — tell them
|
|
387
|
+
// plainly that they can, and give the CLI fallback. No-op when the env is unset.
|
|
388
|
+
const ceBin = process.env.CODE_EXPLORER_BIN;
|
|
389
|
+
const codeExplorerHint = ceBin
|
|
390
|
+
? `CODE EXPLORER is available for the user's indexed code repositories. For ANY question about repos, code structure, blast-radius/impact, dependencies, dead code, cycles, or code search, you MUST use it — call the \`mcp__code-explorer__*\` tools (list_repos, query, context, impact, find_cycles, hotspots, search_code), or if a tool call isn't available run the CLI via bash: \`${ceBin} <subcommand>\` (e.g. \`${ceBin} list\`, \`${ceBin} query "text"\`, \`${ceBin} impact <symbol>\`). Never reply that you cannot list repositories or analyze code — you can, through Code Explorer.`
|
|
391
|
+
: undefined;
|
|
392
|
+
// Python tasks: the system Python is PEP 668-locked (no global pip). Steer the
|
|
393
|
+
// agent to uv (installed) for ephemeral envs, and to save images to an absolute
|
|
394
|
+
// path it names — the handler then delivers that file to the chat as a photo.
|
|
395
|
+
const pythonHint = 'ACTING vs SHOWING: when the user asks you to draw, plot, generate, create, compute or build something, you MUST actually DO it by running tools/code now — do NOT just print the code or instructions and stop. Execute it and deliver the real artifact. ' +
|
|
396
|
+
'For Python work needing packages (plotting, data, etc.): do NOT use `pip`/`pip3 install` (system Python is PEP 668-locked). Use `uv` — e.g. write the script to a file then run `uv run --with matplotlib --with pandas --with numpy python /tmp/plot.py` (matplotlib must use the Agg backend). When you produce a chart/image, SAVE it to an absolute path like `/tmp/<name>.png` and state that exact path in your reply — it is then sent to the user automatically as a photo.';
|
|
397
|
+
const channelSystemPromptAppend = [persona?.systemPrompt, codeExplorerHint, pythonHint].filter(Boolean).join('\n\n') || undefined;
|
|
398
|
+
const { CodeBuddyAgent } = await import('../../agent/codebuddy-agent.js');
|
|
399
|
+
const model = persona?.model || agentConfig.model || resolved.model;
|
|
400
|
+
const agent = new CodeBuddyAgent(resolved.apiKey || 'local', resolved.baseUrl, model, agentConfig.maxToolRounds ?? 6, // bounded (vs the 50-round default)
|
|
401
|
+
true, // useRAGToolSelection — relevant tools on demand, not all ~194
|
|
402
|
+
process.env.CODEBUDDY_CHANNEL_PROMPT_ID || 'auto', // minimal/adaptive prompt, not the 73KB legacy
|
|
403
|
+
process.cwd(), channelSystemPromptAppend);
|
|
404
|
+
// Scope per-bot state (memory/lessons) to this bot so bots don't share facts.
|
|
405
|
+
agent.setChannelBotId(botId);
|
|
406
|
+
// Persistence: reload prior conversation from disk on a cold agent (after a
|
|
407
|
+
// daemon restart or cache eviction), so continuity survives the in-memory cache.
|
|
408
|
+
await restoreChannelSession(agent, sessionKey);
|
|
409
|
+
// Bound cache size: drop the least-recently-used agent.
|
|
410
|
+
if (channelAgentCache.size >= CHANNEL_AGENT_MAX) {
|
|
411
|
+
let lruKey;
|
|
412
|
+
let lruTime = Infinity;
|
|
413
|
+
for (const [key, cached] of channelAgentCache) {
|
|
414
|
+
if (cached.lastUsed < lruTime) {
|
|
415
|
+
lruTime = cached.lastUsed;
|
|
416
|
+
lruKey = key;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
if (lruKey)
|
|
420
|
+
channelAgentCache.delete(lruKey);
|
|
421
|
+
}
|
|
422
|
+
channelAgentCache.set(sessionKey, { agent, lastUsed: now });
|
|
423
|
+
return agent;
|
|
424
|
+
}
|
|
324
425
|
export async function registerAIMessageHandler(manager) {
|
|
325
426
|
if (aiHandlerRegistered)
|
|
326
427
|
return;
|
|
@@ -346,6 +447,70 @@ export async function registerAIMessageHandler(manager) {
|
|
|
346
447
|
if (!message.content || !message.content.trim()) {
|
|
347
448
|
return;
|
|
348
449
|
}
|
|
450
|
+
// /council <task> — convene the multi-LLM council (ask several capable
|
|
451
|
+
// LLMs, an impartial judge keeps the best, and it learns which model is
|
|
452
|
+
// best per task type over time). `/council` alone shows the scoreboard.
|
|
453
|
+
// Trigger is forgiving: optional leading slash + FR alias `conseil`.
|
|
454
|
+
const councilCmd = message.content.trim().match(/^\/?(?:council|conseil)\b\s*([\s\S]*)$/i);
|
|
455
|
+
if (councilCmd) {
|
|
456
|
+
const task = (councilCmd[1] || '').trim();
|
|
457
|
+
await channel.send({
|
|
458
|
+
channelId: message.channel.id,
|
|
459
|
+
content: task
|
|
460
|
+
? `🧠 Council sur « ${task.slice(0, 100)} » — j'interroge plusieurs IA, je juge et j'apprends… (≈30 s)`
|
|
461
|
+
: '📊 Scoreboard du council…',
|
|
462
|
+
replyTo: message.id,
|
|
463
|
+
});
|
|
464
|
+
const lines = [];
|
|
465
|
+
try {
|
|
466
|
+
const { runCouncil } = await import('../../commands/council.js');
|
|
467
|
+
await runCouncil(task, task ? {} : { scoreboard: true }, (s) => lines.push(s));
|
|
468
|
+
}
|
|
469
|
+
catch (councilErr) {
|
|
470
|
+
lines.push(`❌ Council a échoué : ${councilErr instanceof Error ? councilErr.message : String(councilErr)}`);
|
|
471
|
+
}
|
|
472
|
+
// Telegram caps messages ~4096 chars; flush on line boundaries.
|
|
473
|
+
const full = lines.join('\n').trim() || '(aucune sortie)';
|
|
474
|
+
let buf = '';
|
|
475
|
+
const flush = async () => {
|
|
476
|
+
if (buf) {
|
|
477
|
+
await channel.send({ channelId: message.channel.id, content: buf });
|
|
478
|
+
buf = '';
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
for (const ln of full.split('\n')) {
|
|
482
|
+
if (buf.length + ln.length + 1 > 3800)
|
|
483
|
+
await flush();
|
|
484
|
+
buf += (buf ? '\n' : '') + ln;
|
|
485
|
+
}
|
|
486
|
+
await flush();
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
// Remote tool-approval over Telegram. A daemon has no interactive terminal,
|
|
490
|
+
// so tools that need confirmation fail closed. Instead: intercept
|
|
491
|
+
// `/approve <id>` / `/deny <id>` and resolve the pending approval (the agent
|
|
492
|
+
// turn that requested it is blocked awaiting it, on another concurrent
|
|
493
|
+
// message), and register THIS chat as the approval channel + wire it into
|
|
494
|
+
// the confirmation service so the daemon ASKS the user instead of failing.
|
|
495
|
+
const { getRemoteApprovalService } = await import('../../security/remote-approval.js');
|
|
496
|
+
const approvalSvc = getRemoteApprovalService();
|
|
497
|
+
const approvalCmd = message.content.trim().match(/^\/(approve|deny)\s+(\S+)/i);
|
|
498
|
+
if (approvalCmd && approvalCmd[1] && approvalCmd[2]) {
|
|
499
|
+
const ok = approvalCmd[1].toLowerCase() === 'approve';
|
|
500
|
+
const reqId = approvalCmd[2];
|
|
501
|
+
approvalSvc.handleResponse(reqId, ok);
|
|
502
|
+
await channel.send({
|
|
503
|
+
channelId: message.channel.id,
|
|
504
|
+
content: `${ok ? '✅ Approuvé' : '🚫 Refusé'} : ${reqId}`,
|
|
505
|
+
replyTo: message.id,
|
|
506
|
+
});
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
approvalSvc.registerChannel('telegram', async (msg) => {
|
|
510
|
+
await channel.send({ channelId: message.channel.id, content: msg });
|
|
511
|
+
});
|
|
512
|
+
const { ConfirmationService } = await import('../../utils/confirmation-service.js');
|
|
513
|
+
ConfirmationService.getInstance().setRemoteApprovalService(approvalSvc);
|
|
349
514
|
// 2. Context-adaptive agent reply (« comme Claude »): the agent's own
|
|
350
515
|
// query-classifier + buildForQuery scale the system prompt to the
|
|
351
516
|
// request (a greeting → minimal ~800B prompt, NOT the 73KB legacy),
|
|
@@ -353,7 +518,7 @@ export async function registerAIMessageHandler(manager) {
|
|
|
353
518
|
// `tool_search` meta-tool pulls more when actually needed. Bounded
|
|
354
519
|
// rounds keep a simple chat fast while a real task can still act.
|
|
355
520
|
const { resolveProviderFromEnv } = await import('../../fleet/peer-chat-client-factory.js');
|
|
356
|
-
const knownProviders = ['ollama', 'chatgpt', 'gemini', 'grok', 'anthropic'];
|
|
521
|
+
const knownProviders = ['ollama', 'chatgpt', 'chatgpt-oauth', 'gemini', 'gemini-cli', 'grok', 'anthropic'];
|
|
357
522
|
const preferredProvider = process.env.CODEBUDDY_PROVIDER && knownProviders.includes(process.env.CODEBUDDY_PROVIDER)
|
|
358
523
|
? process.env.CODEBUDDY_PROVIDER
|
|
359
524
|
: 'auto';
|
|
@@ -363,40 +528,14 @@ export async function registerAIMessageHandler(manager) {
|
|
|
363
528
|
return;
|
|
364
529
|
}
|
|
365
530
|
const { getRouteAgentConfig } = await import('../../channels/core.js');
|
|
366
|
-
const { CodeBuddyAgent } = await import('../../agent/codebuddy-agent.js');
|
|
367
531
|
const agentConfig = getRouteAgentConfig(message);
|
|
368
|
-
const model = agentConfig.model || resolved.model;
|
|
369
|
-
const agent = new CodeBuddyAgent(resolved.apiKey || 'local', resolved.baseUrl, model, agentConfig.maxToolRounds ?? 6, // bounded (vs the 50-round default)
|
|
370
|
-
true, // useRAGToolSelection — relevant tools on demand, not all ~194
|
|
371
|
-
process.env.CODEBUDDY_CHANNEL_PROMPT_ID || 'auto', // minimal/adaptive prompt, not the 73KB legacy
|
|
372
|
-
process.cwd());
|
|
373
|
-
// Multi-turn: restore prior session history into the agent.
|
|
374
532
|
const sessionKey = message.sessionKey || 'default-global';
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
model,
|
|
382
|
-
createdAt: new Date(),
|
|
383
|
-
lastAccessedAt: new Date(),
|
|
384
|
-
messages: [],
|
|
385
|
-
workingDirectory: process.cwd(),
|
|
386
|
-
};
|
|
387
|
-
await sessionStore.saveSession(session);
|
|
388
|
-
}
|
|
389
|
-
await sessionStore.resumeSession(sessionKey);
|
|
390
|
-
if (session.messages && session.messages.length > 0) {
|
|
391
|
-
const chatHistory = sessionStore.convertMessagesToChatEntries(session.messages);
|
|
392
|
-
const priorMessages = session.messages.map((m) => ({
|
|
393
|
-
role: m.type === 'user' ? 'user' : 'assistant',
|
|
394
|
-
content: m.content,
|
|
395
|
-
}));
|
|
396
|
-
const historyRestorer = agent;
|
|
397
|
-
historyRestorer.historyManager.setChatHistory(chatHistory);
|
|
398
|
-
historyRestorer.historyManager.setMessages(priorMessages);
|
|
399
|
-
}
|
|
533
|
+
// Reuse ONE agent per chat (cached by sessionKey) so multi-turn context
|
|
534
|
+
// persists in-memory across messages; restored from disk on a cold start.
|
|
535
|
+
// botId selects the per-bot persona and is already baked into sessionKey,
|
|
536
|
+
// so different bots keep separate agents + histories.
|
|
537
|
+
const botId = message.channel?.botId;
|
|
538
|
+
const agent = await getOrCreateChannelAgent(sessionKey, resolved, agentConfig, botId);
|
|
400
539
|
const entries = await agent.processUserMessage(message.content);
|
|
401
540
|
const lastEntry = entries[entries.length - 1];
|
|
402
541
|
const response = lastEntry ? String(lastEntry.content) : '';
|
|
@@ -419,6 +558,30 @@ export async function registerAIMessageHandler(manager) {
|
|
|
419
558
|
logger.warn(`Voice reply skipped: ${voiceErr instanceof Error ? voiceErr.message : String(voiceErr)}`);
|
|
420
559
|
}
|
|
421
560
|
}
|
|
561
|
+
// 7b. Deliver image artifacts: if the reply names image paths that exist
|
|
562
|
+
// on disk (e.g. a chart the agent just generated), send them as photos.
|
|
563
|
+
const imageChannel = channel;
|
|
564
|
+
if (typeof imageChannel.sendImageFile === 'function') {
|
|
565
|
+
const os = await import('node:os');
|
|
566
|
+
const fsp = await import('node:fs/promises');
|
|
567
|
+
const matches = response.match(/(?:~\/|\/)[\w./-]+\.(?:png|jpe?g|gif|webp)/gi) || [];
|
|
568
|
+
const seen = new Set();
|
|
569
|
+
for (const raw of matches) {
|
|
570
|
+
const p = raw.startsWith('~/') ? os.homedir() + raw.slice(1) : raw;
|
|
571
|
+
if (seen.has(p) || seen.size >= 4)
|
|
572
|
+
continue;
|
|
573
|
+
seen.add(p);
|
|
574
|
+
try {
|
|
575
|
+
await fsp.access(p);
|
|
576
|
+
await imageChannel.sendImageFile(message.channel.id, p);
|
|
577
|
+
}
|
|
578
|
+
catch {
|
|
579
|
+
// path isn't a real/accessible image — skip
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
// 8. Persist the conversation so it survives a daemon restart / cache eviction.
|
|
584
|
+
await persistChannelSession(agent, sessionKey);
|
|
422
585
|
}
|
|
423
586
|
catch (err) {
|
|
424
587
|
logger.error('Channel AI response failed', { error: err instanceof Error ? err.message : String(err) });
|
|
@@ -439,6 +602,18 @@ export async function instantiateChannel(config) {
|
|
|
439
602
|
switch (config.type) {
|
|
440
603
|
case 'telegram': {
|
|
441
604
|
const { TelegramChannel } = await import('../../channels/telegram/index.js');
|
|
605
|
+
// Multi-bot persona: the token prefix is the bot id. Register this bot's
|
|
606
|
+
// model + appended system prompt (from channels.json `options`) so the
|
|
607
|
+
// agent built for its messages takes on that persona.
|
|
608
|
+
const tgBotId = (config.token || '').split(':')[0];
|
|
609
|
+
const tgOpts = opts;
|
|
610
|
+
if (tgBotId) {
|
|
611
|
+
registerChannelBotPersona(tgBotId, {
|
|
612
|
+
name: tgOpts.name,
|
|
613
|
+
systemPrompt: tgOpts.systemPrompt,
|
|
614
|
+
model: tgOpts.model,
|
|
615
|
+
});
|
|
616
|
+
}
|
|
442
617
|
// TelegramChannel reads `config.token` (client.ts) — pass `token`, not
|
|
443
618
|
// `botToken`, or it throws "Telegram bot token is required" and the
|
|
444
619
|
// channel never starts from channels.json / server intake.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model Scoreboard — the learning layer for the multi-LLM council.
|
|
3
|
+
*
|
|
4
|
+
* Records, per (taskType × model), the outcome of each council run (won?,
|
|
5
|
+
* judge quality 0-1, latency, cost) to an append-only JSON ledger under
|
|
6
|
+
* ~/.codebuddy/fleet-model-performance.json (same spirit as cost-tracker.ts).
|
|
7
|
+
*
|
|
8
|
+
* The council reads `winRate(taskType, model)` to bias model selection toward
|
|
9
|
+
* the historically-best AI for that kind of task, and `ranking(taskType)` to
|
|
10
|
+
* show what it has learned. This is the piece the Fleet was missing: dispatch +
|
|
11
|
+
* ensemble + consensus existed; *learning which model is best over time* did not.
|
|
12
|
+
*/
|
|
13
|
+
export interface OutcomeRecord {
|
|
14
|
+
/** ISO timestamp of the run. */
|
|
15
|
+
at: string;
|
|
16
|
+
/** Inferred or supplied task category (e.g. 'code', 'reasoning', 'french'). */
|
|
17
|
+
taskType: string;
|
|
18
|
+
/** Model id (e.g. 'gpt-5.5', 'grok-3'). */
|
|
19
|
+
model: string;
|
|
20
|
+
/** Provider id (e.g. 'chatgpt', 'grok'). */
|
|
21
|
+
provider: string;
|
|
22
|
+
/** Did this model win the judge's vote this run? */
|
|
23
|
+
won: boolean;
|
|
24
|
+
/** Judge quality score for this answer, 0-1. */
|
|
25
|
+
quality: number;
|
|
26
|
+
/** Wall-clock latency of this model's answer (ms). */
|
|
27
|
+
latencyMs: number;
|
|
28
|
+
/** Marginal cost of this answer in USD (0 for local / flat-fee). */
|
|
29
|
+
costUsd: number;
|
|
30
|
+
}
|
|
31
|
+
export interface ModelStat {
|
|
32
|
+
model: string;
|
|
33
|
+
provider: string;
|
|
34
|
+
runs: number;
|
|
35
|
+
wins: number;
|
|
36
|
+
/** wins / runs, 0 when never run. */
|
|
37
|
+
winRate: number;
|
|
38
|
+
avgQuality: number;
|
|
39
|
+
avgLatencyMs: number;
|
|
40
|
+
avgCostUsd: number;
|
|
41
|
+
}
|
|
42
|
+
export declare class ModelScoreboard {
|
|
43
|
+
private readonly file;
|
|
44
|
+
private records;
|
|
45
|
+
constructor(file?: string);
|
|
46
|
+
private load;
|
|
47
|
+
private save;
|
|
48
|
+
/** Append one model's outcome for a run and persist. */
|
|
49
|
+
recordOutcome(rec: OutcomeRecord): void;
|
|
50
|
+
/** Historical win rate (0-1) of a model for a task type. 0 when never seen. */
|
|
51
|
+
winRate(taskType: string, model: string): number;
|
|
52
|
+
/**
|
|
53
|
+
* Per-model aggregate stats, optionally scoped to one task type, sorted by
|
|
54
|
+
* win rate desc then avg quality desc.
|
|
55
|
+
*/
|
|
56
|
+
ranking(taskType?: string): ModelStat[];
|
|
57
|
+
/** Human-readable learned ranking, for `buddy council --scoreboard`. */
|
|
58
|
+
print(taskType?: string): string;
|
|
59
|
+
}
|
|
60
|
+
export declare function getModelScoreboard(): ModelScoreboard;
|
|
61
|
+
/** Test seam — reset the cached singleton. */
|
|
62
|
+
export declare function resetModelScoreboard(): void;
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model Scoreboard — the learning layer for the multi-LLM council.
|
|
3
|
+
*
|
|
4
|
+
* Records, per (taskType × model), the outcome of each council run (won?,
|
|
5
|
+
* judge quality 0-1, latency, cost) to an append-only JSON ledger under
|
|
6
|
+
* ~/.codebuddy/fleet-model-performance.json (same spirit as cost-tracker.ts).
|
|
7
|
+
*
|
|
8
|
+
* The council reads `winRate(taskType, model)` to bias model selection toward
|
|
9
|
+
* the historically-best AI for that kind of task, and `ranking(taskType)` to
|
|
10
|
+
* show what it has learned. This is the piece the Fleet was missing: dispatch +
|
|
11
|
+
* ensemble + consensus existed; *learning which model is best over time* did not.
|
|
12
|
+
*/
|
|
13
|
+
import * as fs from 'node:fs';
|
|
14
|
+
import * as path from 'node:path';
|
|
15
|
+
import * as os from 'node:os';
|
|
16
|
+
import { logger } from '../utils/logger.js';
|
|
17
|
+
function defaultLedgerPath() {
|
|
18
|
+
return path.join(os.homedir(), '.codebuddy', 'fleet-model-performance.json');
|
|
19
|
+
}
|
|
20
|
+
export class ModelScoreboard {
|
|
21
|
+
file;
|
|
22
|
+
records = [];
|
|
23
|
+
constructor(file = defaultLedgerPath()) {
|
|
24
|
+
this.file = file;
|
|
25
|
+
this.load();
|
|
26
|
+
}
|
|
27
|
+
load() {
|
|
28
|
+
try {
|
|
29
|
+
if (!fs.existsSync(this.file))
|
|
30
|
+
return;
|
|
31
|
+
const raw = fs.readFileSync(this.file, 'utf-8').trim();
|
|
32
|
+
if (!raw)
|
|
33
|
+
return;
|
|
34
|
+
const parsed = JSON.parse(raw);
|
|
35
|
+
if (Array.isArray(parsed))
|
|
36
|
+
this.records = parsed;
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
logger.warn?.('[model-scoreboard] could not read ledger, starting empty', {
|
|
40
|
+
err: err instanceof Error ? err.message : String(err),
|
|
41
|
+
});
|
|
42
|
+
this.records = [];
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
save() {
|
|
46
|
+
try {
|
|
47
|
+
fs.mkdirSync(path.dirname(this.file), { recursive: true });
|
|
48
|
+
fs.writeFileSync(this.file, JSON.stringify(this.records, null, 2), 'utf-8');
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
logger.warn?.('[model-scoreboard] could not write ledger', {
|
|
52
|
+
err: err instanceof Error ? err.message : String(err),
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/** Append one model's outcome for a run and persist. */
|
|
57
|
+
recordOutcome(rec) {
|
|
58
|
+
this.records.push(rec);
|
|
59
|
+
this.save();
|
|
60
|
+
}
|
|
61
|
+
/** Historical win rate (0-1) of a model for a task type. 0 when never seen. */
|
|
62
|
+
winRate(taskType, model) {
|
|
63
|
+
const runs = this.records.filter((r) => r.taskType === taskType && r.model === model);
|
|
64
|
+
if (runs.length === 0)
|
|
65
|
+
return 0;
|
|
66
|
+
const wins = runs.filter((r) => r.won).length;
|
|
67
|
+
return wins / runs.length;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Per-model aggregate stats, optionally scoped to one task type, sorted by
|
|
71
|
+
* win rate desc then avg quality desc.
|
|
72
|
+
*/
|
|
73
|
+
ranking(taskType) {
|
|
74
|
+
const scoped = taskType
|
|
75
|
+
? this.records.filter((r) => r.taskType === taskType)
|
|
76
|
+
: this.records;
|
|
77
|
+
const byModel = new Map();
|
|
78
|
+
for (const r of scoped) {
|
|
79
|
+
const arr = byModel.get(r.model) ?? [];
|
|
80
|
+
arr.push(r);
|
|
81
|
+
byModel.set(r.model, arr);
|
|
82
|
+
}
|
|
83
|
+
const stats = [];
|
|
84
|
+
for (const [model, runs] of byModel) {
|
|
85
|
+
const wins = runs.filter((r) => r.won).length;
|
|
86
|
+
const n = runs.length;
|
|
87
|
+
stats.push({
|
|
88
|
+
model,
|
|
89
|
+
provider: runs[0].provider,
|
|
90
|
+
runs: n,
|
|
91
|
+
wins,
|
|
92
|
+
winRate: wins / n,
|
|
93
|
+
avgQuality: runs.reduce((a, r) => a + r.quality, 0) / n,
|
|
94
|
+
avgLatencyMs: runs.reduce((a, r) => a + r.latencyMs, 0) / n,
|
|
95
|
+
avgCostUsd: runs.reduce((a, r) => a + r.costUsd, 0) / n,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return stats.sort((a, b) => b.winRate - a.winRate || b.avgQuality - a.avgQuality);
|
|
99
|
+
}
|
|
100
|
+
/** Human-readable learned ranking, for `buddy council --scoreboard`. */
|
|
101
|
+
print(taskType) {
|
|
102
|
+
const rows = this.ranking(taskType);
|
|
103
|
+
if (rows.length === 0) {
|
|
104
|
+
return taskType
|
|
105
|
+
? `No council history yet for task type "${taskType}".`
|
|
106
|
+
: 'No council history yet. Run `buddy council "<task>"` a few times.';
|
|
107
|
+
}
|
|
108
|
+
const header = taskType
|
|
109
|
+
? `Learned model ranking for "${taskType}" tasks:`
|
|
110
|
+
: 'Learned model ranking (all task types):';
|
|
111
|
+
const lines = rows.map((s, i) => {
|
|
112
|
+
const wr = `${Math.round(s.winRate * 100)}%`;
|
|
113
|
+
const q = s.avgQuality.toFixed(2);
|
|
114
|
+
const lat = `${Math.round(s.avgLatencyMs)}ms`;
|
|
115
|
+
const cost = s.avgCostUsd === 0 ? '$0' : `$${s.avgCostUsd.toFixed(4)}`;
|
|
116
|
+
return ` ${i + 1}. ${s.model.padEnd(22)} win ${wr.padStart(4)} (${s.wins}/${s.runs}) q${q} ${lat} ${cost}`;
|
|
117
|
+
});
|
|
118
|
+
return [header, ...lines].join('\n');
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
let singleton = null;
|
|
122
|
+
export function getModelScoreboard() {
|
|
123
|
+
if (!singleton)
|
|
124
|
+
singleton = new ModelScoreboard();
|
|
125
|
+
return singleton;
|
|
126
|
+
}
|
|
127
|
+
/** Test seam — reset the cached singleton. */
|
|
128
|
+
export function resetModelScoreboard() {
|
|
129
|
+
singleton = null;
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=model-scoreboard.js.map
|
package/dist/index.js
CHANGED
|
@@ -2345,6 +2345,27 @@ program
|
|
|
2345
2345
|
process.env.CODEBUDDY_LLM_FAILOVER === "1";
|
|
2346
2346
|
cli.stdout(`Auto-failover: ${on ? "ON" : "OFF (enable with [llm].enabled = true, or CODEBUDDY_LLM_FAILOVER=1)"}`);
|
|
2347
2347
|
});
|
|
2348
|
+
// Council — capability-aware multi-LLM router + ensemble (judge + consensus) + learning.
|
|
2349
|
+
program
|
|
2350
|
+
.command("council [task...]")
|
|
2351
|
+
.description("Ask several capable LLMs the same task, judge + reconcile the answers, and learn which model is best per task type")
|
|
2352
|
+
.option("-n, --count <n>", "How many models to consult (default 3)")
|
|
2353
|
+
.option("--models <list>", "Restrict to these providers/models (comma list)")
|
|
2354
|
+
.option("--judge <model>", "Provider/model to use as the impartial judge")
|
|
2355
|
+
.option("--task-type <tag>", "Override inferred task type (code|reasoning|french|vision|general)")
|
|
2356
|
+
.option("--no-consensus", "Skip the consensus/agreement summary")
|
|
2357
|
+
.option("--scoreboard", "Print the learned model ranking and exit")
|
|
2358
|
+
.action(async (taskParts = [], options) => {
|
|
2359
|
+
const { runCouncil } = await import("./commands/council.js");
|
|
2360
|
+
await runCouncil((taskParts || []).join(" ").trim(), {
|
|
2361
|
+
count: options.count ? Number(options.count) : undefined,
|
|
2362
|
+
models: options.models,
|
|
2363
|
+
judge: options.judge,
|
|
2364
|
+
taskType: options.taskType,
|
|
2365
|
+
consensus: options.consensus,
|
|
2366
|
+
scoreboard: options.scoreboard,
|
|
2367
|
+
}, cli.stdout);
|
|
2368
|
+
});
|
|
2348
2369
|
// MCP Server command - run Code Buddy as an MCP tool provider over stdio
|
|
2349
2370
|
program
|
|
2350
2371
|
.command("mcp-server")
|
package/dist/mcp/client.js
CHANGED
|
@@ -211,11 +211,19 @@ export class MCPManager extends EventEmitter {
|
|
|
211
211
|
}
|
|
212
212
|
const { loadMCPConfig } = await import('./config.js');
|
|
213
213
|
const config = loadMCPConfig();
|
|
214
|
-
// Initialize servers in parallel
|
|
214
|
+
// Initialize servers in parallel. Each server gets its OWN timeout so a
|
|
215
|
+
// hanging/unresponsive server (e.g. one whose stdio handshake never
|
|
216
|
+
// completes, or a GUI-bound server whose display is down) can't block the
|
|
217
|
+
// init of the others — previously a single hung server made Promise.all
|
|
218
|
+
// wait forever, so NO MCP tools (incl. healthy ones) ever loaded.
|
|
219
|
+
const INIT_TIMEOUT_MS = Number(process.env.CODEBUDDY_MCP_INIT_TIMEOUT_MS) || 15_000;
|
|
215
220
|
const enabledServers = config.servers.filter(s => s.enabled !== false);
|
|
216
221
|
const initPromises = enabledServers.map(async (serverConfig) => {
|
|
217
222
|
try {
|
|
218
|
-
await
|
|
223
|
+
await Promise.race([
|
|
224
|
+
this.addServer(serverConfig),
|
|
225
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`MCP server "${serverConfig.name}" init timed out after ${INIT_TIMEOUT_MS}ms — skipped so other servers still load`)), INIT_TIMEOUT_MS)),
|
|
226
|
+
]);
|
|
219
227
|
}
|
|
220
228
|
catch (error) {
|
|
221
229
|
logger.warn(`Failed to initialize MCP server ${serverConfig.name}`, { error });
|
|
@@ -152,6 +152,12 @@ export declare class PersistentMemoryManager extends EventEmitter {
|
|
|
152
152
|
total: number;
|
|
153
153
|
};
|
|
154
154
|
}
|
|
155
|
-
|
|
155
|
+
/**
|
|
156
|
+
* Get the memory manager. With no `botId`, returns the global singleton (default
|
|
157
|
+
* — CLI / desktop / server behavior unchanged). With a `botId` (multi-bot
|
|
158
|
+
* channels), returns a per-bot instance whose memory files live under
|
|
159
|
+
* `~/.codebuddy/bots/<botId>/`, so bots never share each other's `remember` facts.
|
|
160
|
+
*/
|
|
161
|
+
export declare function getMemoryManager(config?: Partial<MemoryConfig>, botId?: string): PersistentMemoryManager;
|
|
156
162
|
export declare function resetMemoryManagerForTests(): void;
|
|
157
163
|
export declare function initializeMemory(config?: Partial<MemoryConfig>): Promise<PersistentMemoryManager>;
|
|
@@ -832,16 +832,37 @@ export class PersistentMemoryManager extends EventEmitter {
|
|
|
832
832
|
};
|
|
833
833
|
}
|
|
834
834
|
}
|
|
835
|
-
//
|
|
835
|
+
// Default singleton instance (no bot scope) + per-bot instances.
|
|
836
836
|
let memoryManagerInstance = null;
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
837
|
+
const memoryManagerByBot = new Map();
|
|
838
|
+
/**
|
|
839
|
+
* Get the memory manager. With no `botId`, returns the global singleton (default
|
|
840
|
+
* — CLI / desktop / server behavior unchanged). With a `botId` (multi-bot
|
|
841
|
+
* channels), returns a per-bot instance whose memory files live under
|
|
842
|
+
* `~/.codebuddy/bots/<botId>/`, so bots never share each other's `remember` facts.
|
|
843
|
+
*/
|
|
844
|
+
export function getMemoryManager(config, botId) {
|
|
845
|
+
if (!botId) {
|
|
846
|
+
if (!memoryManagerInstance) {
|
|
847
|
+
memoryManagerInstance = new PersistentMemoryManager(config);
|
|
848
|
+
}
|
|
849
|
+
return memoryManagerInstance;
|
|
850
|
+
}
|
|
851
|
+
let inst = memoryManagerByBot.get(botId);
|
|
852
|
+
if (!inst) {
|
|
853
|
+
const botDir = path.join(os.homedir(), '.codebuddy', 'bots', botId);
|
|
854
|
+
inst = new PersistentMemoryManager({
|
|
855
|
+
...(config ?? {}),
|
|
856
|
+
userMemoryPath: path.join(botDir, 'memory.md'),
|
|
857
|
+
projectMemoryPath: path.join(botDir, 'CODEBUDDY_MEMORY.md'),
|
|
858
|
+
});
|
|
859
|
+
memoryManagerByBot.set(botId, inst);
|
|
840
860
|
}
|
|
841
|
-
return
|
|
861
|
+
return inst;
|
|
842
862
|
}
|
|
843
863
|
export function resetMemoryManagerForTests() {
|
|
844
864
|
memoryManagerInstance = null;
|
|
865
|
+
memoryManagerByBot.clear();
|
|
845
866
|
}
|
|
846
867
|
export async function initializeMemory(config) {
|
|
847
868
|
const manager = getMemoryManager(config);
|
|
@@ -75,6 +75,10 @@ const SAFE_COMMANDS = new Set([
|
|
|
75
75
|
'npm list', 'npm ls', 'npm outdated', 'npm view',
|
|
76
76
|
'node --version', 'npm --version', 'python --version',
|
|
77
77
|
'cargo --version', 'go version', 'rustc --version',
|
|
78
|
+
// Code Explorer (read-oriented code-graph companion) — safe to auto-approve;
|
|
79
|
+
// its writes only touch its own index, and destructive shell patterns are
|
|
80
|
+
// still caught by DESTRUCTIVE_PATTERNS above.
|
|
81
|
+
'code-explorer', 'gitnexus',
|
|
78
82
|
]);
|
|
79
83
|
// Network-related commands
|
|
80
84
|
const NETWORK_COMMANDS = new Set([
|
|
@@ -9,11 +9,11 @@
|
|
|
9
9
|
* - forget: Remove a memory entry
|
|
10
10
|
*/
|
|
11
11
|
import type { ToolResult } from '../../types/index.js';
|
|
12
|
-
import type { ITool, ToolSchema, IToolMetadata, IValidationResult } from './types.js';
|
|
12
|
+
import type { ITool, ToolSchema, IToolMetadata, IValidationResult, IToolExecutionContext } from './types.js';
|
|
13
13
|
export declare class RememberTool implements ITool {
|
|
14
14
|
readonly name = "remember";
|
|
15
15
|
readonly description = "Store important information, decisions, or preferences in persistent memory. This survives across sessions and is project-scoped by default.";
|
|
16
|
-
execute(input: Record<string, unknown
|
|
16
|
+
execute(input: Record<string, unknown>, context?: IToolExecutionContext): Promise<ToolResult>;
|
|
17
17
|
getSchema(): ToolSchema;
|
|
18
18
|
validate(input: unknown): IValidationResult;
|
|
19
19
|
getMetadata(): IToolMetadata;
|
|
@@ -22,7 +22,7 @@ export declare class RememberTool implements ITool {
|
|
|
22
22
|
export declare class ReplaceMemoryTool implements ITool {
|
|
23
23
|
readonly name = "replace_memory";
|
|
24
24
|
readonly description = "Replace an existing persistent memory entry. Use when a stored fact is obsolete or too verbose and must be rewritten under the memory char budget.";
|
|
25
|
-
execute(input: Record<string, unknown
|
|
25
|
+
execute(input: Record<string, unknown>, context?: IToolExecutionContext): Promise<ToolResult>;
|
|
26
26
|
getSchema(): ToolSchema;
|
|
27
27
|
validate(input: unknown): IValidationResult;
|
|
28
28
|
getMetadata(): IToolMetadata;
|
|
@@ -40,7 +40,7 @@ export declare class MemoryProposeTool implements ITool {
|
|
|
40
40
|
export declare class RecallTool implements ITool {
|
|
41
41
|
readonly name = "recall";
|
|
42
42
|
readonly description = "Explicitly retrieve a specific memory entry by its key. Use this if the information is not currently in your system prompt.";
|
|
43
|
-
execute(input: Record<string, unknown
|
|
43
|
+
execute(input: Record<string, unknown>, context?: IToolExecutionContext): Promise<ToolResult>;
|
|
44
44
|
getSchema(): ToolSchema;
|
|
45
45
|
validate(input: unknown): IValidationResult;
|
|
46
46
|
getMetadata(): IToolMetadata;
|
|
@@ -49,7 +49,7 @@ export declare class RecallTool implements ITool {
|
|
|
49
49
|
export declare class ForgetTool implements ITool {
|
|
50
50
|
readonly name = "forget";
|
|
51
51
|
readonly description = "Remove a memory entry that is no longer valid or useful.";
|
|
52
|
-
execute(input: Record<string, unknown
|
|
52
|
+
execute(input: Record<string, unknown>, context?: IToolExecutionContext): Promise<ToolResult>;
|
|
53
53
|
getSchema(): ToolSchema;
|
|
54
54
|
validate(input: unknown): IValidationResult;
|
|
55
55
|
getMetadata(): IToolMetadata;
|