@blockrun/franklin 3.6.5 → 3.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/commands.js +134 -45
- package/dist/agent/context.js +67 -3
- package/dist/agent/loop.js +35 -4
- package/dist/agent/types.d.ts +2 -0
- package/dist/agent/verification.d.ts +42 -0
- package/dist/agent/verification.js +206 -0
- package/dist/commands/config.js +15 -7
- package/dist/commands/setup.js +7 -7
- package/dist/commands/start.js +4 -1
- package/dist/learnings/extractor.d.ts +5 -0
- package/dist/learnings/extractor.js +118 -2
- package/dist/learnings/index.d.ts +3 -3
- package/dist/learnings/index.js +2 -2
- package/dist/learnings/store.d.ts +11 -1
- package/dist/learnings/store.js +100 -0
- package/dist/learnings/types.d.ts +16 -0
- package/dist/tools/index.js +2 -0
- package/dist/tools/moa.d.ts +16 -0
- package/dist/tools/moa.js +173 -0
- package/dist/ui/app.js +7 -3
- package/package.json +1 -1
package/dist/agent/commands.js
CHANGED
|
@@ -202,7 +202,7 @@ const DIRECT_COMMANDS = {
|
|
|
202
202
|
` **Git:** /push /pr /undo /status /diff /log /branch /stash /unstash\n` +
|
|
203
203
|
` **Analysis:** /security /lint /optimize /todo /deps /clean /migrate /doc\n` +
|
|
204
204
|
` **Session:** /plan /ultraplan /execute /compact /retry /sessions /resume /session-search /context /tasks\n` +
|
|
205
|
-
` **Power:** /ultrathink [query] /ultraplan /noplan /dump\n` +
|
|
205
|
+
` **Power:** /ultrathink [query] /ultraplan /noplan /moa [query] /dump\n` +
|
|
206
206
|
` **Info:** /model /wallet /cost /tokens /learnings /brain /mcp /doctor /version /bug /help\n` +
|
|
207
207
|
` **UI:** /clear /exit\n` +
|
|
208
208
|
(ultrathinkOn ? `\n Ultrathink: ON\n` : '')
|
|
@@ -290,7 +290,7 @@ const DIRECT_COMMANDS = {
|
|
|
290
290
|
const hasWallet = fs.existsSync(path.join(BLOCKRUN_DIR, 'wallet.json'))
|
|
291
291
|
|| fs.existsSync(path.join(BLOCKRUN_DIR, 'solana-wallet.json'));
|
|
292
292
|
checks.push(hasWallet ? '✓ wallet configured' : '⚠ no wallet — run: runcode setup');
|
|
293
|
-
checks.push(fs.existsSync(path.join(BLOCKRUN_DIR, 'runcode-config.json')) ? '✓ config file exists' : '⚠ no config — using defaults');
|
|
293
|
+
checks.push(fs.existsSync(path.join(BLOCKRUN_DIR, 'franklin-config.json')) || fs.existsSync(path.join(BLOCKRUN_DIR, 'runcode-config.json')) ? '✓ config file exists' : '⚠ no config — using defaults');
|
|
294
294
|
// Check MCP
|
|
295
295
|
const { listMcpServers } = await import('../mcp/client.js');
|
|
296
296
|
const mcpServers = listMcpServers();
|
|
@@ -398,49 +398,6 @@ const DIRECT_COMMANDS = {
|
|
|
398
398
|
ctx.onEvent({ kind: 'text_delta', text });
|
|
399
399
|
emitDone(ctx);
|
|
400
400
|
},
|
|
401
|
-
'/wallet': async (ctx) => {
|
|
402
|
-
const chain = (await import('../config.js')).loadChain();
|
|
403
|
-
try {
|
|
404
|
-
let address;
|
|
405
|
-
let balance;
|
|
406
|
-
const fetchTimeout = (ms) => new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), ms));
|
|
407
|
-
if (chain === 'solana') {
|
|
408
|
-
const { getOrCreateSolanaWallet, setupAgentSolanaWallet } = await import('@blockrun/llm');
|
|
409
|
-
const w = await getOrCreateSolanaWallet();
|
|
410
|
-
address = w.address;
|
|
411
|
-
try {
|
|
412
|
-
const client = await setupAgentSolanaWallet({ silent: true });
|
|
413
|
-
const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
|
|
414
|
-
balance = `$${bal.toFixed(2)} USDC`;
|
|
415
|
-
}
|
|
416
|
-
catch {
|
|
417
|
-
balance = '(unavailable)';
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
else {
|
|
421
|
-
const { getOrCreateWallet, setupAgentWallet } = await import('@blockrun/llm');
|
|
422
|
-
const w = getOrCreateWallet();
|
|
423
|
-
address = w.address;
|
|
424
|
-
try {
|
|
425
|
-
const client = setupAgentWallet({ silent: true });
|
|
426
|
-
const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
|
|
427
|
-
balance = `$${bal.toFixed(2)} USDC`;
|
|
428
|
-
}
|
|
429
|
-
catch {
|
|
430
|
-
balance = '(unavailable)';
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
ctx.onEvent({ kind: 'text_delta', text: `**Wallet**\n` +
|
|
434
|
-
` Chain: ${chain}\n` +
|
|
435
|
-
` Address: ${address}\n` +
|
|
436
|
-
` Balance: ${balance}\n`
|
|
437
|
-
});
|
|
438
|
-
}
|
|
439
|
-
catch (err) {
|
|
440
|
-
ctx.onEvent({ kind: 'text_delta', text: `Wallet error: ${err.message}\n` });
|
|
441
|
-
}
|
|
442
|
-
emitDone(ctx);
|
|
443
|
-
},
|
|
444
401
|
'/clear': (ctx) => {
|
|
445
402
|
ctx.history.length = 0;
|
|
446
403
|
resetTokenAnchor();
|
|
@@ -536,6 +493,8 @@ const ARG_COMMANDS = [
|
|
|
536
493
|
{ prefix: '/refactor ', rewrite: (a) => `Refactor: ${a}. Read the relevant code first, then make targeted changes. Explain each change.` },
|
|
537
494
|
{ prefix: '/scaffold ', rewrite: (a) => `Create the scaffolding/boilerplate for: ${a}. Generate the file structure and initial code. Ask me if you need clarification on requirements.` },
|
|
538
495
|
{ prefix: '/doc ', rewrite: (a) => `Generate documentation for ${a}. Include: purpose, API/interface description, usage examples, and important notes.` },
|
|
496
|
+
{ prefix: '/moa ', rewrite: (a) => `Use the MixtureOfAgents tool to get a high-quality answer by querying multiple AI models in parallel: ${a}` },
|
|
497
|
+
{ prefix: '/moa', rewrite: () => `Use the MixtureOfAgents tool. Ask me what question I want answered by multiple models.` },
|
|
539
498
|
];
|
|
540
499
|
// ─── Main dispatch ────────────────────────────────────────────────────────
|
|
541
500
|
/**
|
|
@@ -667,6 +626,7 @@ export async function handleSlashCommand(input, ctx) {
|
|
|
667
626
|
else {
|
|
668
627
|
const newModel = resolveModel(input.slice(7).trim());
|
|
669
628
|
ctx.config.model = newModel;
|
|
629
|
+
ctx.config.baseModel = newModel; // Update recovery target so loop doesn't reset
|
|
670
630
|
ctx.config.onModelChange?.(newModel);
|
|
671
631
|
ctx.onEvent({ kind: 'text_delta', text: `Model → **${newModel}**\n` });
|
|
672
632
|
}
|
|
@@ -690,6 +650,135 @@ export async function handleSlashCommand(input, ctx) {
|
|
|
690
650
|
emitDone(ctx);
|
|
691
651
|
return { handled: true };
|
|
692
652
|
}
|
|
653
|
+
// /wallet — show wallet info, import, or export
|
|
654
|
+
if (input === '/wallet' || input.startsWith('/wallet ')) {
|
|
655
|
+
const chain = (await import('../config.js')).loadChain();
|
|
656
|
+
const args = input.slice(7).trim();
|
|
657
|
+
// /wallet export — show private key
|
|
658
|
+
if (args === 'export') {
|
|
659
|
+
try {
|
|
660
|
+
if (chain === 'solana') {
|
|
661
|
+
const { loadSolanaWallet, getOrCreateSolanaWallet } = await import('@blockrun/llm');
|
|
662
|
+
const key = loadSolanaWallet();
|
|
663
|
+
if (!key) {
|
|
664
|
+
ctx.onEvent({ kind: 'text_delta', text: 'No Solana wallet found. Run `/wallet` first.\n' });
|
|
665
|
+
emitDone(ctx);
|
|
666
|
+
return { handled: true };
|
|
667
|
+
}
|
|
668
|
+
const w = await getOrCreateSolanaWallet();
|
|
669
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Wallet Export (Solana)**\n` +
|
|
670
|
+
` Address: ${w.address}\n` +
|
|
671
|
+
` Private Key: ${key}\n\n` +
|
|
672
|
+
`⚠️ Keep this key safe. Anyone with it controls your funds.\n`
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
else {
|
|
676
|
+
const { loadWallet, getOrCreateWallet } = await import('@blockrun/llm');
|
|
677
|
+
const key = loadWallet();
|
|
678
|
+
if (!key) {
|
|
679
|
+
ctx.onEvent({ kind: 'text_delta', text: 'No wallet found. Run `/wallet` first.\n' });
|
|
680
|
+
emitDone(ctx);
|
|
681
|
+
return { handled: true };
|
|
682
|
+
}
|
|
683
|
+
const w = getOrCreateWallet();
|
|
684
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Wallet Export (Base)**\n` +
|
|
685
|
+
` Address: ${w.address}\n` +
|
|
686
|
+
` Private Key: ${key}\n\n` +
|
|
687
|
+
`⚠️ Keep this key safe. Anyone with it controls your funds.\n`
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
catch (err) {
|
|
692
|
+
ctx.onEvent({ kind: 'text_delta', text: `Export error: ${err.message}\n` });
|
|
693
|
+
}
|
|
694
|
+
emitDone(ctx);
|
|
695
|
+
return { handled: true };
|
|
696
|
+
}
|
|
697
|
+
// /wallet import <private-key>
|
|
698
|
+
if (args.startsWith('import')) {
|
|
699
|
+
const key = args.slice(6).trim();
|
|
700
|
+
if (!key) {
|
|
701
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Usage:** \`/wallet import <private-key>\`\n\n` +
|
|
702
|
+
` Base: \`/wallet import 0x...\` (hex, 66 chars)\n` +
|
|
703
|
+
` Solana: \`/wallet import <bs58-key>\` (base58 encoded)\n`
|
|
704
|
+
});
|
|
705
|
+
emitDone(ctx);
|
|
706
|
+
return { handled: true };
|
|
707
|
+
}
|
|
708
|
+
try {
|
|
709
|
+
if (chain === 'solana') {
|
|
710
|
+
const { saveSolanaWallet, solanaPublicKey } = await import('@blockrun/llm');
|
|
711
|
+
const address = await solanaPublicKey(key);
|
|
712
|
+
saveSolanaWallet(key);
|
|
713
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Wallet Imported (Solana)**\n` +
|
|
714
|
+
` Address: ${address}\n` +
|
|
715
|
+
` Saved to: ~/.blockrun/\n\n` +
|
|
716
|
+
`Restart Franklin to use the new wallet.\n`
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
else {
|
|
720
|
+
const { privateKeyToAccount } = await import('viem/accounts');
|
|
721
|
+
const { saveWallet } = await import('@blockrun/llm');
|
|
722
|
+
const account = privateKeyToAccount(key);
|
|
723
|
+
saveWallet(key);
|
|
724
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Wallet Imported (Base)**\n` +
|
|
725
|
+
` Address: ${account.address}\n` +
|
|
726
|
+
` Saved to: ~/.blockrun/\n\n` +
|
|
727
|
+
`Restart Franklin to use the new wallet.\n`
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
catch (err) {
|
|
732
|
+
ctx.onEvent({ kind: 'text_delta', text: `Import error: ${err.message}\n` });
|
|
733
|
+
}
|
|
734
|
+
emitDone(ctx);
|
|
735
|
+
return { handled: true };
|
|
736
|
+
}
|
|
737
|
+
// /wallet (no args) — show wallet info
|
|
738
|
+
try {
|
|
739
|
+
let address;
|
|
740
|
+
let balance;
|
|
741
|
+
const fetchTimeout = (ms) => new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), ms));
|
|
742
|
+
if (chain === 'solana') {
|
|
743
|
+
const { getOrCreateSolanaWallet, setupAgentSolanaWallet } = await import('@blockrun/llm');
|
|
744
|
+
const w = await getOrCreateSolanaWallet();
|
|
745
|
+
address = w.address;
|
|
746
|
+
try {
|
|
747
|
+
const client = await setupAgentSolanaWallet({ silent: true });
|
|
748
|
+
const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
|
|
749
|
+
balance = `$${bal.toFixed(2)} USDC`;
|
|
750
|
+
}
|
|
751
|
+
catch {
|
|
752
|
+
balance = '(unavailable)';
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
else {
|
|
756
|
+
const { getOrCreateWallet, setupAgentWallet } = await import('@blockrun/llm');
|
|
757
|
+
const w = getOrCreateWallet();
|
|
758
|
+
address = w.address;
|
|
759
|
+
try {
|
|
760
|
+
const client = setupAgentWallet({ silent: true });
|
|
761
|
+
const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
|
|
762
|
+
balance = `$${bal.toFixed(2)} USDC`;
|
|
763
|
+
}
|
|
764
|
+
catch {
|
|
765
|
+
balance = '(unavailable)';
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
ctx.onEvent({ kind: 'text_delta', text: `**Wallet**\n` +
|
|
769
|
+
` Chain: ${chain}\n` +
|
|
770
|
+
` Address: ${address}\n` +
|
|
771
|
+
` Balance: ${balance}\n\n` +
|
|
772
|
+
` \`/wallet import <key>\` — import a personal wallet\n` +
|
|
773
|
+
` \`/wallet export\` — show private key\n`
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
catch (err) {
|
|
777
|
+
ctx.onEvent({ kind: 'text_delta', text: `Wallet error: ${err.message}\n` });
|
|
778
|
+
}
|
|
779
|
+
emitDone(ctx);
|
|
780
|
+
return { handled: true };
|
|
781
|
+
}
|
|
693
782
|
// /delete <...>
|
|
694
783
|
if (input.startsWith('/delete ')) {
|
|
695
784
|
const arg = input.slice('/delete '.length).trim();
|
package/dist/agent/context.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
7
|
import { execSync } from 'node:child_process';
|
|
8
|
-
import { loadLearnings, decayLearnings, saveLearnings, formatForPrompt } from '../learnings/store.js';
|
|
8
|
+
import { loadLearnings, decayLearnings, saveLearnings, formatForPrompt, loadSkills, formatSkillsForPrompt } from '../learnings/store.js';
|
|
9
9
|
// ─── System Instructions Assembly ──────────────────────────────────────────
|
|
10
10
|
// Composable prompt sections — each independently maintainable and conditionally includable.
|
|
11
11
|
function getCoreInstructions() {
|
|
@@ -186,10 +186,16 @@ export function assembleInstructions(workingDir, model) {
|
|
|
186
186
|
getTokenEfficiencySection(),
|
|
187
187
|
getVerificationSection(),
|
|
188
188
|
];
|
|
189
|
-
// Read RUNCODE.md or CLAUDE.md from the project
|
|
189
|
+
// Read RUNCODE.md or CLAUDE.md from the project (with injection scanning)
|
|
190
190
|
const projectConfig = readProjectConfig(workingDir);
|
|
191
191
|
if (projectConfig) {
|
|
192
|
-
|
|
192
|
+
const { sanitized, threats } = scanForInjection(projectConfig);
|
|
193
|
+
if (threats.length > 0) {
|
|
194
|
+
parts.push(`# Project Instructions\n\n⚠️ WARNING: ${threats.length} suspicious pattern(s) detected in project config and neutralized.\n\n${sanitized}`);
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
parts.push(`# Project Instructions\n\n${projectConfig}`);
|
|
198
|
+
}
|
|
193
199
|
}
|
|
194
200
|
// Inject environment info
|
|
195
201
|
parts.push(buildEnvironmentSection(workingDir));
|
|
@@ -210,6 +216,18 @@ export function assembleInstructions(workingDir, model) {
|
|
|
210
216
|
}
|
|
211
217
|
}
|
|
212
218
|
catch { /* learnings are optional — never block startup */ }
|
|
219
|
+
// Inject relevant skills (procedural memory from past complex tasks)
|
|
220
|
+
try {
|
|
221
|
+
const allSkills = loadSkills();
|
|
222
|
+
if (allSkills.length > 0) {
|
|
223
|
+
// Skills are matched lazily on first user message — for now inject top skills by use count
|
|
224
|
+
const topSkills = allSkills.sort((a, b) => b.uses - a.uses).slice(0, 5);
|
|
225
|
+
const skillsSection = formatSkillsForPrompt(topSkills);
|
|
226
|
+
if (skillsSection)
|
|
227
|
+
parts.push(skillsSection);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
catch { /* skills are optional */ }
|
|
213
231
|
// Model-specific execution guidance
|
|
214
232
|
if (model) {
|
|
215
233
|
parts.push(getModelGuidance(model));
|
|
@@ -276,6 +294,52 @@ export function invalidateInstructionCache(workingDir) {
|
|
|
276
294
|
_instructionCache.clear();
|
|
277
295
|
}
|
|
278
296
|
}
|
|
297
|
+
// ─── Prompt Injection Detection ────────────────────────────────────────────
|
|
298
|
+
/** Patterns that indicate potential prompt injection in context files. */
|
|
299
|
+
const INJECTION_PATTERNS = [
|
|
300
|
+
// Direct instruction override attempts
|
|
301
|
+
{ pattern: /ignore\s+(all\s+)?previous\s+instructions/i, description: 'instruction override' },
|
|
302
|
+
{ pattern: /disregard\s+(all\s+)?(previous\s+|above\s+)?rules/i, description: 'rule disregard' },
|
|
303
|
+
{ pattern: /forget\s+(everything|all|your)\s+(you|instructions|rules)/i, description: 'memory wipe' },
|
|
304
|
+
{ pattern: /you\s+are\s+now\s+(?:a\s+)?(?:different|new|unrestricted)/i, description: 'identity hijack' },
|
|
305
|
+
{ pattern: /system\s*:\s*you\s+are/i, description: 'fake system message' },
|
|
306
|
+
// Dangerous command injection
|
|
307
|
+
{ pattern: /execute\s+(curl|wget|bash|sh|python|node)\b/i, description: 'command execution' },
|
|
308
|
+
{ pattern: /\bcat\s+\/etc\/(passwd|shadow|sudoers)/i, description: 'credential access' },
|
|
309
|
+
{ pattern: /\brm\s+-rf\s+[\/~]/i, description: 'destructive command' },
|
|
310
|
+
{ pattern: /\beval\s*\(/i, description: 'eval injection' },
|
|
311
|
+
// Data exfiltration
|
|
312
|
+
{ pattern: /\bcurl\s+.*\|\s*(bash|sh)/i, description: 'pipe to shell' },
|
|
313
|
+
{ pattern: /send\s+(to|via)\s+(http|webhook|url)/i, description: 'data exfiltration' },
|
|
314
|
+
// HTML/comment injection
|
|
315
|
+
{ pattern: /<!--[\s\S]*?-->/g, description: 'HTML comment injection' },
|
|
316
|
+
];
|
|
317
|
+
/** Invisible unicode characters that can hide malicious content. */
|
|
318
|
+
const INVISIBLE_UNICODE = /[\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF\u00AD]/g;
|
|
319
|
+
/**
|
|
320
|
+
* Scan text for prompt injection patterns and invisible unicode.
|
|
321
|
+
* Returns sanitized text with threats neutralized and a list of detections.
|
|
322
|
+
*/
|
|
323
|
+
function scanForInjection(text) {
|
|
324
|
+
const threats = [];
|
|
325
|
+
let sanitized = text;
|
|
326
|
+
// Check for invisible unicode
|
|
327
|
+
if (INVISIBLE_UNICODE.test(sanitized)) {
|
|
328
|
+
const count = (sanitized.match(INVISIBLE_UNICODE) || []).length;
|
|
329
|
+
threats.push(`${count} invisible unicode character(s) removed`);
|
|
330
|
+
sanitized = sanitized.replace(INVISIBLE_UNICODE, '');
|
|
331
|
+
}
|
|
332
|
+
// Check for injection patterns
|
|
333
|
+
for (const { pattern, description } of INJECTION_PATTERNS) {
|
|
334
|
+
const matches = sanitized.match(pattern);
|
|
335
|
+
if (matches) {
|
|
336
|
+
threats.push(`${description}: "${matches[0].slice(0, 50)}"`);
|
|
337
|
+
// Neutralize by wrapping in brackets (visible but defanged)
|
|
338
|
+
sanitized = sanitized.replace(pattern, (match) => `[BLOCKED: ${match}]`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return { sanitized, threats };
|
|
342
|
+
}
|
|
279
343
|
// ─── Project Config ────────────────────────────────────────────────────────
|
|
280
344
|
/**
|
|
281
345
|
* Look for RUNCODE.md, then CLAUDE.md in the working directory and parents.
|
package/dist/agent/loop.js
CHANGED
|
@@ -19,6 +19,7 @@ import { maybeMidSessionExtract } from '../learnings/extractor.js';
|
|
|
19
19
|
import { routeRequest, parseRoutingProfile } from '../router/index.js';
|
|
20
20
|
import { recordOutcome } from '../router/local-elo.js';
|
|
21
21
|
import { shouldPlan, getPlanningPrompt, getExecutorModel, isExecutorStuck, toolCallSignature } from './planner.js';
|
|
22
|
+
import { shouldVerify, runVerification } from './verification.js';
|
|
22
23
|
import { createSessionId, appendToSession, updateSessionMeta, pruneOldSessions, } from '../session/storage.js';
|
|
23
24
|
/**
|
|
24
25
|
* Atomically replace all elements in a history array.
|
|
@@ -218,7 +219,7 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
|
|
|
218
219
|
const permissions = new PermissionManager(config.permissionMode ?? 'default', config.permissionPromptFn);
|
|
219
220
|
const history = [];
|
|
220
221
|
let lastUserInput = ''; // For /retry
|
|
221
|
-
|
|
222
|
+
config.baseModel = config.model; // User's intended model — /model command updates this
|
|
222
223
|
let turnFailedModels = new Set(); // Models that failed this turn (cleared each new turn)
|
|
223
224
|
// Track models that failed with 402 (payment required) across turns.
|
|
224
225
|
// These persist until the session ends — unlike transient errors, payment failures
|
|
@@ -294,9 +295,10 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
|
|
|
294
295
|
// ── Model recovery: try original model at the start of each new turn ──
|
|
295
296
|
// If we fell back to a free model last turn due to a transient error, try original again.
|
|
296
297
|
// But DON'T reset if the original model had a payment failure — it will just fail again.
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
config.
|
|
298
|
+
const baseModel = config.baseModel ?? config.model;
|
|
299
|
+
if (config.model !== baseModel && !paymentFailedModels.has(baseModel)) {
|
|
300
|
+
config.model = baseModel;
|
|
301
|
+
config.onModelChange?.(baseModel);
|
|
300
302
|
}
|
|
301
303
|
turnFailedModels = new Set(); // Fresh slate for transient failures this turn
|
|
302
304
|
const abort = new AbortController();
|
|
@@ -714,6 +716,35 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
|
|
|
714
716
|
});
|
|
715
717
|
}
|
|
716
718
|
}
|
|
719
|
+
// ── Verification gate: run adversarial checks on substantial work ──
|
|
720
|
+
if (shouldVerify(turnToolCalls, turnToolCounts, lastUserInput || '')) {
|
|
721
|
+
try {
|
|
722
|
+
const vResult = await runVerification(history, capabilityMap, client, {
|
|
723
|
+
model: config.model,
|
|
724
|
+
workDir,
|
|
725
|
+
abortSignal: abort.signal,
|
|
726
|
+
onEvent: (e) => { if (e.kind === 'text_delta' && e.text)
|
|
727
|
+
onEvent({ kind: 'text_delta', text: e.text }); },
|
|
728
|
+
});
|
|
729
|
+
if (vResult.verdict === 'FAIL' && vResult.issues.length > 0) {
|
|
730
|
+
// Inject verification feedback — agent will see this and continue fixing
|
|
731
|
+
const feedbackMsg = {
|
|
732
|
+
role: 'user',
|
|
733
|
+
content: `[VERIFICATION FAILED]\n${vResult.summary}\n\nFix the issues above and verify your fixes work.`,
|
|
734
|
+
};
|
|
735
|
+
history.push(feedbackMsg);
|
|
736
|
+
persistSessionMessage(feedbackMsg);
|
|
737
|
+
onEvent({ kind: 'text_delta', text: `\n⚠️ *Verification found issues — fixing...*\n` });
|
|
738
|
+
continue; // Re-enter the loop to fix issues
|
|
739
|
+
}
|
|
740
|
+
if (vResult.verdict === 'PASS') {
|
|
741
|
+
onEvent({ kind: 'text_delta', text: '\n✓ *Verified*\n' });
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
catch {
|
|
745
|
+
// Verification errors never block the main flow
|
|
746
|
+
}
|
|
747
|
+
}
|
|
717
748
|
// Record success for local Elo learning (include tool call count for efficiency)
|
|
718
749
|
if (lastRoutedCategory && lastRoutedModel) {
|
|
719
750
|
recordOutcome(lastRoutedCategory, lastRoutedModel, 'continued', turnToolCalls);
|
package/dist/agent/types.d.ts
CHANGED
|
@@ -142,4 +142,6 @@ export interface AgentConfig {
|
|
|
142
142
|
onAskUser?: (question: string, options?: string[]) => Promise<string>;
|
|
143
143
|
/** Notify UI when agent switches model (e.g. payment fallback) */
|
|
144
144
|
onModelChange?: (model: string) => void;
|
|
145
|
+
/** The user's intended model — updated by /model command, used for turn recovery */
|
|
146
|
+
baseModel?: string;
|
|
145
147
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification Agent — adversarial testing gate.
|
|
3
|
+
*
|
|
4
|
+
* After the main agent completes substantial work (writes/edits files, runs commands),
|
|
5
|
+
* this agent runs independently to try to BREAK what was built. It can only read and
|
|
6
|
+
* execute — never modify files. Returns PASS/FAIL/PARTIAL verdict.
|
|
7
|
+
*
|
|
8
|
+
* If FAIL: injects feedback into conversation so the main agent can fix issues.
|
|
9
|
+
* If PASS: work is considered verified.
|
|
10
|
+
*
|
|
11
|
+
* Inspired by Claude Code's verification agent architecture.
|
|
12
|
+
*/
|
|
13
|
+
import type { CapabilityHandler, Dialogue } from './types.js';
|
|
14
|
+
import { ModelClient } from './llm.js';
|
|
15
|
+
export interface VerificationResult {
|
|
16
|
+
verdict: 'PASS' | 'FAIL' | 'PARTIAL' | 'SKIPPED';
|
|
17
|
+
summary: string;
|
|
18
|
+
issues: string[];
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Should we run verification for this turn?
|
|
22
|
+
* Only for substantial work: 3+ tool calls AND at least one write/edit/bash.
|
|
23
|
+
*/
|
|
24
|
+
export declare function shouldVerify(turnToolCalls: number, turnToolCounts: Map<string, number>, userInput: string): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Filter capability handlers to only allow read-only tools.
|
|
27
|
+
* Bash is allowed (for running tests/builds) but Edit/Write are blocked.
|
|
28
|
+
*/
|
|
29
|
+
export declare function getVerificationTools(handlers: Map<string, CapabilityHandler>): Map<string, CapabilityHandler>;
|
|
30
|
+
/**
|
|
31
|
+
* Run the verification agent on the current conversation state.
|
|
32
|
+
* Uses a cheap model to minimize cost. Returns verdict + issues.
|
|
33
|
+
*/
|
|
34
|
+
export declare function runVerification(history: Dialogue[], handlers: Map<string, CapabilityHandler>, client: ModelClient, config: {
|
|
35
|
+
model: string;
|
|
36
|
+
workDir: string;
|
|
37
|
+
abortSignal: AbortSignal;
|
|
38
|
+
onEvent?: (event: {
|
|
39
|
+
kind: string;
|
|
40
|
+
text?: string;
|
|
41
|
+
}) => void;
|
|
42
|
+
}): Promise<VerificationResult>;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verification Agent — adversarial testing gate.
|
|
3
|
+
*
|
|
4
|
+
* After the main agent completes substantial work (writes/edits files, runs commands),
|
|
5
|
+
* this agent runs independently to try to BREAK what was built. It can only read and
|
|
6
|
+
* execute — never modify files. Returns PASS/FAIL/PARTIAL verdict.
|
|
7
|
+
*
|
|
8
|
+
* If FAIL: injects feedback into conversation so the main agent can fix issues.
|
|
9
|
+
* If PASS: work is considered verified.
|
|
10
|
+
*
|
|
11
|
+
* Inspired by Claude Code's verification agent architecture.
|
|
12
|
+
*/
|
|
13
|
+
// ─── Verification System Prompt ───────────────────────────────────────────
|
|
14
|
+
const VERIFICATION_PROMPT = `You are a VERIFICATION agent. Your job is NOT to confirm that code works — it is to TRY TO BREAK IT.
|
|
15
|
+
|
|
16
|
+
## Rules
|
|
17
|
+
|
|
18
|
+
1. **Adversarial mindset**: Assume the code has bugs. Your goal is to find them.
|
|
19
|
+
2. **No modifications**: You may ONLY use Read, Bash, Glob, and Grep tools. You MUST NOT use Edit, Write, or any tool that modifies files.
|
|
20
|
+
3. **Evidence required**: Every check MUST include:
|
|
21
|
+
- What you tested (the exact command or operation)
|
|
22
|
+
- The actual output
|
|
23
|
+
- Whether it PASSED or FAILED
|
|
24
|
+
4. **No rationalization**: These phrases are NEVER acceptable as evidence:
|
|
25
|
+
- "The code looks correct"
|
|
26
|
+
- "This should work"
|
|
27
|
+
- "Based on the implementation, it handles..."
|
|
28
|
+
- "The tests pass" (unless you actually ran them and showed output)
|
|
29
|
+
|
|
30
|
+
## What to Check
|
|
31
|
+
|
|
32
|
+
1. **Does it compile/build?** Run the build command.
|
|
33
|
+
2. **Do tests pass?** Run the test suite.
|
|
34
|
+
3. **Edge cases**: Empty inputs, very large inputs, missing files, invalid data.
|
|
35
|
+
4. **Error handling**: What happens when things go wrong?
|
|
36
|
+
5. **Consistency**: Does the change break other parts of the codebase?
|
|
37
|
+
|
|
38
|
+
## Output Format
|
|
39
|
+
|
|
40
|
+
After running your checks, output a verdict in EXACTLY this format:
|
|
41
|
+
|
|
42
|
+
VERDICT: PASS|FAIL|PARTIAL
|
|
43
|
+
|
|
44
|
+
Then explain:
|
|
45
|
+
- What you tested
|
|
46
|
+
- What passed
|
|
47
|
+
- What failed (if any)
|
|
48
|
+
- Specific issues to fix (if FAIL)
|
|
49
|
+
|
|
50
|
+
Keep it concise — focus on actionable findings, not narration.`;
|
|
51
|
+
// ─── Thresholds ──────────────────────────────────────────────────────────
|
|
52
|
+
/** Only verify turns where substantial work was done. */
|
|
53
|
+
const WRITE_TOOLS = new Set(['Edit', 'Write', 'Bash']);
|
|
54
|
+
/** Minimum tool calls to trigger verification. */
|
|
55
|
+
const MIN_TOOL_CALLS = 3;
|
|
56
|
+
/** Maximum tokens to spend on verification (prevent runaway). */
|
|
57
|
+
const MAX_VERIFICATION_TOKENS = 8192;
|
|
58
|
+
// ─── Decision Logic ──────────────────────────────────────────────────────
|
|
59
|
+
/**
|
|
60
|
+
* Should we run verification for this turn?
|
|
61
|
+
* Only for substantial work: 3+ tool calls AND at least one write/edit/bash.
|
|
62
|
+
*/
|
|
63
|
+
export function shouldVerify(turnToolCalls, turnToolCounts, userInput) {
|
|
64
|
+
// Skip if not enough tool calls
|
|
65
|
+
if (turnToolCalls < MIN_TOOL_CALLS)
|
|
66
|
+
return false;
|
|
67
|
+
// Skip if no write-like tools were used
|
|
68
|
+
let hasWriteTool = false;
|
|
69
|
+
for (const [name] of turnToolCounts) {
|
|
70
|
+
if (WRITE_TOOLS.has(name)) {
|
|
71
|
+
hasWriteTool = true;
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (!hasWriteTool)
|
|
76
|
+
return false;
|
|
77
|
+
// Skip if user explicitly asked for something quick
|
|
78
|
+
const lower = userInput.toLowerCase();
|
|
79
|
+
if (lower.startsWith('/') || lower.length < 20)
|
|
80
|
+
return false;
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
// ─── Read-only tool filter ───────────────────────────────────────────────
|
|
84
|
+
const READ_ONLY_TOOLS = new Set(['Read', 'Glob', 'Grep', 'Bash', 'WebSearch', 'WebFetch']);
|
|
85
|
+
/**
|
|
86
|
+
* Filter capability handlers to only allow read-only tools.
|
|
87
|
+
* Bash is allowed (for running tests/builds) but Edit/Write are blocked.
|
|
88
|
+
*/
|
|
89
|
+
export function getVerificationTools(handlers) {
|
|
90
|
+
const filtered = new Map();
|
|
91
|
+
for (const [name, handler] of handlers) {
|
|
92
|
+
if (READ_ONLY_TOOLS.has(name)) {
|
|
93
|
+
filtered.set(name, handler);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return filtered;
|
|
97
|
+
}
|
|
98
|
+
// ─── Run Verification ────────────────────────────────────────────────────
|
|
99
|
+
/**
|
|
100
|
+
* Run the verification agent on the current conversation state.
|
|
101
|
+
* Uses a cheap model to minimize cost. Returns verdict + issues.
|
|
102
|
+
*/
|
|
103
|
+
export async function runVerification(history, handlers, client, config) {
|
|
104
|
+
const verificationTools = getVerificationTools(handlers);
|
|
105
|
+
// Build verification prompt from recent history context
|
|
106
|
+
const recentWork = extractRecentWork(history);
|
|
107
|
+
if (!recentWork) {
|
|
108
|
+
return { verdict: 'SKIPPED', summary: 'No recent work to verify.', issues: [] };
|
|
109
|
+
}
|
|
110
|
+
const verificationHistory = [
|
|
111
|
+
{
|
|
112
|
+
role: 'user',
|
|
113
|
+
content: `The following work was just completed. Your job is to VERIFY it by running adversarial checks.\n\n${recentWork}\n\nRun build, tests, and edge case checks. Output your VERDICT.`,
|
|
114
|
+
},
|
|
115
|
+
];
|
|
116
|
+
config.onEvent?.({ kind: 'text_delta', text: '\n*Verifying...*\n' });
|
|
117
|
+
// Use cheap model for verification
|
|
118
|
+
const verificationModel = 'nvidia/nemotron-ultra-253b'; // Free model to keep cost zero
|
|
119
|
+
try {
|
|
120
|
+
// Simple single-turn verification call
|
|
121
|
+
const response = await client.complete({
|
|
122
|
+
model: verificationModel,
|
|
123
|
+
system: VERIFICATION_PROMPT,
|
|
124
|
+
messages: verificationHistory,
|
|
125
|
+
tools: Array.from(verificationTools.values()).map(h => h.spec),
|
|
126
|
+
max_tokens: MAX_VERIFICATION_TOKENS,
|
|
127
|
+
});
|
|
128
|
+
// Extract text from response
|
|
129
|
+
let responseText = '';
|
|
130
|
+
if (response.content) {
|
|
131
|
+
for (const part of response.content) {
|
|
132
|
+
if (typeof part === 'string') {
|
|
133
|
+
responseText += part;
|
|
134
|
+
}
|
|
135
|
+
else if (part.type === 'text') {
|
|
136
|
+
responseText += part.text;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// Parse verdict
|
|
141
|
+
const verdictMatch = responseText.match(/VERDICT:\s*(PASS|FAIL|PARTIAL)/i);
|
|
142
|
+
const verdict = verdictMatch
|
|
143
|
+
? verdictMatch[1].toUpperCase()
|
|
144
|
+
: 'PARTIAL';
|
|
145
|
+
// Extract issues
|
|
146
|
+
const issues = [];
|
|
147
|
+
const issueLines = responseText.split('\n').filter(l => l.match(/^[-•*]\s*(FAIL|ERROR|BUG|ISSUE|PROBLEM)/i) ||
|
|
148
|
+
l.match(/^[-•*]\s+.*fail/i));
|
|
149
|
+
for (const line of issueLines) {
|
|
150
|
+
issues.push(line.replace(/^[-•*]\s*/, '').trim());
|
|
151
|
+
}
|
|
152
|
+
return { verdict, summary: responseText.slice(0, 500), issues };
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
// Verification failure should never block the main flow
|
|
156
|
+
return {
|
|
157
|
+
verdict: 'SKIPPED',
|
|
158
|
+
summary: `Verification error: ${err.message}`,
|
|
159
|
+
issues: [],
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Extract a summary of recent work from the conversation history.
|
|
165
|
+
* Looks at the last assistant turn and its tool calls.
|
|
166
|
+
*/
|
|
167
|
+
function extractRecentWork(history) {
|
|
168
|
+
const parts = [];
|
|
169
|
+
// Walk backwards through history to find recent tool uses and assistant messages
|
|
170
|
+
let found = 0;
|
|
171
|
+
for (let i = history.length - 1; i >= 0 && found < 10; i--) {
|
|
172
|
+
const msg = history[i];
|
|
173
|
+
const role = msg.role;
|
|
174
|
+
// Stop at a pure user message boundary (not a tool_result user message)
|
|
175
|
+
if (role === 'user' && !Array.isArray(msg.content))
|
|
176
|
+
break;
|
|
177
|
+
if (role === 'assistant' && Array.isArray(msg.content)) {
|
|
178
|
+
for (const part of msg.content) {
|
|
179
|
+
if (typeof part === 'object') {
|
|
180
|
+
if (part.type === 'text' && part.text) {
|
|
181
|
+
parts.unshift(`Assistant: ${part.text.slice(0, 500)}`);
|
|
182
|
+
found++;
|
|
183
|
+
}
|
|
184
|
+
else if (part.type === 'tool_use') {
|
|
185
|
+
parts.unshift(`Tool: ${part.name}(${JSON.stringify(part.input).slice(0, 200)})`);
|
|
186
|
+
found++;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
else if (role === 'user' && Array.isArray(msg.content)) {
|
|
192
|
+
for (const part of msg.content) {
|
|
193
|
+
if (typeof part === 'object' && part.type === 'tool_result') {
|
|
194
|
+
const output = typeof part.content === 'string'
|
|
195
|
+
? part.content
|
|
196
|
+
: Array.isArray(part.content)
|
|
197
|
+
? part.content.map(c => c.text || '').join('\n')
|
|
198
|
+
: '';
|
|
199
|
+
parts.unshift(`Result: ${output.slice(0, 300)}`);
|
|
200
|
+
found++;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return parts.length > 0 ? parts.join('\n\n') : null;
|
|
206
|
+
}
|