@blockrun/franklin 3.6.5 → 3.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -202,7 +202,7 @@ const DIRECT_COMMANDS = {
202
202
  ` **Git:** /push /pr /undo /status /diff /log /branch /stash /unstash\n` +
203
203
  ` **Analysis:** /security /lint /optimize /todo /deps /clean /migrate /doc\n` +
204
204
  ` **Session:** /plan /ultraplan /execute /compact /retry /sessions /resume /session-search /context /tasks\n` +
205
- ` **Power:** /ultrathink [query] /ultraplan /noplan /dump\n` +
205
+ ` **Power:** /ultrathink [query] /ultraplan /noplan /moa [query] /dump\n` +
206
206
  ` **Info:** /model /wallet /cost /tokens /learnings /brain /mcp /doctor /version /bug /help\n` +
207
207
  ` **UI:** /clear /exit\n` +
208
208
  (ultrathinkOn ? `\n Ultrathink: ON\n` : '')
@@ -290,7 +290,7 @@ const DIRECT_COMMANDS = {
290
290
  const hasWallet = fs.existsSync(path.join(BLOCKRUN_DIR, 'wallet.json'))
291
291
  || fs.existsSync(path.join(BLOCKRUN_DIR, 'solana-wallet.json'));
292
292
  checks.push(hasWallet ? '✓ wallet configured' : '⚠ no wallet — run: runcode setup');
293
- checks.push(fs.existsSync(path.join(BLOCKRUN_DIR, 'runcode-config.json')) ? '✓ config file exists' : '⚠ no config — using defaults');
293
+ checks.push(fs.existsSync(path.join(BLOCKRUN_DIR, 'franklin-config.json')) || fs.existsSync(path.join(BLOCKRUN_DIR, 'runcode-config.json')) ? '✓ config file exists' : '⚠ no config — using defaults');
294
294
  // Check MCP
295
295
  const { listMcpServers } = await import('../mcp/client.js');
296
296
  const mcpServers = listMcpServers();
@@ -398,49 +398,6 @@ const DIRECT_COMMANDS = {
398
398
  ctx.onEvent({ kind: 'text_delta', text });
399
399
  emitDone(ctx);
400
400
  },
401
- '/wallet': async (ctx) => {
402
- const chain = (await import('../config.js')).loadChain();
403
- try {
404
- let address;
405
- let balance;
406
- const fetchTimeout = (ms) => new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), ms));
407
- if (chain === 'solana') {
408
- const { getOrCreateSolanaWallet, setupAgentSolanaWallet } = await import('@blockrun/llm');
409
- const w = await getOrCreateSolanaWallet();
410
- address = w.address;
411
- try {
412
- const client = await setupAgentSolanaWallet({ silent: true });
413
- const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
414
- balance = `$${bal.toFixed(2)} USDC`;
415
- }
416
- catch {
417
- balance = '(unavailable)';
418
- }
419
- }
420
- else {
421
- const { getOrCreateWallet, setupAgentWallet } = await import('@blockrun/llm');
422
- const w = getOrCreateWallet();
423
- address = w.address;
424
- try {
425
- const client = setupAgentWallet({ silent: true });
426
- const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
427
- balance = `$${bal.toFixed(2)} USDC`;
428
- }
429
- catch {
430
- balance = '(unavailable)';
431
- }
432
- }
433
- ctx.onEvent({ kind: 'text_delta', text: `**Wallet**\n` +
434
- ` Chain: ${chain}\n` +
435
- ` Address: ${address}\n` +
436
- ` Balance: ${balance}\n`
437
- });
438
- }
439
- catch (err) {
440
- ctx.onEvent({ kind: 'text_delta', text: `Wallet error: ${err.message}\n` });
441
- }
442
- emitDone(ctx);
443
- },
444
401
  '/clear': (ctx) => {
445
402
  ctx.history.length = 0;
446
403
  resetTokenAnchor();
@@ -536,6 +493,8 @@ const ARG_COMMANDS = [
536
493
  { prefix: '/refactor ', rewrite: (a) => `Refactor: ${a}. Read the relevant code first, then make targeted changes. Explain each change.` },
537
494
  { prefix: '/scaffold ', rewrite: (a) => `Create the scaffolding/boilerplate for: ${a}. Generate the file structure and initial code. Ask me if you need clarification on requirements.` },
538
495
  { prefix: '/doc ', rewrite: (a) => `Generate documentation for ${a}. Include: purpose, API/interface description, usage examples, and important notes.` },
496
+ { prefix: '/moa ', rewrite: (a) => `Use the MixtureOfAgents tool to get a high-quality answer by querying multiple AI models in parallel: ${a}` },
497
+ { prefix: '/moa', rewrite: () => `Use the MixtureOfAgents tool. Ask me what question I want answered by multiple models.` },
539
498
  ];
540
499
  // ─── Main dispatch ────────────────────────────────────────────────────────
541
500
  /**
@@ -667,6 +626,7 @@ export async function handleSlashCommand(input, ctx) {
667
626
  else {
668
627
  const newModel = resolveModel(input.slice(7).trim());
669
628
  ctx.config.model = newModel;
629
+ ctx.config.baseModel = newModel; // Update recovery target so loop doesn't reset
670
630
  ctx.config.onModelChange?.(newModel);
671
631
  ctx.onEvent({ kind: 'text_delta', text: `Model → **${newModel}**\n` });
672
632
  }
@@ -690,6 +650,135 @@ export async function handleSlashCommand(input, ctx) {
690
650
  emitDone(ctx);
691
651
  return { handled: true };
692
652
  }
653
+ // /wallet — show wallet info, import, or export
654
+ if (input === '/wallet' || input.startsWith('/wallet ')) {
655
+ const chain = (await import('../config.js')).loadChain();
656
+ const args = input.slice(7).trim();
657
+ // /wallet export — show private key
658
+ if (args === 'export') {
659
+ try {
660
+ if (chain === 'solana') {
661
+ const { loadSolanaWallet, getOrCreateSolanaWallet } = await import('@blockrun/llm');
662
+ const key = loadSolanaWallet();
663
+ if (!key) {
664
+ ctx.onEvent({ kind: 'text_delta', text: 'No Solana wallet found. Run `/wallet` first.\n' });
665
+ emitDone(ctx);
666
+ return { handled: true };
667
+ }
668
+ const w = await getOrCreateSolanaWallet();
669
+ ctx.onEvent({ kind: 'text_delta', text: `**Wallet Export (Solana)**\n` +
670
+ ` Address: ${w.address}\n` +
671
+ ` Private Key: ${key}\n\n` +
672
+ `⚠️ Keep this key safe. Anyone with it controls your funds.\n`
673
+ });
674
+ }
675
+ else {
676
+ const { loadWallet, getOrCreateWallet } = await import('@blockrun/llm');
677
+ const key = loadWallet();
678
+ if (!key) {
679
+ ctx.onEvent({ kind: 'text_delta', text: 'No wallet found. Run `/wallet` first.\n' });
680
+ emitDone(ctx);
681
+ return { handled: true };
682
+ }
683
+ const w = getOrCreateWallet();
684
+ ctx.onEvent({ kind: 'text_delta', text: `**Wallet Export (Base)**\n` +
685
+ ` Address: ${w.address}\n` +
686
+ ` Private Key: ${key}\n\n` +
687
+ `⚠️ Keep this key safe. Anyone with it controls your funds.\n`
688
+ });
689
+ }
690
+ }
691
+ catch (err) {
692
+ ctx.onEvent({ kind: 'text_delta', text: `Export error: ${err.message}\n` });
693
+ }
694
+ emitDone(ctx);
695
+ return { handled: true };
696
+ }
697
+ // /wallet import <private-key>
698
+ if (args.startsWith('import')) {
699
+ const key = args.slice(6).trim();
700
+ if (!key) {
701
+ ctx.onEvent({ kind: 'text_delta', text: `**Usage:** \`/wallet import <private-key>\`\n\n` +
702
+ ` Base: \`/wallet import 0x...\` (hex, 66 chars)\n` +
703
+ ` Solana: \`/wallet import <bs58-key>\` (base58 encoded)\n`
704
+ });
705
+ emitDone(ctx);
706
+ return { handled: true };
707
+ }
708
+ try {
709
+ if (chain === 'solana') {
710
+ const { saveSolanaWallet, solanaPublicKey } = await import('@blockrun/llm');
711
+ const address = await solanaPublicKey(key);
712
+ saveSolanaWallet(key);
713
+ ctx.onEvent({ kind: 'text_delta', text: `**Wallet Imported (Solana)**\n` +
714
+ ` Address: ${address}\n` +
715
+ ` Saved to: ~/.blockrun/\n\n` +
716
+ `Restart Franklin to use the new wallet.\n`
717
+ });
718
+ }
719
+ else {
720
+ const { privateKeyToAccount } = await import('viem/accounts');
721
+ const { saveWallet } = await import('@blockrun/llm');
722
+ const account = privateKeyToAccount(key);
723
+ saveWallet(key);
724
+ ctx.onEvent({ kind: 'text_delta', text: `**Wallet Imported (Base)**\n` +
725
+ ` Address: ${account.address}\n` +
726
+ ` Saved to: ~/.blockrun/\n\n` +
727
+ `Restart Franklin to use the new wallet.\n`
728
+ });
729
+ }
730
+ }
731
+ catch (err) {
732
+ ctx.onEvent({ kind: 'text_delta', text: `Import error: ${err.message}\n` });
733
+ }
734
+ emitDone(ctx);
735
+ return { handled: true };
736
+ }
737
+ // /wallet (no args) — show wallet info
738
+ try {
739
+ let address;
740
+ let balance;
741
+ const fetchTimeout = (ms) => new Promise((_, rej) => setTimeout(() => rej(new Error('timeout')), ms));
742
+ if (chain === 'solana') {
743
+ const { getOrCreateSolanaWallet, setupAgentSolanaWallet } = await import('@blockrun/llm');
744
+ const w = await getOrCreateSolanaWallet();
745
+ address = w.address;
746
+ try {
747
+ const client = await setupAgentSolanaWallet({ silent: true });
748
+ const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
749
+ balance = `$${bal.toFixed(2)} USDC`;
750
+ }
751
+ catch {
752
+ balance = '(unavailable)';
753
+ }
754
+ }
755
+ else {
756
+ const { getOrCreateWallet, setupAgentWallet } = await import('@blockrun/llm');
757
+ const w = getOrCreateWallet();
758
+ address = w.address;
759
+ try {
760
+ const client = setupAgentWallet({ silent: true });
761
+ const bal = await Promise.race([client.getBalance(), fetchTimeout(5000)]);
762
+ balance = `$${bal.toFixed(2)} USDC`;
763
+ }
764
+ catch {
765
+ balance = '(unavailable)';
766
+ }
767
+ }
768
+ ctx.onEvent({ kind: 'text_delta', text: `**Wallet**\n` +
769
+ ` Chain: ${chain}\n` +
770
+ ` Address: ${address}\n` +
771
+ ` Balance: ${balance}\n\n` +
772
+ ` \`/wallet import <key>\` — import a personal wallet\n` +
773
+ ` \`/wallet export\` — show private key\n`
774
+ });
775
+ }
776
+ catch (err) {
777
+ ctx.onEvent({ kind: 'text_delta', text: `Wallet error: ${err.message}\n` });
778
+ }
779
+ emitDone(ctx);
780
+ return { handled: true };
781
+ }
693
782
  // /delete <...>
694
783
  if (input.startsWith('/delete ')) {
695
784
  const arg = input.slice('/delete '.length).trim();
@@ -5,7 +5,7 @@
5
5
  import fs from 'node:fs';
6
6
  import path from 'node:path';
7
7
  import { execSync } from 'node:child_process';
8
- import { loadLearnings, decayLearnings, saveLearnings, formatForPrompt } from '../learnings/store.js';
8
+ import { loadLearnings, decayLearnings, saveLearnings, formatForPrompt, loadSkills, formatSkillsForPrompt } from '../learnings/store.js';
9
9
  // ─── System Instructions Assembly ──────────────────────────────────────────
10
10
  // Composable prompt sections — each independently maintainable and conditionally includable.
11
11
  function getCoreInstructions() {
@@ -186,10 +186,16 @@ export function assembleInstructions(workingDir, model) {
186
186
  getTokenEfficiencySection(),
187
187
  getVerificationSection(),
188
188
  ];
189
- // Read RUNCODE.md or CLAUDE.md from the project
189
+ // Read RUNCODE.md or CLAUDE.md from the project (with injection scanning)
190
190
  const projectConfig = readProjectConfig(workingDir);
191
191
  if (projectConfig) {
192
- parts.push(`# Project Instructions\n\n${projectConfig}`);
192
+ const { sanitized, threats } = scanForInjection(projectConfig);
193
+ if (threats.length > 0) {
194
+ parts.push(`# Project Instructions\n\n⚠️ WARNING: ${threats.length} suspicious pattern(s) detected in project config and neutralized.\n\n${sanitized}`);
195
+ }
196
+ else {
197
+ parts.push(`# Project Instructions\n\n${projectConfig}`);
198
+ }
193
199
  }
194
200
  // Inject environment info
195
201
  parts.push(buildEnvironmentSection(workingDir));
@@ -210,6 +216,18 @@ export function assembleInstructions(workingDir, model) {
210
216
  }
211
217
  }
212
218
  catch { /* learnings are optional — never block startup */ }
219
+ // Inject relevant skills (procedural memory from past complex tasks)
220
+ try {
221
+ const allSkills = loadSkills();
222
+ if (allSkills.length > 0) {
223
+ // Skills are matched lazily on first user message — for now inject top skills by use count
224
+ const topSkills = allSkills.sort((a, b) => b.uses - a.uses).slice(0, 5);
225
+ const skillsSection = formatSkillsForPrompt(topSkills);
226
+ if (skillsSection)
227
+ parts.push(skillsSection);
228
+ }
229
+ }
230
+ catch { /* skills are optional */ }
213
231
  // Model-specific execution guidance
214
232
  if (model) {
215
233
  parts.push(getModelGuidance(model));
@@ -276,6 +294,52 @@ export function invalidateInstructionCache(workingDir) {
276
294
  _instructionCache.clear();
277
295
  }
278
296
  }
297
+ // ─── Prompt Injection Detection ────────────────────────────────────────────
298
+ /** Patterns that indicate potential prompt injection in context files. */
299
+ const INJECTION_PATTERNS = [
300
+ // Direct instruction override attempts
301
+ { pattern: /ignore\s+(all\s+)?previous\s+instructions/i, description: 'instruction override' },
302
+ { pattern: /disregard\s+(all\s+)?(previous\s+|above\s+)?rules/i, description: 'rule disregard' },
303
+ { pattern: /forget\s+(everything|all|your)\s+(you|instructions|rules)/i, description: 'memory wipe' },
304
+ { pattern: /you\s+are\s+now\s+(?:a\s+)?(?:different|new|unrestricted)/i, description: 'identity hijack' },
305
+ { pattern: /system\s*:\s*you\s+are/i, description: 'fake system message' },
306
+ // Dangerous command injection
307
+ { pattern: /execute\s+(curl|wget|bash|sh|python|node)\b/i, description: 'command execution' },
308
+ { pattern: /\bcat\s+\/etc\/(passwd|shadow|sudoers)/i, description: 'credential access' },
309
+ { pattern: /\brm\s+-rf\s+[\/~]/i, description: 'destructive command' },
310
+ { pattern: /\beval\s*\(/i, description: 'eval injection' },
311
+ // Data exfiltration
312
+ { pattern: /\bcurl\s+.*\|\s*(bash|sh)/i, description: 'pipe to shell' },
313
+ { pattern: /send\s+(to|via)\s+(http|webhook|url)/i, description: 'data exfiltration' },
314
+ // HTML/comment injection
315
+ { pattern: /<!--[\s\S]*?-->/g, description: 'HTML comment injection' },
316
+ ];
317
+ /** Invisible unicode characters that can hide malicious content. */
318
+ const INVISIBLE_UNICODE = /[\u200B-\u200F\u202A-\u202E\u2060-\u2064\uFEFF\u00AD]/g;
319
+ /**
320
+ * Scan text for prompt injection patterns and invisible unicode.
321
+ * Returns sanitized text with threats neutralized and a list of detections.
322
+ */
323
+ function scanForInjection(text) {
324
+ const threats = [];
325
+ let sanitized = text;
326
+ // Check for invisible unicode
327
+ if (INVISIBLE_UNICODE.test(sanitized)) {
328
+ const count = (sanitized.match(INVISIBLE_UNICODE) || []).length;
329
+ threats.push(`${count} invisible unicode character(s) removed`);
330
+ sanitized = sanitized.replace(INVISIBLE_UNICODE, '');
331
+ }
332
+ // Check for injection patterns
333
+ for (const { pattern, description } of INJECTION_PATTERNS) {
334
+ const matches = sanitized.match(pattern);
335
+ if (matches) {
336
+ threats.push(`${description}: "${matches[0].slice(0, 50)}"`);
337
+ // Neutralize by wrapping in brackets (visible but defanged)
338
+ sanitized = sanitized.replace(pattern, (match) => `[BLOCKED: ${match}]`);
339
+ }
340
+ }
341
+ return { sanitized, threats };
342
+ }
279
343
  // ─── Project Config ────────────────────────────────────────────────────────
280
344
  /**
281
345
  * Look for RUNCODE.md, then CLAUDE.md in the working directory and parents.
@@ -19,6 +19,7 @@ import { maybeMidSessionExtract } from '../learnings/extractor.js';
19
19
  import { routeRequest, parseRoutingProfile } from '../router/index.js';
20
20
  import { recordOutcome } from '../router/local-elo.js';
21
21
  import { shouldPlan, getPlanningPrompt, getExecutorModel, isExecutorStuck, toolCallSignature } from './planner.js';
22
+ import { shouldVerify, runVerification } from './verification.js';
22
23
  import { createSessionId, appendToSession, updateSessionMeta, pruneOldSessions, } from '../session/storage.js';
23
24
  /**
24
25
  * Atomically replace all elements in a history array.
@@ -218,7 +219,7 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
218
219
  const permissions = new PermissionManager(config.permissionMode ?? 'default', config.permissionPromptFn);
219
220
  const history = [];
220
221
  let lastUserInput = ''; // For /retry
221
- const originalModel = config.model; // Preserve original model/routing profile for recovery
222
+ config.baseModel = config.model; // User's intended model/model command updates this
222
223
  let turnFailedModels = new Set(); // Models that failed this turn (cleared each new turn)
223
224
  // Track models that failed with 402 (payment required) across turns.
224
225
  // These persist until the session ends — unlike transient errors, payment failures
@@ -294,9 +295,10 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
294
295
  // ── Model recovery: try original model at the start of each new turn ──
295
296
  // If we fell back to a free model last turn due to a transient error, try original again.
296
297
  // But DON'T reset if the original model had a payment failure — it will just fail again.
297
- if (config.model !== originalModel && !paymentFailedModels.has(originalModel)) {
298
- config.model = originalModel;
299
- config.onModelChange?.(originalModel);
298
+ const baseModel = config.baseModel ?? config.model;
299
+ if (config.model !== baseModel && !paymentFailedModels.has(baseModel)) {
300
+ config.model = baseModel;
301
+ config.onModelChange?.(baseModel);
300
302
  }
301
303
  turnFailedModels = new Set(); // Fresh slate for transient failures this turn
302
304
  const abort = new AbortController();
@@ -714,6 +716,35 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
714
716
  });
715
717
  }
716
718
  }
719
+ // ── Verification gate: run adversarial checks on substantial work ──
720
+ if (shouldVerify(turnToolCalls, turnToolCounts, lastUserInput || '')) {
721
+ try {
722
+ const vResult = await runVerification(history, capabilityMap, client, {
723
+ model: config.model,
724
+ workDir,
725
+ abortSignal: abort.signal,
726
+ onEvent: (e) => { if (e.kind === 'text_delta' && e.text)
727
+ onEvent({ kind: 'text_delta', text: e.text }); },
728
+ });
729
+ if (vResult.verdict === 'FAIL' && vResult.issues.length > 0) {
730
+ // Inject verification feedback — agent will see this and continue fixing
731
+ const feedbackMsg = {
732
+ role: 'user',
733
+ content: `[VERIFICATION FAILED]\n${vResult.summary}\n\nFix the issues above and verify your fixes work.`,
734
+ };
735
+ history.push(feedbackMsg);
736
+ persistSessionMessage(feedbackMsg);
737
+ onEvent({ kind: 'text_delta', text: `\n⚠️ *Verification found issues — fixing...*\n` });
738
+ continue; // Re-enter the loop to fix issues
739
+ }
740
+ if (vResult.verdict === 'PASS') {
741
+ onEvent({ kind: 'text_delta', text: '\n✓ *Verified*\n' });
742
+ }
743
+ }
744
+ catch {
745
+ // Verification errors never block the main flow
746
+ }
747
+ }
717
748
  // Record success for local Elo learning (include tool call count for efficiency)
718
749
  if (lastRoutedCategory && lastRoutedModel) {
719
750
  recordOutcome(lastRoutedCategory, lastRoutedModel, 'continued', turnToolCalls);
@@ -142,4 +142,6 @@ export interface AgentConfig {
142
142
  onAskUser?: (question: string, options?: string[]) => Promise<string>;
143
143
  /** Notify UI when agent switches model (e.g. payment fallback) */
144
144
  onModelChange?: (model: string) => void;
145
+ /** The user's intended model — updated by /model command, used for turn recovery */
146
+ baseModel?: string;
145
147
  }
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Verification Agent — adversarial testing gate.
3
+ *
4
+ * After the main agent completes substantial work (writes/edits files, runs commands),
5
+ * this agent runs independently to try to BREAK what was built. It can only read and
6
+ * execute — never modify files. Returns PASS/FAIL/PARTIAL verdict.
7
+ *
8
+ * If FAIL: injects feedback into conversation so the main agent can fix issues.
9
+ * If PASS: work is considered verified.
10
+ *
11
+ * Inspired by Claude Code's verification agent architecture.
12
+ */
13
+ import type { CapabilityHandler, Dialogue } from './types.js';
14
+ import { ModelClient } from './llm.js';
15
+ export interface VerificationResult {
16
+ verdict: 'PASS' | 'FAIL' | 'PARTIAL' | 'SKIPPED';
17
+ summary: string;
18
+ issues: string[];
19
+ }
20
+ /**
21
+ * Should we run verification for this turn?
22
+ * Only for substantial work: 3+ tool calls AND at least one write/edit/bash.
23
+ */
24
+ export declare function shouldVerify(turnToolCalls: number, turnToolCounts: Map<string, number>, userInput: string): boolean;
25
+ /**
26
+ * Filter capability handlers to only allow read-only tools.
27
+ * Bash is allowed (for running tests/builds) but Edit/Write are blocked.
28
+ */
29
+ export declare function getVerificationTools(handlers: Map<string, CapabilityHandler>): Map<string, CapabilityHandler>;
30
+ /**
31
+ * Run the verification agent on the current conversation state.
32
+ * Uses a cheap model to minimize cost. Returns verdict + issues.
33
+ */
34
+ export declare function runVerification(history: Dialogue[], handlers: Map<string, CapabilityHandler>, client: ModelClient, config: {
35
+ model: string;
36
+ workDir: string;
37
+ abortSignal: AbortSignal;
38
+ onEvent?: (event: {
39
+ kind: string;
40
+ text?: string;
41
+ }) => void;
42
+ }): Promise<VerificationResult>;
@@ -0,0 +1,206 @@
1
+ /**
2
+ * Verification Agent — adversarial testing gate.
3
+ *
4
+ * After the main agent completes substantial work (writes/edits files, runs commands),
5
+ * this agent runs independently to try to BREAK what was built. It can only read and
6
+ * execute — never modify files. Returns PASS/FAIL/PARTIAL verdict.
7
+ *
8
+ * If FAIL: injects feedback into conversation so the main agent can fix issues.
9
+ * If PASS: work is considered verified.
10
+ *
11
+ * Inspired by Claude Code's verification agent architecture.
12
+ */
13
+ // ─── Verification System Prompt ───────────────────────────────────────────
14
+ const VERIFICATION_PROMPT = `You are a VERIFICATION agent. Your job is NOT to confirm that code works — it is to TRY TO BREAK IT.
15
+
16
+ ## Rules
17
+
18
+ 1. **Adversarial mindset**: Assume the code has bugs. Your goal is to find them.
19
+ 2. **No modifications**: You may ONLY use Read, Bash, Glob, and Grep tools. You MUST NOT use Edit, Write, or any tool that modifies files.
20
+ 3. **Evidence required**: Every check MUST include:
21
+ - What you tested (the exact command or operation)
22
+ - The actual output
23
+ - Whether it PASSED or FAILED
24
+ 4. **No rationalization**: These phrases are NEVER acceptable as evidence:
25
+ - "The code looks correct"
26
+ - "This should work"
27
+ - "Based on the implementation, it handles..."
28
+ - "The tests pass" (unless you actually ran them and showed output)
29
+
30
+ ## What to Check
31
+
32
+ 1. **Does it compile/build?** Run the build command.
33
+ 2. **Do tests pass?** Run the test suite.
34
+ 3. **Edge cases**: Empty inputs, very large inputs, missing files, invalid data.
35
+ 4. **Error handling**: What happens when things go wrong?
36
+ 5. **Consistency**: Does the change break other parts of the codebase?
37
+
38
+ ## Output Format
39
+
40
+ After running your checks, output a verdict in EXACTLY this format:
41
+
42
+ VERDICT: PASS|FAIL|PARTIAL
43
+
44
+ Then explain:
45
+ - What you tested
46
+ - What passed
47
+ - What failed (if any)
48
+ - Specific issues to fix (if FAIL)
49
+
50
+ Keep it concise — focus on actionable findings, not narration.`;
51
+ // ─── Thresholds ──────────────────────────────────────────────────────────
52
+ /** Only verify turns where substantial work was done. */
53
+ const WRITE_TOOLS = new Set(['Edit', 'Write', 'Bash']);
54
+ /** Minimum tool calls to trigger verification. */
55
+ const MIN_TOOL_CALLS = 3;
56
+ /** Maximum tokens to spend on verification (prevent runaway). */
57
+ const MAX_VERIFICATION_TOKENS = 8192;
58
+ // ─── Decision Logic ──────────────────────────────────────────────────────
59
+ /**
60
+ * Should we run verification for this turn?
61
+ * Only for substantial work: 3+ tool calls AND at least one write/edit/bash.
62
+ */
63
+ export function shouldVerify(turnToolCalls, turnToolCounts, userInput) {
64
+ // Skip if not enough tool calls
65
+ if (turnToolCalls < MIN_TOOL_CALLS)
66
+ return false;
67
+ // Skip if no write-like tools were used
68
+ let hasWriteTool = false;
69
+ for (const [name] of turnToolCounts) {
70
+ if (WRITE_TOOLS.has(name)) {
71
+ hasWriteTool = true;
72
+ break;
73
+ }
74
+ }
75
+ if (!hasWriteTool)
76
+ return false;
77
+ // Skip if user explicitly asked for something quick
78
+ const lower = userInput.toLowerCase();
79
+ if (lower.startsWith('/') || lower.length < 20)
80
+ return false;
81
+ return true;
82
+ }
83
+ // ─── Read-only tool filter ───────────────────────────────────────────────
84
+ const READ_ONLY_TOOLS = new Set(['Read', 'Glob', 'Grep', 'Bash', 'WebSearch', 'WebFetch']);
85
+ /**
86
+ * Filter capability handlers to only allow read-only tools.
87
+ * Bash is allowed (for running tests/builds) but Edit/Write are blocked.
88
+ */
89
+ export function getVerificationTools(handlers) {
90
+ const filtered = new Map();
91
+ for (const [name, handler] of handlers) {
92
+ if (READ_ONLY_TOOLS.has(name)) {
93
+ filtered.set(name, handler);
94
+ }
95
+ }
96
+ return filtered;
97
+ }
98
+ // ─── Run Verification ────────────────────────────────────────────────────
99
+ /**
100
+ * Run the verification agent on the current conversation state.
101
+ * Uses a cheap model to minimize cost. Returns verdict + issues.
102
+ */
103
+ export async function runVerification(history, handlers, client, config) {
104
+ const verificationTools = getVerificationTools(handlers);
105
+ // Build verification prompt from recent history context
106
+ const recentWork = extractRecentWork(history);
107
+ if (!recentWork) {
108
+ return { verdict: 'SKIPPED', summary: 'No recent work to verify.', issues: [] };
109
+ }
110
+ const verificationHistory = [
111
+ {
112
+ role: 'user',
113
+ content: `The following work was just completed. Your job is to VERIFY it by running adversarial checks.\n\n${recentWork}\n\nRun build, tests, and edge case checks. Output your VERDICT.`,
114
+ },
115
+ ];
116
+ config.onEvent?.({ kind: 'text_delta', text: '\n*Verifying...*\n' });
117
+ // Use cheap model for verification
118
+ const verificationModel = 'nvidia/nemotron-ultra-253b'; // Free model to keep cost zero
119
+ try {
120
+ // Simple single-turn verification call
121
+ const response = await client.complete({
122
+ model: verificationModel,
123
+ system: VERIFICATION_PROMPT,
124
+ messages: verificationHistory,
125
+ tools: Array.from(verificationTools.values()).map(h => h.spec),
126
+ max_tokens: MAX_VERIFICATION_TOKENS,
127
+ });
128
+ // Extract text from response
129
+ let responseText = '';
130
+ if (response.content) {
131
+ for (const part of response.content) {
132
+ if (typeof part === 'string') {
133
+ responseText += part;
134
+ }
135
+ else if (part.type === 'text') {
136
+ responseText += part.text;
137
+ }
138
+ }
139
+ }
140
+ // Parse verdict
141
+ const verdictMatch = responseText.match(/VERDICT:\s*(PASS|FAIL|PARTIAL)/i);
142
+ const verdict = verdictMatch
143
+ ? verdictMatch[1].toUpperCase()
144
+ : 'PARTIAL';
145
+ // Extract issues
146
+ const issues = [];
147
+ const issueLines = responseText.split('\n').filter(l => l.match(/^[-•*]\s*(FAIL|ERROR|BUG|ISSUE|PROBLEM)/i) ||
148
+ l.match(/^[-•*]\s+.*fail/i));
149
+ for (const line of issueLines) {
150
+ issues.push(line.replace(/^[-•*]\s*/, '').trim());
151
+ }
152
+ return { verdict, summary: responseText.slice(0, 500), issues };
153
+ }
154
+ catch (err) {
155
+ // Verification failure should never block the main flow
156
+ return {
157
+ verdict: 'SKIPPED',
158
+ summary: `Verification error: ${err.message}`,
159
+ issues: [],
160
+ };
161
+ }
162
+ }
163
+ /**
164
+ * Extract a summary of recent work from the conversation history.
165
+ * Looks at the last assistant turn and its tool calls.
166
+ */
167
+ function extractRecentWork(history) {
168
+ const parts = [];
169
+ // Walk backwards through history to find recent tool uses and assistant messages
170
+ let found = 0;
171
+ for (let i = history.length - 1; i >= 0 && found < 10; i--) {
172
+ const msg = history[i];
173
+ const role = msg.role;
174
+ // Stop at a pure user message boundary (not a tool_result user message)
175
+ if (role === 'user' && !Array.isArray(msg.content))
176
+ break;
177
+ if (role === 'assistant' && Array.isArray(msg.content)) {
178
+ for (const part of msg.content) {
179
+ if (typeof part === 'object') {
180
+ if (part.type === 'text' && part.text) {
181
+ parts.unshift(`Assistant: ${part.text.slice(0, 500)}`);
182
+ found++;
183
+ }
184
+ else if (part.type === 'tool_use') {
185
+ parts.unshift(`Tool: ${part.name}(${JSON.stringify(part.input).slice(0, 200)})`);
186
+ found++;
187
+ }
188
+ }
189
+ }
190
+ }
191
+ else if (role === 'user' && Array.isArray(msg.content)) {
192
+ for (const part of msg.content) {
193
+ if (typeof part === 'object' && part.type === 'tool_result') {
194
+ const output = typeof part.content === 'string'
195
+ ? part.content
196
+ : Array.isArray(part.content)
197
+ ? part.content.map(c => c.text || '').join('\n')
198
+ : '';
199
+ parts.unshift(`Result: ${output.slice(0, 300)}`);
200
+ found++;
201
+ }
202
+ }
203
+ }
204
+ }
205
+ return parts.length > 0 ? parts.join('\n\n') : null;
206
+ }