@blockrun/franklin 3.8.15 → 3.8.16

package/dist/agent/evaluator.d.ts

@@ -53,4 +53,15 @@ export declare function checkGrounding(userInput: string, history: Dialogue[], a
  * user when the check agreed the answer was sound.
  */
 export declare function renderGroundingFollowup(result: GroundingResult): string;
+/**
+ * Build a synthetic user message that instructs the agent to retry with the
+ * missing tools. Returned message goes into history so the model's next
+ * generation sees it as the most recent instruction. This is the GAN-like
+ * feedback loop pattern from Anthropic's harness-design writeup —
+ * evaluator findings feed back into the generator until PASS (or retry cap).
+ *
+ * Intentionally terse: the agent already has the original question in
+ * history; we only need to name the gap + the tools to use.
+ */
+export declare function buildGroundingRetryInstruction(result: GroundingResult, originalUserQuestion: string): string;
 export type { CapabilityHandler };

package/dist/agent/evaluator.js

@@ -231,3 +231,27 @@ export function renderGroundingFollowup(result) {
         : '(evaluator returned no specific items — check the transcript manually)';
     return `\n\n${header}\n${body}\n\n_Ask again with an explicit instruction to call the tools, or disable these checks with \`FRANKLIN_NO_EVAL=1\`._`;
 }
+/**
+ * Build a synthetic user message that instructs the agent to retry with the
+ * missing tools. Returned message goes into history so the model's next
+ * generation sees it as the most recent instruction. This is the GAN-like
+ * feedback loop pattern from Anthropic's harness-design writeup —
+ * evaluator findings feed back into the generator until PASS (or retry cap).
+ *
+ * Intentionally terse: the agent already has the original question in
+ * history; we only need to name the gap + the tools to use.
+ */
+export function buildGroundingRetryInstruction(result, originalUserQuestion) {
+    const lines = [
+        '[GROUNDING CHECK FAILED]',
+        'Your previous answer stated facts without calling the relevant tools. Specifically:',
+    ];
+    for (const issue of result.issues) {
+        lines.push(`- ${issue}`);
+    }
+    lines.push('');
+    lines.push('Retry: call the missing tools first, then give a concise final answer based on the tool results. Only claim what the tool outputs actually say. If a tool fails, say so rather than falling back to memory.');
+    lines.push('');
+    lines.push(`Original user question: ${originalUserQuestion.trim().slice(0, 500)}`);
+    return lines.join('\n');
+}

package/dist/agent/loop.js

@@ -21,11 +21,11 @@ import { appendAudit, extractLastUserPrompt } from '../stats/audit.js';
 import { estimateCost, OPUS_PRICING } from '../pricing.js';
 import { maybeMidSessionExtract } from '../learnings/extractor.js';
 import { extractMentions, buildEntityContext, loadEntities } from '../brain/store.js';
-import { routeRequest, parseRoutingProfile } from '../router/index.js';
+import { routeRequestAsync, parseRoutingProfile } from '../router/index.js';
 import { recordOutcome } from '../router/local-elo.js';
 import { shouldPlan, getPlanningPrompt, getExecutorModel, isExecutorStuck, toolCallSignature } from './planner.js';
 import { shouldVerify, runVerification } from './verification.js';
-import { shouldCheckGrounding, checkGrounding, renderGroundingFollowup } from './evaluator.js';
+import { shouldCheckGrounding, checkGrounding, renderGroundingFollowup, buildGroundingRetryInstruction, } from './evaluator.js';
 import { createSessionId, appendToSession, updateSessionMeta, pruneOldSessions, loadSessionHistory, loadSessionMeta, } from '../session/storage.js';
 /**
  * Atomically replace all elements in a history array.
@@ -525,6 +525,14 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
         let maxTokensOverride;
         const turnIdleReference = lastSessionActivity;
         lastSessionActivity = Date.now();
+        // ── Grounding retry state (per turn) ──
+        // When the post-response evaluator finds UNGROUNDED claims, we inject a
+        // corrective user message and re-enter the loop so the generator can
+        // answer again with the missing tool calls. 1-retry cap: if round 2
+        // still UNGROUNDED, ship the annotated response and let the user
+        // decide — avoids pathological loops, caps wall-clock cost.
+        let groundingRetryCount = 0;
+        const MAX_GROUNDING_RETRIES = 1;
         // ── Plan-then-execute state (per turn) ──
         let planActive = false;
         let planPlannerModel = '';
@@ -688,7 +696,7 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                             .map(p => p.text ?? '')
                             .join(' ')
                         : '';
-                const routing = routeRequest(userText, routingProfile);
+                const routing = await routeRequestAsync(userText, routingProfile);
                 resolvedModel = routing.model;
                 routingTier = routing.tier;
                 routingConfidence = routing.confidence;
@@ -1107,12 +1115,15 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                     }
                 }
                 // ── Grounding gate: check that factual claims trace to tool calls ──
-                // Fires on any substantive answer to a non-trivial question. Designed
-                // to catch the failure mode the code-verifier misses: model answers
-                // a "what's X / should I buy Y" question from memory instead of
-                // calling the live tools. Evaluator runs as a separate agent on a
-                // cheap model; never blocks the turn, only appends a ⚠️ note when
-                // the answer looks ungrounded so the user can re-ask.
+                // Fires on any substantive answer to a non-trivial question. Catches
+                // the failure mode the code-verifier misses: model answers a
+                // "what's X / should I buy Y" question from memory instead of
+                // calling the live tools.
+                //
+                // On UNGROUNDED: inject a corrective user message (GAN-style feedback)
+                // and re-enter the loop so the generator can answer again with the
+                // right tools. Up to MAX_GROUNDING_RETRIES attempts — after that,
+                // annotate and ship so the user can decide.
                 try {
                     const assistantText = responseParts
                         .filter(p => p.type === 'text' && typeof p.text === 'string')
@@ -1122,6 +1133,21 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                         const gResult = await checkGrounding(lastUserInput, history, assistantText, client, {
                             abortSignal: abort.signal,
                         });
+                        if (gResult.verdict === 'UNGROUNDED' && groundingRetryCount < MAX_GROUNDING_RETRIES) {
+                            groundingRetryCount++;
+                            const retryMsg = buildGroundingRetryInstruction(gResult, lastUserInput);
+                            const feedbackMsg = { role: 'user', content: retryMsg };
+                            history.push(feedbackMsg);
+                            persistSessionMessage(feedbackMsg);
+                            onEvent({
+                                kind: 'text_delta',
+                                text: '\n\n*Ungrounded claims detected — retrying with required tool calls...*\n\n',
+                            });
+                            continue; // Re-enter outer loop — generator will produce a new response.
+                        }
+                        // Either the verdict is acceptable (GROUNDED / PARTIAL / SKIPPED)
+                        // or we've hit the retry cap with UNGROUNDED still outstanding.
+                        // In both cases, surface the followup if one applies and exit.
                         const followup = renderGroundingFollowup(gResult);
                         if (followup) {
                             onEvent({ kind: 'text_delta', text: followup });

package/dist/router/index.d.ts

@@ -18,6 +18,20 @@ export interface RoutingResult {
     signals: string[];
     savings: number;
 }
+export type TierClassifier = (prompt: string) => Promise<Tier | null>;
+/**
+ * Default LLM classifier — lazy-imports the ModelClient to avoid a hard
+ * cycle with agent/llm.ts (which itself imports routing helpers for virtual
+ * profile resolution). Callers can substitute their own classifier for
+ * tests by passing one to `routeRequestAsync`.
+ */
+export declare function llmClassifyRequest(prompt: string): Promise<Tier | null>;
+/**
+ * Async router — LLM classifier first, keyword classifier as fallback.
+ * Profile-specific tier tables (AUTO / ECO / PREMIUM / FREE) still pick
+ * the concrete model; the classifier only picks the TIER.
+ */
+export declare function routeRequestAsync(prompt: string, profile?: RoutingProfile, classify?: TierClassifier): Promise<RoutingResult>;
 export declare function routeRequest(prompt: string, profile?: RoutingProfile): RoutingResult;
 /**
  * Get fallback models for a tier

package/dist/router/index.js

@@ -267,6 +267,129 @@ function classicRouteRequest(prompt, profile) {
     const savings = computeSavings(model);
     return { model, tier, confidence, signals, savings };
 }
+// ─── LLM-based classifier ───
+//
+// Historical router was a 15-dimension keyword scorer — every new failure
+// mode needed another KEYWORD list (CODE, REASONING, ANALYSIS, ...). Cheap
+// to run but structurally wrong: keywords always lag reality, and users
+// phrase the same intent fifty different ways. A free model can just
+// *read* the prompt and tell us the tier.
+//
+// Design:
+//   - Classification prompt is one word answer: SIMPLE | MEDIUM | COMPLEX | REASONING
+//   - Runs on a free NVIDIA model — $0/call, so we can afford it on every turn
+//   - 2s hard timeout + strict parse; any failure falls through to the
+//     keyword classifier so we always have a routing answer
+//   - Exposed via async `routeRequestAsync(prompt, profile, classify?)`. Callers
+//     that can't be async (proxy, LLM-client bootstrap) keep using the sync
+//     `routeRequest`, which silently does keyword-only routing.
+const CLASSIFIER_MODEL = process.env.FRANKLIN_ROUTER_MODEL || 'nvidia/nemotron-ultra-253b';
+const CLASSIFIER_TIMEOUT_MS = 2_500;
+const CLASSIFIER_SYSTEM = `You classify a user's message into ONE routing tier for a CLI agent. Reply with EXACTLY ONE WORD from the allowed set. No explanation, no punctuation, no quotes.
+
+Tiers:
+- SIMPLE    — greetings, trivia, arithmetic, short definitions, yes/no questions. A single memory-based reply is acceptable.
+- MEDIUM    — multi-turn code edits, targeted bug fixes, lookups, summaries. Some tool use expected.
+- COMPLEX   — substantive engineering, analysis, recommendations, research questions that depend on current-world data (stock prices, current events, live market state). Multiple tool calls + synthesis.
+- REASONING — formal proofs, derivations, deep chains of logic, multi-variable optimization.
+
+If the message names a ticker, asks for a recommendation, or asks "why did X happen", it is COMPLEX or REASONING — never SIMPLE.
+
+Answer format: a single word. SIMPLE or MEDIUM or COMPLEX or REASONING.`;
+/**
+ * Parse a one-word classifier reply into a Tier. Returns null on junk so
+ * the caller can fall back to keyword classification.
+ */
+function parseTierWord(reply) {
+    const m = reply.trim().toUpperCase().match(/\b(SIMPLE|MEDIUM|COMPLEX|REASONING)\b/);
+    return m ? m[1] : null;
+}
+/**
+ * Default LLM classifier — lazy-imports the ModelClient to avoid a hard
+ * cycle with agent/llm.ts (which itself imports routing helpers for virtual
+ * profile resolution). Callers can substitute their own classifier for
+ * tests by passing one to `routeRequestAsync`.
+ */
+export async function llmClassifyRequest(prompt) {
+    if (!prompt || prompt.trim().length === 0)
+        return null;
+    // Very short messages: skip the classifier call, let keyword path decide.
+    // Saves ~500ms on "hi" / "thanks" / slash commands.
+    if (prompt.trim().length < 10)
+        return null;
+    let ModelClientCtor;
+    let chain;
+    let apiUrl;
+    try {
+        const llmMod = await import('../agent/llm.js');
+        const cfgMod = await import('../config.js');
+        ModelClientCtor = llmMod.ModelClient;
+        chain = cfgMod.loadChain();
+        apiUrl = cfgMod.API_URLS[chain];
+    }
+    catch {
+        return null;
+    }
+    const client = new ModelClientCtor({ apiUrl, chain });
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), CLASSIFIER_TIMEOUT_MS);
+    try {
+        const result = await client.complete({
+            model: CLASSIFIER_MODEL,
+            system: CLASSIFIER_SYSTEM,
+            messages: [{ role: 'user', content: prompt.slice(0, 2000) }],
+            tools: [],
+            max_tokens: 8,
+        }, ctrl.signal);
+        let text = '';
+        for (const part of result.content) {
+            if (typeof part === 'object' && part.type === 'text' && part.text)
+                text += part.text;
+        }
+        return parseTierWord(text);
+    }
+    catch {
+        return null;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Async router — LLM classifier first, keyword classifier as fallback.
+ * Profile-specific tier tables (AUTO / ECO / PREMIUM / FREE) still pick
+ * the concrete model; the classifier only picks the TIER.
+ */
+export async function routeRequestAsync(prompt, profile = 'auto', classify = llmClassifyRequest) {
+    // Free / short-circuit profiles — no classifier needed.
+    if (profile === 'free')
+        return routeRequest(prompt, profile);
+    const tier = await classify(prompt).catch(() => null);
+    if (!tier) {
+        // Classifier miss or disabled — fall through to the sync keyword router.
+        return routeRequest(prompt, profile);
+    }
+    // Build a RoutingResult from the LLM-picked tier using the same tier
+    // tables the keyword path uses. Keeps downstream code path-identical.
+    let tierConfigs;
+    switch (profile) {
+        case 'eco':
+            tierConfigs = ECO_TIERS;
+            break;
+        case 'premium':
+            tierConfigs = PREMIUM_TIERS;
+            break;
+        default: tierConfigs = AUTO_TIERS;
+    }
+    const model = tierConfigs[tier].primary;
+    return {
+        model,
+        tier,
+        confidence: 0.85, // LLM classification — medium-high confidence
+        signals: ['llm-classified'],
+        savings: computeSavings(model),
+    };
+}
 // ─── Main Router ───
 export function routeRequest(prompt, profile = 'auto') {
     // Free profile — always use free model

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blockrun/franklin",
-  "version": "3.8.15",
+  "version": "3.8.16",
   "description": "Franklin — The AI agent with a wallet. Spends USDC autonomously to get real work done. Pay per action, no subscriptions.",
   "type": "module",
   "exports": {