@blockrun/franklin 3.8.36 → 3.8.38

package/dist/agent/evaluator.d.ts

@@ -31,7 +31,9 @@ export interface GroundingResult {
 }
 /**
  * Decide whether this turn warrants a grounding check. Principles:
- * - Non-trivial user input (not a greeting, not a slash command)
+ * - Non-trivial user input (not a greeting, not a slash command), OR
+ *   the assistant answer contains specific factual claims (numbers + units,
+ *   currency, dates, times) regardless of input length
  * - Non-trivial assistant text output (not just a tool-result echo)
  *
  * Intentionally NOT gating on tool-type (read vs write) — the whole point
@@ -69,5 +71,17 @@ export declare function renderGroundingFollowup(result: GroundingResult): string
  * Intentionally terse: the agent already has the original question in
  * history; we only need to name the gap + the tools to use.
  */
+/**
+ * Pull the tool names the evaluator suggested out of its issue lines.
+ * Issue lines look like:
+ *   Claim: "..." → missing tool: WebSearch
+ *   Refusal: "..." → should have called: TradingMarket
+ *   ... → missing tool: WebSearch (or any distance calculation tool)
+ *
+ * Returns first-token-of-each-comma/pipe-segment names, deduplicated.
+ * Used by both the retry instruction (to name them in prose) and the
+ * loop's tool_choice selection (to pin the next request to a tool).
+ */
+export declare function extractMissingToolNames(result: GroundingResult): string[];
 export declare function buildGroundingRetryInstruction(result: GroundingResult, originalUserQuestion: string): string;
 export type { CapabilityHandler };

package/dist/agent/evaluator.js

@@ -71,11 +71,19 @@ If not GROUNDED, list each issue on its own line starting with "- " and the tool
 
 Empty line between verdict and list. No other text. No preamble. No apology. Be terse.`;
 // ─── Trigger policy ──────────────────────────────────────────────────────
-const MIN_USER_CHARS = 20; // Short inputs are greetings/acks, not questions
+const MIN_USER_CHARS = 3; // "hi"/"ok"/"no" skip; "BTC"/"21044" do not
 const MIN_ANSWER_CHARS = 50; // Short answers are acks, not factual claims
+// Factual-content patterns: digits paired with units, currency, dates, or
+// percent/temperature/time signs. If the assistant emitted any of these in
+// a >= MIN_ANSWER_CHARS reply, we check grounding regardless of how short
+// the user's input was — a 5-char ZIP code "21044" can elicit a fabricated
+// weather paragraph, and the original user-length gate let that through.
+const FACTUAL_PATTERN = /(\$\s*\d|\d[\d,]*\s*(?:°[CF]?|%|km|mi|miles?|mph|kph|kg|lbs?|ft|in|cm|hours?|hrs?|minutes?|mins?|seconds?|secs?|GB|MB|KB|TB|USD|EUR|CNY|JPY|BTC|ETH|SOL)|\b(?:19|20)\d{2}-\d{1,2}-\d{1,2}\b|\b\d{1,2}:\d{2}\s*(?:AM|PM|am|pm)?\b|\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2}\b)/;
 /**
  * Decide whether this turn warrants a grounding check. Principles:
- * - Non-trivial user input (not a greeting, not a slash command)
+ * - Non-trivial user input (not a greeting, not a slash command), OR
+ *   the assistant answer contains specific factual claims (numbers + units,
+ *   currency, dates, times) regardless of input length
  * - Non-trivial assistant text output (not just a tool-result echo)
  *
  * Intentionally NOT gating on tool-type (read vs write) — the whole point
@@ -85,11 +93,17 @@ export function shouldCheckGrounding(userInput, assistantText) {
     if (process.env.FRANKLIN_NO_EVAL === '1')
         return false;
     const ui = userInput.trim();
-    if (ui.length < MIN_USER_CHARS)
-        return false;
     if (ui.startsWith('/'))
         return false;
-    if (assistantText.trim().length < MIN_ANSWER_CHARS)
+    const at = assistantText.trim();
+    if (at.length < MIN_ANSWER_CHARS)
+        return false;
+    // If the answer looks factual (numbers + units, dates, prices), check
+    // even when the user's prompt was a single token. The 21044 zip-code
+    // case lived here.
+    if (FACTUAL_PATTERN.test(at))
+        return true;
+    if (ui.length < MIN_USER_CHARS)
         return false;
     return true;
 }
@@ -293,16 +307,49 @@ export function renderGroundingFollowup(result) {
  * Intentionally terse: the agent already has the original question in
  * history; we only need to name the gap + the tools to use.
  */
+/**
+ * Pull the tool names the evaluator suggested out of its issue lines.
+ * Issue lines look like:
+ *   Claim: "..." → missing tool: WebSearch
+ *   Refusal: "..." → should have called: TradingMarket
+ *   ... → missing tool: WebSearch (or any distance calculation tool)
+ *
+ * Returns first-token-of-each-comma/pipe-segment names, deduplicated.
+ * Used by both the retry instruction (to name them in prose) and the
+ * loop's tool_choice selection (to pin the next request to a tool).
+ */
+export function extractMissingToolNames(result) {
+    const names = new Set();
+    for (const issue of result.issues) {
+        const m = issue.match(/(?:missing tool|should have called):\s*([A-Za-z][\w| ,/-]*)/i);
+        if (!m)
+            continue;
+        for (const tok of m[1].split(/[|,/]/)) {
+            const t = tok.trim().split(/\s+/)[0];
+            if (t && t !== '...' && t !== '(or' && t !== '(any')
+                names.add(t);
+        }
+    }
+    return Array.from(names);
+}
 export function buildGroundingRetryInstruction(result, originalUserQuestion) {
+    const namedTools = extractMissingToolNames(result);
+    const toolList = namedTools.length > 0
+        ? namedTools.join(', ')
+        : '(see the missing-tool fields in the issues above)';
     const lines = [
-        '[GROUNDING CHECK FAILED]',
-        'Your previous answer stated facts without calling the relevant tools. Specifically:',
+        '[GROUNDING CHECK FAILED — RETRY ROUND]',
+        'Your previous answer stated facts without calling tools. Specifically:',
     ];
     for (const issue of result.issues) {
         lines.push(`- ${issue}`);
     }
     lines.push('');
-    lines.push('Retry: call the missing tools first, then give a concise final answer based on the tool results. Only claim what the tool outputs actually say. If a tool fails, say so rather than falling back to memory.');
+    lines.push('## What you must do this round');
+    lines.push(`1. **Call these tools first**, before any prose: ${toolList}.`);
+    lines.push('2. **Do not write a single factual sentence until the tool results return.** No restatement of the prior answer, no hedging, no "based on general knowledge".');
+    lines.push('3. **Do NOT invent source names** (no fake URLs, no fabricated citation domains, no "per Trippy" / "per drivvin.com" — if you cite a source, it must come from a tool result you just ran).');
+    lines.push('4. After tools return, write a concise answer that ONLY restates what the tool outputs say. If a result is partial or a tool failed, say so explicitly — do not paper over with memory.');
     lines.push('');
     lines.push(`Original user question: ${originalUserQuestion.trim().slice(0, 500)}`);
     return lines.join('\n');

package/dist/agent/llm.d.ts

@@ -5,6 +5,30 @@
  */
 import { type Chain } from '../config.js';
 import type { Dialogue, CapabilityDefinition, ContentPart, CapabilityInvocation } from './types.js';
+/**
+ * Anthropic-compatible tool_choice. Forwarded as-is through the proxy and on
+ * to the backend (Anthropic / OpenAI / Gemini gateways translate as needed).
+ *
+ * - `auto`  — model decides (default if omitted)
+ * - `any`   — must call SOME tool, model picks which
+ * - `tool`  — must call the specifically named tool
+ * - `none`  — must not call any tool
+ *
+ * Used by the grounding-retry path in `loop.ts`: when the evaluator catches
+ * an ungrounded answer that should have invoked tools, the next round sets
+ * `tool_choice` to force tool use rather than relying on a soft instruction
+ * the model can defy by fabricating citations.
+ */
+export type ToolChoice = {
+    type: 'auto';
+} | {
+    type: 'any';
+} | {
+    type: 'tool';
+    name: string;
+} | {
+    type: 'none';
+};
 export interface ModelRequest {
     model: string;
     messages: Dialogue[];
@@ -13,6 +37,7 @@ export interface ModelRequest {
     max_tokens?: number;
     stream?: boolean;
     temperature?: number;
+    tool_choice?: ToolChoice;
 }
 export interface StreamChunk {
     kind: 'content_block_start' | 'content_block_delta' | 'content_block_stop' | 'message_start' | 'message_delta' | 'message_stop' | 'ping' | 'error';

package/dist/agent/llm.js

@@ -273,6 +273,12 @@ export class ModelClient {
         const isGLM = request.model.startsWith('zai/') || request.model.includes('glm');
         // Build the request payload, injecting model-specific optimizations
         let requestPayload = { ...request, stream: true };
+        // Safety: tool_choice without tools causes upstream 400. Strip rather
+        // than reject so callers don't have to coordinate the two fields.
+        if (requestPayload['tool_choice'] !== undefined &&
+            (!Array.isArray(requestPayload['tools']) || requestPayload['tools'].length === 0)) {
+            delete requestPayload['tool_choice'];
+        }
         // ── GLM-specific optimizations ───────────────────────────────────────────
         // GLM models work best with temperature=0.8 per official zai spec.
         // Enable thinking mode only for explicit reasoning variants (-thinking-).

package/dist/agent/loop.js

@@ -25,7 +25,7 @@ import { routeRequestAsync, resolveTierToModel, parseRoutingProfile } from '../r
 import { recordOutcome } from '../router/local-elo.js';
 import { shouldPlan, getPlanningPrompt, getExecutorModel, isExecutorStuck, toolCallSignature } from './planner.js';
 import { shouldVerify, runVerification } from './verification.js';
-import { shouldCheckGrounding, checkGrounding, renderGroundingFollowup, buildGroundingRetryInstruction, } from './evaluator.js';
+import { shouldCheckGrounding, checkGrounding, renderGroundingFollowup, buildGroundingRetryInstruction, extractMissingToolNames, } from './evaluator.js';
 import { augmentUserMessage, prefetchForIntent } from './intent-prefetch.js';
 import { analyzeTurn } from './turn-analyzer.js';
 import { createSessionId, appendToSession, updateSessionMeta, pruneOldSessions, loadSessionHistory, loadSessionMeta, } from '../session/storage.js';
@@ -464,6 +464,12 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
         // decide — avoids pathological loops, caps wall-clock cost.
         let groundingRetryCount = 0;
         const MAX_GROUNDING_RETRIES = 1;
+        // When the previous round failed grounding and we're retrying, force the
+        // model to actually call a tool this round instead of trusting it to
+        // comply with a soft instruction. Single-shot — cleared after attached.
+        // Set to `{ type: "tool", name: "X" }` if the evaluator named exactly
+        // one available tool, else `{ type: "any" }` so the model picks.
+        let forceToolChoiceNextRound = null;
         // ── Plan-then-execute state (per turn) ──
         let planActive = false;
         let planPlannerModel = '';
@@ -767,6 +773,11 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
             if (sanitized.length !== history.length) {
                 replaceHistory(history, sanitized);
             }
+            // Consume any pending forced tool_choice from the previous round's
+            // grounding-retry decision. `tool_choice` is dropped automatically in
+            // llm.ts if `tools` ended up empty, so it's safe to attach here.
+            const callToolChoice = forceToolChoiceNextRound;
+            forceToolChoiceNextRound = null;
             try {
                 const result = await client.complete({
                     model: resolvedModel,
@@ -775,6 +786,7 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                     tools: callToolDefs,
                     max_tokens: callMaxTokens,
                     stream: true,
+                    ...(callToolChoice ? { tool_choice: callToolChoice } : {}),
                 }, abort.signal, 
                 // Start concurrent tools as soon as their input is fully received
                 (tool) => streamExec.onToolReceived(tool), 
@@ -1144,9 +1156,24 @@ export async function interactiveSession(config, getUserInput, onEvent, onAbortR
                             const feedbackMsg = { role: 'user', content: retryMsg };
                             history.push(feedbackMsg);
                             persistSessionMessage(feedbackMsg);
+                            // Hard enforcement: set tool_choice so the model can't fabricate
+                            // citations in lieu of running tools (the round-2 failure mode
+                            // from the Tampa→Miami log). If the evaluator named exactly one
+                            // available tool, pin to it; otherwise force "any" tool use.
+                            const namedTools = extractMissingToolNames(gResult);
+                            const availableNames = new Set(buildCallToolDefs().map(t => t.name));
+                            const matched = namedTools.filter(n => availableNames.has(n));
+                            if (matched.length === 1) {
+                                forceToolChoiceNextRound = { type: 'tool', name: matched[0] };
+                            }
+                            else if (availableNames.size > 0) {
+                                forceToolChoiceNextRound = { type: 'any' };
+                            }
                             onEvent({
                                 kind: 'text_delta',
-                                text: '\n\n*Ungrounded claims detected — retrying with required tool calls...*\n\n',
+                                text: forceToolChoiceNextRound
+                                    ? `\n\n*Ungrounded claims detected — forcing tool use (${forceToolChoiceNextRound.type === 'tool' ? forceToolChoiceNextRound.name : 'any'}) and retrying...*\n\n`
+                                    : '\n\n*Ungrounded claims detected — retrying with required tool calls...*\n\n',
                             });
                             continue; // Re-enter outer loop — generator will produce a new response.
                         }

package/dist/proxy/server.js

@@ -5,7 +5,7 @@ import os from 'node:os';
 import { getOrCreateWallet, getOrCreateSolanaWallet, createPaymentPayload, createSolanaPaymentPayload, parsePaymentRequired, extractPaymentDetails, solanaKeyToBytes, SOLANA_NETWORK, } from '@blockrun/llm';
 import { recordUsage } from '../stats/tracker.js';
 import { appendAudit } from '../stats/audit.js';
-import { fetchWithFallback, buildFallbackChain, DEFAULT_FALLBACK_CONFIG, ROUTING_PROFILES, } from './fallback.js';
+import { buildFallbackChain, DEFAULT_FALLBACK_CONFIG, ROUTING_PROFILES, } from './fallback.js';
 import { routeRequest, parseRoutingProfile, } from '../router/index.js';
 import { estimateCost } from '../pricing.js';
 import { VERSION } from '../config.js';
@@ -41,6 +41,57 @@ function log(...args) {
     catch { /* ignore */ }
 }
 const DEFAULT_MAX_TOKENS = 4096;
+const DEFAULT_PROXY_REQUEST_TIMEOUT_MS = 45_000;
+const DEFAULT_PROXY_STREAM_TIMEOUT_MS = 5 * 60 * 1000;
+function parseTimeoutEnv(name, fallback) {
+    const raw = process.env[name];
+    if (!raw)
+        return fallback;
+    const parsed = Number.parseInt(raw, 10);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback;
+}
+function getProxyRequestTimeoutMs() {
+    return parseTimeoutEnv('FRANKLIN_PROXY_REQUEST_TIMEOUT_MS', DEFAULT_PROXY_REQUEST_TIMEOUT_MS);
+}
+function getProxyStreamTimeoutMs() {
+    return parseTimeoutEnv('FRANKLIN_PROXY_STREAM_TIMEOUT_MS', DEFAULT_PROXY_STREAM_TIMEOUT_MS);
+}
+function createProxyTimeoutError(label, timeoutMs) {
+    return new Error(`${label} timed out after ${timeoutMs}ms`);
+}
+async function fetchWithTimeout(url, init, timeoutMs, label) {
+    if (timeoutMs <= 0)
+        return fetch(url, init);
+    const controller = new AbortController();
+    const timeoutError = createProxyTimeoutError(label, timeoutMs);
+    const timeout = setTimeout(() => {
+        try {
+            controller.abort(timeoutError);
+        }
+        catch { /* ignore */ }
+    }, timeoutMs);
+    try {
+        return await fetch(url, { ...init, signal: controller.signal });
+    }
+    catch (err) {
+        if (controller.signal.aborted)
+            throw timeoutError;
+        throw err;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+function replaceModelInBody(body, model) {
+    try {
+        const parsed = JSON.parse(body);
+        parsed.model = model;
+        return JSON.stringify(parsed);
+    }
+    catch {
+        return body;
+    }
+}
 // Per-model last output tokens for adaptive max_tokens (avoids cross-request pollution)
 const MAX_TRACKED_MODELS = 50;
 const lastOutputByModel = new Map();
@@ -369,13 +420,21 @@ export function createProxy(options) {
                 };
                 let response;
                 let finalModel = requestModel;
+                const requestTimeoutMs = getProxyRequestTimeoutMs();
                 // Use fallback chain if enabled
                 if (fallbackEnabled && body && requestPath.includes('messages')) {
                     const fallbackConfig = {
                         ...DEFAULT_FALLBACK_CONFIG,
                         chain: buildFallbackChain(requestModel),
                     };
-                    const result = await fetchWithFallback(targetUrl, requestInit, body, fallbackConfig, (failedModel, status, nextModel) => {
+                    const result = await fetchWithPaymentFallback(targetUrl, requestInit, body, fallbackConfig, {
+                        method: req.method || 'POST',
+                        headers,
+                        chain,
+                        baseWallet,
+                        solanaWallet,
+                        timeoutMs: requestTimeoutMs,
+                    }, (failedModel, status, nextModel) => {
                         log(`⚠️  ${failedModel} returned ${status}, falling back to ${nextModel}`);
                     });
                     response = result.response;
@@ -388,20 +447,14 @@ export function createProxy(options) {
                     }
                 }
                 else {
-                    // Direct fetch without fallback (with timeout)
-                    const directCtrl = new AbortController();
-                    const directTimeout = setTimeout(() => directCtrl.abort(), 120_000); // 2min
-                    response = await fetch(targetUrl, { ...requestInit, signal: directCtrl.signal });
-                    clearTimeout(directTimeout);
-                }
-                // Handle 402 payment — body now has the correct model after fallback
-                if (response.status === 402) {
-                    if (chain === 'solana' && solanaWallet) {
-                        response = await handleSolanaPayment(response, targetUrl, req.method || 'POST', headers, body, solanaWallet.privateKey, solanaWallet.address);
-                    }
-                    else if (baseWallet) {
-                        response = await handleBasePayment(response, targetUrl, req.method || 'POST', headers, body, baseWallet.privateKey, baseWallet.address);
-                    }
+                    response = await fetchModelAttempt(targetUrl, requestInit, body, requestModel, {
+                        method: req.method || 'POST',
+                        headers,
+                        chain,
+                        baseWallet,
+                        solanaWallet,
+                        timeoutMs: requestTimeoutMs,
+                    });
                 }
                 const responseHeaders = {};
                 response.headers.forEach((v, k) => {
@@ -452,7 +505,7 @@ export function createProxy(options) {
                     const decoder = new TextDecoder();
                     let fullResponse = '';
                     const STREAM_CAP = 5_000_000; // 5MB cap on accumulated stream
-                    const STREAM_TIMEOUT_MS = 5 * 60 * 1000; // 5 min timeout for entire stream
+                    const STREAM_TIMEOUT_MS = getProxyStreamTimeoutMs();
                     const streamDeadline = Date.now() + STREAM_TIMEOUT_MS;
                     const pump = async () => {
                         while (true) {
@@ -563,10 +616,77 @@ export function createProxy(options) {
     });
     return server;
 }
+async function fetchModelAttempt(url, init, body, model, payment) {
+    let response = await fetchWithTimeout(url, { ...init, body: body || undefined }, payment.timeoutMs, `Proxy request for ${model}`);
+    if (response.status !== 402)
+        return response;
+    if (payment.chain === 'solana' && payment.solanaWallet) {
+        return handleSolanaPayment(response, url, payment.method, payment.headers, body, payment.solanaWallet.privateKey, payment.solanaWallet.address, payment.timeoutMs, model);
+    }
+    if (payment.baseWallet) {
+        return handleBasePayment(response, url, payment.method, payment.headers, body, payment.baseWallet.privateKey, payment.baseWallet.address, payment.timeoutMs, model);
+    }
+    return response;
+}
+/**
+ * Try each fallback model as a full x402 attempt:
+ * unpaid 402 probe, payment signing, then the paid provider call. The older
+ * flow only applied fallback to the probe, which meant a slow paid call could
+ * hang Franklin until the outer client gave up.
+ */
+async function fetchWithPaymentFallback(url, init, originalBody, config, payment, onFallback) {
+    const failedModels = [];
+    let attempts = 0;
+    for (let i = 0; i < config.chain.length && attempts < config.maxRetries; i++) {
+        const model = config.chain[i];
+        const body = replaceModelInBody(originalBody, model);
+        try {
+            attempts++;
+            const response = await fetchModelAttempt(url, init, body, model, payment);
+            if (!config.retryOn.includes(response.status)) {
+                return {
+                    response,
+                    modelUsed: model,
+                    bodyUsed: body,
+                    fallbackUsed: i > 0,
+                    attemptsCount: attempts,
+                    failedModels,
+                };
+            }
+            try {
+                await response.body?.cancel();
+            }
+            catch { /* ignore */ }
+            failedModels.push(model);
+            const nextModel = config.chain[i + 1];
+            if (nextModel && onFallback) {
+                onFallback(model, response.status, nextModel);
+            }
+            if (i < config.chain.length - 1) {
+                await sleep(config.retryDelayMs);
+            }
+        }
+        catch (err) {
+            failedModels.push(model);
+            const nextModel = config.chain[i + 1];
+            if (nextModel && onFallback) {
+                onFallback(model, 0, nextModel);
+            }
+            log(`[fallback] ${model} request error: ${err instanceof Error ? err.message : String(err)}`);
+            if (i < config.chain.length - 1) {
+                await sleep(config.retryDelayMs);
+            }
+        }
+    }
+    throw new Error(`All models in fallback chain failed: ${failedModels.join(', ')}`);
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 // ======================================================================
 // Base (EIP-712) payment handler
 // ======================================================================
-async function handleBasePayment(response, url, method, headers, body, privateKey, fromAddress) {
+async function handleBasePayment(response, url, method, headers, body, privateKey, fromAddress, timeoutMs = getProxyRequestTimeoutMs(), model = 'unknown') {
     const paymentHeader = await extractPaymentHeader(response);
     if (!paymentHeader) {
         throw new Error('402 Payment Required — wallet may need funding. Run: franklin balance');
@@ -579,19 +699,19 @@ async function handleBasePayment(response, url, method, headers, body, privateKe
         maxTimeoutSeconds: details.maxTimeoutSeconds || 300,
         extra: details.extra,
     });
-    return fetch(url, {
+    return fetchWithTimeout(url, {
         method,
         headers: {
             ...headers,
             'PAYMENT-SIGNATURE': paymentPayload,
         },
         body: body || undefined,
-    });
+    }, timeoutMs, `Paid proxy request for ${model}`);
 }
 // ======================================================================
 // Solana payment handler
 // ======================================================================
-async function handleSolanaPayment(response, url, method, headers, body, privateKey, fromAddress) {
+async function handleSolanaPayment(response, url, method, headers, body, privateKey, fromAddress, timeoutMs = getProxyRequestTimeoutMs(), model = 'unknown') {
     const paymentHeader = await extractPaymentHeader(response);
     if (!paymentHeader) {
         throw new Error('402 Payment Required — wallet may need funding. Run: franklin balance');
@@ -606,14 +726,14 @@ async function handleSolanaPayment(response, url, method, headers, body, private
         maxTimeoutSeconds: details.maxTimeoutSeconds || 300,
         extra: details.extra,
     });
-    return fetch(url, {
+    return fetchWithTimeout(url, {
         method,
         headers: {
             ...headers,
             'PAYMENT-SIGNATURE': paymentPayload,
         },
         body: body || undefined,
-    });
+    }, timeoutMs, `Paid proxy request for ${model}`);
 }
 export function classifyRequest(body) {
     try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blockrun/franklin",
-  "version": "3.8.36",
+  "version": "3.8.38",
   "description": "Franklin — The AI agent with a wallet. Spends USDC autonomously to get real work done. Pay per action, no subscriptions.",
   "type": "module",
   "exports": {