brownian-code 2026.2.10

package/src/evals/components/EvalApp.tsx

@@ -0,0 +1,206 @@
+import React, { useState, useEffect } from 'react';
+import { Box, Text, useApp } from 'ink';
+import { colors } from '../../theme.js';
+import { EvalProgress } from './EvalProgress.js';
+import { EvalCurrentQuestion } from './EvalCurrentQuestion.js';
+import { EvalStats } from './EvalStats.js';
+import { EvalRecentResults, type EvalResult } from './EvalRecentResults.js';
+
+
+const SHOW_STATS = true;
+
+interface EvalState {
+  status: 'loading' | 'running' | 'complete';
+  total: number;
+  completed: number;
+  correct: number;
+  currentQuestion: string | null;
+  results: EvalResult[];
+  startTime: number;
+  experimentName: string | null;
+  datasetName: string | null;
+}
+
+export interface EvalProgressEvent {
+  type: 'init' | 'question_start' | 'question_end' | 'complete';
+  total?: number;
+  datasetName?: string;
+  question?: string;
+  score?: number;
+  comment?: string;
+  experimentName?: string;
+  averageScore?: number;
+}
+
+interface EvalAppProps {
+  runEvaluation: () => AsyncGenerator<EvalProgressEvent, void, unknown>;
+}
+
+/**
+ * Main Ink component that orchestrates the eval UI
+ */
+export function EvalApp({ runEvaluation }: EvalAppProps) {
+  const { exit } = useApp();
+  const [state, setState] = useState<EvalState>({
+    status: 'loading',
+    total: 0,
+    completed: 0,
+    correct: 0,
+    currentQuestion: null,
+    results: [],
+    startTime: Date.now(),
+    experimentName: null,
+    datasetName: null,
+  });
+
+  useEffect(() => {
+    let cancelled = false;
+
+    async function run() {
+      for await (const event of runEvaluation()) {
+        if (cancelled) break;
+
+        switch (event.type) {
+          case 'init':
+            setState(prev => ({
+              ...prev,
+              status: 'running',
+              total: event.total ?? 0,
+              datasetName: event.datasetName ?? null,
+              startTime: Date.now(),
+            }));
+            break;
+
+          case 'question_start':
+            setState(prev => ({
+              ...prev,
+              currentQuestion: event.question ?? null,
+            }));
+            break;
+
+          case 'question_end':
+            setState(prev => ({
+              ...prev,
+              completed: prev.completed + 1,
+              correct: prev.correct + (event.score === 1 ? 1 : 0),
+              currentQuestion: null,
+              results: [
+                ...prev.results,
+                {
+                  question: event.question ?? '',
+                  score: event.score ?? 0,
+                  comment: event.comment ?? '',
+                },
+              ],
+            }));
+            break;
+
+          case 'complete':
+            setState(prev => ({
+              ...prev,
+              status: 'complete',
+              experimentName: event.experimentName ?? null,
+              currentQuestion: null,
+            }));
+            break;
+        }
+      }
+
+      // Exit after a short delay to let the user see the final summary
+      setTimeout(() => {
+        exit();
+      }, 100);
+    }
+
+    run();
+
+    return () => {
+      cancelled = true;
+    };
+  }, [runEvaluation, exit]);
+
+  // Loading state
+  if (state.status === 'loading') {
+    return (
+      <Box flexDirection="column" marginTop={1}>
+        <Text color={colors.primary} bold>Brownian Eval</Text>
+        <Text color={colors.muted}>Loading dataset...</Text>
+      </Box>
+    );
+  }
+
+  // Complete state - show final summary
+  if (state.status === 'complete') {
+    const avgScore = state.results.length > 0
+      ? state.results.reduce((sum, r) => sum + r.score, 0) / state.results.length
+      : 0;
+
+    return (
+      <Box flexDirection="column" marginTop={1}>
+        <Text>{'═'.repeat(70)}</Text>
+        <Text bold>EVALUATION COMPLETE</Text>
+        <Text>{'═'.repeat(70)}</Text>
+        <Text>Experiment: {state.experimentName ?? 'unknown'}</Text>
+        <Text>Examples evaluated: {state.results.length}</Text>
+        <Text>Average correctness score: <Text color={colors.primary} bold>{(avgScore * 100).toFixed(1)}%</Text></Text>
+        <Text> </Text>
+        <Text>Results by question:</Text>
+        <Text>{'─'.repeat(70)}</Text>
+        {state.results.map((r, i) => {
+          const icon = r.score === 1 ? '✓' : '✗';
+          const iconColor = r.score === 1 ? colors.success : colors.error;
+          return (
+            <Box key={i} flexDirection="column">
+              <Box>
+                <Text color={iconColor}>{icon} </Text>
+                <Text color={colors.muted}>[{r.score}] </Text>
+                <Text>{r.question.slice(0, 65)}{r.question.length > 65 ? '...' : ''}</Text>
+              </Box>
+              {r.comment && r.score !== 1 && (
+                <Text color={colors.muted}>    {r.comment.slice(0, 80)}{r.comment.length > 80 ? '...' : ''}</Text>
+              )}
+            </Box>
+          );
+        })}
+        <Text> </Text>
+        <Text>{'─'.repeat(70)}</Text>
+        <Text color={colors.muted}>View full results: https://smith.langchain.com</Text>
+      </Box>
+    );
+  }
+
+  // Running state - show progress UI
+  return (
+    <Box flexDirection="column" marginTop={1}>
+      {/* Header */}
+      <Box marginBottom={1}>
+        <Text color={colors.primary} bold>Brownian Eval</Text>
+        {state.datasetName && (
+          <Text color={colors.muted}> • {state.datasetName}</Text>
+        )}
+      </Box>
+
+      {/* Progress bar */}
+      <EvalProgress completed={state.completed} total={state.total} />
+
+      {/* Current question with spinner */}
+      <Box marginTop={1}>
+        <EvalCurrentQuestion question={state.currentQuestion} />
+      </Box>
+
+      {/* Live stats */}
+      {SHOW_STATS && (
+        <Box marginTop={1}>
+          <EvalStats
+            correct={state.correct}
+            incorrect={state.completed - state.correct}
+            startTime={state.startTime}
+          />
+        </Box>
+      )}
+
+      {/* Recent results */}
+      <EvalRecentResults results={state.results} maxDisplay={5} />
+    </Box>
+  );
+}

package/src/evals/components/EvalCurrentQuestion.tsx

@@ -0,0 +1,42 @@
+import React from 'react';
+import { Box, Text } from 'ink';
+import Spinner from 'ink-spinner';
+import { colors } from '../../theme.js';
+
+interface EvalCurrentQuestionProps {
+  question: string | null;
+}
+
+/**
+ * Truncate string at word boundary
+ */
+function truncateAtWord(str: string, maxLength: number): string {
+  if (str.length <= maxLength) return str;
+  
+  const lastSpace = str.lastIndexOf(' ', maxLength);
+  if (lastSpace > maxLength * 0.5) {
+    return str.slice(0, lastSpace) + '...';
+  }
+  
+  return str.slice(0, maxLength) + '...';
+}
+
+/**
+ * Shows the question currently being evaluated with a spinner
+ * 
+ * Example: ⠋ How has Netflix's Average Revenue Per Paying User Changed...
+ */
+export function EvalCurrentQuestion({ question }: EvalCurrentQuestionProps) {
+  if (!question) {
+    return null;
+  }
+  
+  return (
+    <Box>
+      <Text color={colors.primary}>
+        <Spinner type="dots" />
+      </Text>
+      <Text> {truncateAtWord(question, 65)}</Text>
+    </Box>
+  );
+}

package/src/evals/components/EvalProgress.tsx

@@ -0,0 +1,33 @@
+import React from 'react';
+import { Box, Text } from 'ink';
+import { colors } from '../../theme.js';
+
+interface EvalProgressProps {
+  completed: number;
+  total: number;
+}
+
+/**
+ * Visual progress bar showing evaluation completion percentage
+ * 
+ * Example: Evaluating ████████░░░░░░░░░░░░ 40% (6/15)
+ */
+export function EvalProgress({ completed, total }: EvalProgressProps) {
+  const percentage = total > 0 ? Math.round((completed / total) * 100) : 0;
+  const barWidth = 20;
+  const filledWidth = Math.round((completed / total) * barWidth) || 0;
+  const emptyWidth = barWidth - filledWidth;
+  
+  const filledBar = '█'.repeat(filledWidth);
+  const emptyBar = '░'.repeat(emptyWidth);
+  
+  return (
+    <Box>
+      <Text color={colors.muted}>Evaluating </Text>
+      <Text color={colors.primary}>{filledBar}</Text>
+      <Text color={colors.mutedDark}>{emptyBar}</Text>
+      <Text color={colors.muted}> {percentage}% </Text>
+      <Text color={colors.muted}>({completed}/{total})</Text>
+    </Box>
+  );
+}

package/src/evals/components/EvalRecentResults.tsx

@@ -0,0 +1,63 @@
+import React from 'react';
+import { Box, Text } from 'ink';
+import { colors } from '../../theme.js';
+
+export interface EvalResult {
+  question: string;
+  score: number;
+  comment: string;
+}
+
+interface EvalRecentResultsProps {
+  results: EvalResult[];
+  maxDisplay?: number;
+}
+
+/**
+ * Truncate string at word boundary
+ */
+function truncateAtWord(str: string, maxLength: number): string {
+  if (str.length <= maxLength) return str;
+  
+  const lastSpace = str.lastIndexOf(' ', maxLength);
+  if (lastSpace > maxLength * 0.5) {
+    return str.slice(0, lastSpace) + '...';
+  }
+  
+  return str.slice(0, maxLength) + '...';
+}
+
+/**
+ * Shows the last N evaluation results with pass/fail indicators
+ * 
+ * Example:
+ * Recent:
+ * ✓ Who is the current CFO of Airbnb?
+ * ✗ Calculate the 3 year revenue CAGR for Palantir...
+ * ✓ What was FND same-store sales growth in Q4 2024?
+ */
+export function EvalRecentResults({ results, maxDisplay = 5 }: EvalRecentResultsProps) {
+  if (results.length === 0) {
+    return null;
+  }
+  
+  const recentResults = results.slice(-maxDisplay);
+  
+  return (
+    <Box flexDirection="column" marginTop={1}>
+      <Text color={colors.muted}>Recent:</Text>
+      {recentResults.map((result, index) => {
+        const isCorrect = result.score === 1;
+        const icon = isCorrect ? '✓' : '✗';
+        const iconColor = isCorrect ? colors.success : colors.error;
+        
+        return (
+          <Box key={index}>
+            <Text color={iconColor}>{icon} </Text>
+            <Text>{truncateAtWord(result.question, 60)}</Text>
+          </Box>
+        );
+      })}
+    </Box>
+  );
+}

package/src/evals/components/EvalStats.tsx

@@ -0,0 +1,49 @@
+import React, { useState, useEffect } from 'react';
+import { Box, Text } from 'ink';
+import { colors } from '../../theme.js';
+
+interface EvalStatsProps {
+  correct: number;
+  incorrect: number;
+  startTime: number;
+}
+
+/**
+ * Format elapsed time as Xm Ys
+ */
+function formatElapsed(startTime: number): string {
+  const elapsed = Math.floor((Date.now() - startTime) / 1000);
+  const minutes = Math.floor(elapsed / 60);
+  const seconds = elapsed % 60;
+  
+  if (minutes > 0) {
+    return `${minutes}m ${seconds}s`;
+  }
+  return `${seconds}s`;
+}
+
+/**
+ * Live stats display showing correct/incorrect counts and elapsed time
+ * 
+ * Example: ✓ 5 correct  ✗ 1 incorrect  ⏱ 2m 34s
+ */
+export function EvalStats({ correct, incorrect, startTime }: EvalStatsProps) {
+  const [, setTick] = useState(0);
+  
+  // Update every second to refresh elapsed time
+  useEffect(() => {
+    const interval = setInterval(() => {
+      setTick(t => t + 1);
+    }, 1000);
+    
+    return () => clearInterval(interval);
+  }, []);
+  
+  return (
+    <Box gap={2}>
+      <Text color={colors.success}>✓ {correct} correct</Text>
+      <Text color={colors.error}>✗ {incorrect} incorrect</Text>
+      <Text color={colors.muted}>⏱ {formatElapsed(startTime)}</Text>
+    </Box>
+  );
+}

package/src/evals/components/index.ts

@@ -0,0 +1,5 @@
+export { EvalApp, type EvalProgressEvent } from './EvalApp.js';
+export { EvalProgress } from './EvalProgress.js';
+export { EvalCurrentQuestion } from './EvalCurrentQuestion.js';
+export { EvalStats } from './EvalStats.js';
+export { EvalRecentResults, type EvalResult } from './EvalRecentResults.js';

package/src/evals/dataset/crypto_agent.csv

@@ -0,0 +1,16 @@
+question,answer
+What is the current price of Bitcoin?,"Bitcoin's current price should be retrieved via the simple_price_browser endpoint with real-time data."
+What is the price of Ethereum?,"Ethereum's current price should be retrieved from market data endpoints."
+Compare BTC vs ETH vs SOL market performance,"A comparison table of BTC, ETH, and SOL including price, market cap, 24h change, and volume."
+Analyze this wallet: 0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045,"Wallet analysis showing token holdings, total value, and recent activity for Vitalik's known wallet."
+What are the top DeFi protocols by TVL?,"A list of top DeFi protocols ranked by Total Value Locked, sourced from DefiLlama data."
+Show me trending tokens,"A list of currently trending tokens with price and volume data."
+What is the TVL of Aave?,"Aave's current Total Value Locked across all chains."
+What is the market cap of Solana?,"Solana's current market capitalization in USD."
+What are the gas prices on Ethereum right now?,"Current Ethereum gas prices (slow, standard, fast) in gwei."
+Show me the top NFT collections by volume,"Top NFT collections ranked by trading volume."
+What is the 24h trading volume of Bitcoin?,"Bitcoin's 24-hour trading volume across exchanges."
+Compare the top 5 cryptocurrencies by market cap,"A table comparing the top 5 cryptos by market cap including price and 24h change."
+What is the circulating supply of Ethereum?,"Ethereum's current circulating supply."
+Show me BNB Chain network stats,"BNB Chain network statistics including block time and transaction count."
+What are the top lending protocols?,"Top lending/borrowing protocols by TVL including Aave, Compound, and others."