brownian-code 2026.2.10

package/src/evals/run.ts

@@ -0,0 +1,355 @@
+/**
+ * LangSmith Evaluation Runner for Brownian Code
+ * 
+ * Usage:
+ *   bun run src/evals/run.ts              # Run on all questions
+ *   bun run src/evals/run.ts --sample 10  # Run on random sample of 10 questions
+ */
+
+import 'dotenv/config';
+import React from 'react';
+import { render } from 'ink';
+import { Client } from 'langsmith';
+import type { EvaluationResult } from 'langsmith/evaluation';
+import { ChatOpenAI } from '@langchain/openai';
+import { z } from 'zod';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { Agent } from '../agent/agent.js';
+import { EvalApp, type EvalProgressEvent } from './components/index.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Types
+interface Example {
+  inputs: { question: string };
+  outputs: { answer: string };
+}
+
+// ============================================================================
+// CSV Parser - handles multi-line quoted fields
+// ============================================================================
+
+function parseCSV(csvContent: string): Example[] {
+  const examples: Example[] = [];
+  const lines = csvContent.split('\n');
+  
+  let i = 1; // Skip header row
+  
+  while (i < lines.length) {
+    const result = parseRow(lines, i);
+    if (result) {
+      const { row, nextIndex } = result;
+      if (row.length >= 2 && row[0].trim()) {
+        examples.push({
+          inputs: { question: row[0] },
+          outputs: { answer: row[1] }
+        });
+      }
+      i = nextIndex;
+    } else {
+      i++;
+    }
+  }
+  
+  return examples;
+}
+
+function parseRow(lines: string[], startIndex: number): { row: string[]; nextIndex: number } | null {
+  if (startIndex >= lines.length || !lines[startIndex].trim()) {
+    return null;
+  }
+  
+  const fields: string[] = [];
+  let currentField = '';
+  let inQuotes = false;
+  let lineIndex = startIndex;
+  let charIndex = 0;
+  
+  while (lineIndex < lines.length) {
+    const line = lines[lineIndex];
+    
+    while (charIndex < line.length) {
+      const char = line[charIndex];
+      const nextChar = line[charIndex + 1];
+      
+      if (inQuotes) {
+        if (char === '"' && nextChar === '"') {
+          // Escaped quote
+          currentField += '"';
+          charIndex += 2;
+        } else if (char === '"') {
+          // End of quoted field
+          inQuotes = false;
+          charIndex++;
+        } else {
+          currentField += char;
+          charIndex++;
+        }
+      } else {
+        if (char === '"') {
+          // Start of quoted field
+          inQuotes = true;
+          charIndex++;
+        } else if (char === ',') {
+          // End of field
+          fields.push(currentField);
+          currentField = '';
+          charIndex++;
+        } else {
+          currentField += char;
+          charIndex++;
+        }
+      }
+    }
+    
+    if (inQuotes) {
+      // Continue to next line (multi-line field)
+      currentField += '\n';
+      lineIndex++;
+      charIndex = 0;
+    } else {
+      // Row complete
+      fields.push(currentField);
+      return { row: fields, nextIndex: lineIndex + 1 };
+    }
+  }
+  
+  // Handle case where file ends while in quotes
+  if (currentField) {
+    fields.push(currentField);
+  }
+  return { row: fields, nextIndex: lineIndex };
+}
+
+// ============================================================================
+// Sampling utilities
+// ============================================================================
+
+function shuffleArray<T>(array: T[]): T[] {
+  const shuffled = [...array];
+  for (let i = shuffled.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+  }
+  return shuffled;
+}
+
+// ============================================================================
+// Target function - wraps Brownian Code agent
+// ============================================================================
+
+async function target(inputs: { question: string }): Promise<{ answer: string }> {
+  const agent = Agent.create({ model: 'claude-sonnet-4-5', maxIterations: 10 });
+  let answer = '';
+  
+  for await (const event of agent.run(inputs.question)) {
+    if (event.type === 'done') {
+      answer = event.answer;
+    }
+  }
+  
+  return { answer };
+}
+
+// ============================================================================
+// Correctness evaluator - LLM-as-judge using gpt-5.2
+// ============================================================================
+
+const EvaluatorOutputSchema = z.object({
+  score: z.number().min(0).max(1),
+  comment: z.string(),
+});
+
+const llm = new ChatOpenAI({
+  model: 'gpt-5.2',
+  apiKey: process.env.OPENAI_API_KEY,
+});
+
+const structuredLlm = llm.withStructuredOutput(EvaluatorOutputSchema);
+
+async function correctnessEvaluator({
+  outputs,
+  referenceOutputs,
+}: {
+  inputs: Record<string, unknown>;
+  outputs: Record<string, unknown>;
+  referenceOutputs?: Record<string, unknown>;
+}): Promise<EvaluationResult> {
+  const actualAnswer = (outputs?.answer as string) || '';
+  const expectedAnswer = (referenceOutputs?.answer as string) || '';
+
+  const prompt = `You are evaluating the correctness of an AI assistant's answer to a crypto research question.
+
+Compare the actual answer to the expected answer. The actual answer is considered correct if it conveys the same key information as the expected answer. Minor differences in wording, formatting, or additional context are acceptable as long as the core facts are correct.
+
+Expected Answer:
+${expectedAnswer}
+
+Actual Answer:
+${actualAnswer}
+
+Evaluate and provide:
+- score: 1 if the answer is correct (contains the key information), 0 if incorrect
+- comment: brief explanation of why the answer is correct or incorrect`;
+
+  try {
+    const result = await structuredLlm.invoke(prompt);
+    return {
+      key: 'correctness',
+      score: result.score,
+      comment: result.comment,
+    };
+  } catch (error) {
+    return {
+      key: 'correctness',
+      score: 0,
+      comment: `Evaluator error: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+// ============================================================================
+// Evaluation generator - yields progress events for the UI
+// ============================================================================
+
+function createEvaluationRunner(sampleSize?: number) {
+  return async function* runEvaluation(): AsyncGenerator<EvalProgressEvent, void, unknown> {
+    // Load and parse dataset
+    const csvPath = path.join(__dirname, 'dataset', 'crypto_agent.csv');
+    const csvContent = fs.readFileSync(csvPath, 'utf-8');
+    let examples = parseCSV(csvContent);
+    const totalCount = examples.length;
+
+    // Apply sampling if requested
+    if (sampleSize && sampleSize < examples.length) {
+      examples = shuffleArray(examples).slice(0, sampleSize);
+    }
+
+    // Create LangSmith client
+    const client = new Client();
+
+    // Create a unique dataset name for this run (sampling creates different datasets)
+    const datasetName = sampleSize
+      ? `brownian-crypto-eval-sample-${sampleSize}-${Date.now()}`
+      : 'brownian-crypto-eval';
+
+    // Yield init event
+    yield {
+      type: 'init',
+      total: examples.length,
+      datasetName: sampleSize ? `crypto_agent (sample ${sampleSize}/${totalCount})` : 'crypto_agent',
+    };
+
+    // Check if dataset exists (only for full runs)
+    let dataset;
+    if (!sampleSize) {
+      try {
+        dataset = await client.readDataset({ datasetName });
+      } catch {
+        // Dataset doesn't exist, will create
+        dataset = null;
+      }
+    }
+
+    // Create dataset if needed
+    if (!dataset) {
+      dataset = await client.createDataset(datasetName, {
+        description: sampleSize
+          ? `Crypto agent evaluation (sample of ${sampleSize})`
+          : 'Crypto agent evaluation dataset',
+      });
+
+      // Upload examples
+      await client.createExamples({
+        datasetId: dataset.id,
+        inputs: examples.map((e) => e.inputs),
+        outputs: examples.map((e) => e.outputs),
+      });
+    }
+
+    // Generate experiment name for tracking
+    const experimentName = `brownian-eval-${Date.now().toString(36)}`;
+
+    // Run evaluation manually - process each example one by one
+    for (const example of examples) {
+      const question = example.inputs.question;
+
+      // Yield question start - UI shows this immediately
+      yield {
+        type: 'question_start',
+        question,
+      };
+
+      // Run the agent to get an answer
+      const startTime = Date.now();
+      const outputs = await target(example.inputs);
+      const endTime = Date.now();
+
+      // Run the correctness evaluator
+      const evalResult = await correctnessEvaluator({
+        inputs: example.inputs,
+        outputs,
+        referenceOutputs: example.outputs,
+      });
+
+      // Log to LangSmith for tracking
+      await client.createRun({
+        name: 'brownian-eval-run',
+        run_type: 'chain',
+        inputs: example.inputs,
+        outputs,
+        start_time: startTime,
+        end_time: endTime,
+        project_name: experimentName,
+        extra: {
+          dataset: datasetName,
+          reference_outputs: example.outputs,
+          evaluation: {
+            score: evalResult.score,
+            comment: evalResult.comment,
+          },
+        },
+      });
+
+      // Yield question end with result - UI updates progress bar
+      yield {
+        type: 'question_end',
+        question,
+        score: typeof evalResult.score === 'number' ? evalResult.score : 0,
+        comment: evalResult.comment || '',
+      };
+    }
+
+    // Yield complete event
+    yield {
+      type: 'complete',
+      experimentName,
+    };
+  };
+}
+
+// ============================================================================
+// Main entry point
+// ============================================================================
+
+async function main() {
+  // Parse CLI arguments
+  const args = process.argv.slice(2);
+  const sampleIndex = args.indexOf('--sample');
+  const sampleSize = sampleIndex !== -1 ? parseInt(args[sampleIndex + 1]) : undefined;
+
+  // Create the evaluation runner with the sample size
+  const runEvaluation = createEvaluationRunner(sampleSize);
+
+  // Render the Ink UI
+  const { waitUntilExit } = render(
+    React.createElement(EvalApp, { runEvaluation })
+  );
+  
+  await waitUntilExit();
+}
+
+main().catch(console.error);

package/src/gateway/channels/whatsapp/auth-store.ts

@@ -0,0 +1,15 @@
+/**
+ * WhatsApp credential persistence.
+ *
+ * Wraps Baileys' useMultiFileAuthState with the ~/.brownian/credentials/ path.
+ */
+import { useMultiFileAuthState } from '@whiskeysockets/baileys';
+import { getCredentialsDir } from '../../config.js';
+
+/**
+ * Load or create auth state from ~/.brownian/credentials/.
+ */
+export async function loadAuthState() {
+  const dir = getCredentialsDir();
+  return useMultiFileAuthState(dir);
+}

package/src/gateway/channels/whatsapp/inbound.ts

@@ -0,0 +1,86 @@
+/**
+ * Incoming WhatsApp message handler.
+ *
+ * Routes incoming messages to the Brownian agent and sends responses back.
+ * Only processes self-messages (messages from the linked account) that don't
+ * start with the [Brownian] prefix.
+ */
+import type { WASocket } from '@whiskeysockets/baileys';
+import { config } from 'dotenv';
+import { Agent } from '../../../agent/agent.js';
+import { sendResponse } from './outbound.js';
+import type { WhatsAppChannelConfig } from '../../config.js';
+
+// Load env so the agent has access to API keys
+config({ quiet: true });
+
+const BROWNIAN_PREFIX = '[Brownian]';
+
+/**
+ * Extract text content from a WhatsApp message.
+ */
+function extractText(message: Record<string, unknown>): string | null {
+  if (typeof message.conversation === 'string') return message.conversation;
+  if (
+    message.extendedTextMessage &&
+    typeof (message.extendedTextMessage as Record<string, unknown>).text === 'string'
+  ) {
+    return (message.extendedTextMessage as Record<string, unknown>).text as string;
+  }
+  return null;
+}
+
+/**
+ * Register the inbound message handler on a WhatsApp socket.
+ */
+export function registerInboundHandler(
+  sock: WASocket,
+  channelConfig: WhatsAppChannelConfig,
+): void {
+  sock.ev.on('messages.upsert', async ({ messages, type }) => {
+    if (type !== 'notify') return;
+
+    for (const msg of messages) {
+      if (!msg.message || !msg.key.remoteJid) continue;
+
+      // Only respond to messages from ourselves (self-message pattern)
+      if (!msg.key.fromMe) continue;
+
+      const text = extractText(msg.message as Record<string, unknown>);
+      if (!text) continue;
+
+      // Skip our own responses
+      if (text.startsWith(BROWNIAN_PREFIX)) continue;
+
+      // Check allowFrom filter
+      if (channelConfig.allowFrom.length > 0) {
+        const sender = msg.key.remoteJid.replace(/@.*/, '');
+        const allowed = channelConfig.allowFrom.some(
+          (num) => sender.includes(num.replace(/\D/g, '')),
+        );
+        if (!allowed) continue;
+      }
+
+      console.log(`Received query: ${text.slice(0, 80)}${text.length > 80 ? '...' : ''}`);
+
+      try {
+        const agent = Agent.create();
+        const events = agent.run(text);
+
+        let answer = '';
+        for await (const event of events) {
+          if (event.type === 'done') {
+            answer = event.answer || 'No answer generated.';
+          }
+        }
+
+        await sendResponse(sock, msg.key.remoteJid, answer);
+        console.log(`Responded (${answer.length} chars).`);
+      } catch (err) {
+        const errMsg = err instanceof Error ? err.message : String(err);
+        console.error(`Agent error: ${errMsg}`);
+        await sendResponse(sock, msg.key.remoteJid, `Error: ${errMsg}`);
+      }
+    }
+  });
+}

package/src/gateway/channels/whatsapp/login.ts

@@ -0,0 +1,28 @@
+#!/usr/bin/env bun
+/**
+ * WhatsApp QR code login flow.
+ *
+ * Run:  bun run gateway:login
+ *
+ * Displays a QR code in the terminal. Scan it with WhatsApp to link.
+ * Credentials are saved to ~/.brownian/credentials/ for subsequent use.
+ * Exits automatically once connected.
+ */
+import { createWhatsAppSession } from './session.js';
+
+console.log('Brownian WhatsApp Gateway — Login');
+console.log('Waiting for QR code...\n');
+
+await createWhatsAppSession({
+  printQR: true,
+  onReady: () => {
+    console.log('\nLogin successful! Credentials saved.');
+    console.log('You can now run:  bun run gateway');
+    // Give Baileys a moment to finish writing credentials
+    setTimeout(() => process.exit(0), 1000);
+  },
+  onLoggedOut: () => {
+    console.error('Login failed or session was logged out.');
+    process.exit(1);
+  },
+});

package/src/gateway/channels/whatsapp/outbound.ts

@@ -0,0 +1,27 @@
+/**
+ * Send a response back to WhatsApp.
+ *
+ * Prefixes all outgoing messages with [Brownian] to distinguish agent replies.
+ */
+import type { WASocket } from '@whiskeysockets/baileys';
+
+const PREFIX = '[Brownian] ';
+const MAX_MESSAGE_LENGTH = 4096;
+
+/**
+ * Send a text message to a WhatsApp JID.
+ * Long messages are truncated with a notice.
+ */
+export async function sendResponse(
+  sock: WASocket,
+  jid: string,
+  text: string,
+): Promise<void> {
+  let body = PREFIX + text;
+
+  if (body.length > MAX_MESSAGE_LENGTH) {
+    body = body.slice(0, MAX_MESSAGE_LENGTH - 30) + '\n\n... (truncated)';
+  }
+
+  await sock.sendMessage(jid, { text: body });
+}

package/src/gateway/channels/whatsapp/session.ts

@@ -0,0 +1,69 @@
+/**
+ * WhatsApp session management via Baileys.
+ *
+ * Handles connection lifecycle, QR auth, reconnection, and credential persistence.
+ */
+import makeWASocket, {
+  DisconnectReason,
+  makeCacheableSignalKeyStore,
+  type WASocket,
+} from '@whiskeysockets/baileys';
+import { Boom } from '@hapi/boom';
+import { loadAuthState } from './auth-store.js';
+
+export type MessageHandler = (sock: WASocket) => void;
+
+interface SessionOptions {
+  /** If true, print QR code in terminal for scanning. */
+  printQR?: boolean;
+  /** Called once connected successfully. Registers event listeners. */
+  onReady?: MessageHandler;
+  /** Called when the session is logged out (not a transient disconnect). */
+  onLoggedOut?: () => void;
+}
+
+/**
+ * Create a WhatsApp socket, handle connection lifecycle, and auto-reconnect.
+ */
+export async function createWhatsAppSession(opts: SessionOptions = {}): Promise<WASocket> {
+  const { printQR = true, onReady, onLoggedOut } = opts;
+  const { state, saveCreds } = await loadAuthState();
+
+  const sock = makeWASocket({
+    auth: {
+      creds: state.creds,
+      keys: makeCacheableSignalKeyStore(state.keys),
+    },
+    printQRInTerminal: printQR,
+  });
+
+  sock.ev.on('connection.update', (update) => {
+    const { connection, lastDisconnect, qr } = update;
+
+    if (qr) {
+      console.log('\nScan this QR code with WhatsApp to link your account.\n');
+    }
+
+    if (connection === 'close') {
+      const statusCode = (lastDisconnect?.error as Boom)?.output?.statusCode;
+      const shouldReconnect = statusCode !== DisconnectReason.loggedOut;
+
+      if (shouldReconnect) {
+        console.log('Connection lost, reconnecting...');
+        createWhatsAppSession(opts);
+      } else {
+        console.log('Logged out from WhatsApp.');
+        onLoggedOut?.();
+      }
+    }
+
+    if (connection === 'open') {
+      console.log('Connected to WhatsApp.');
+      onReady?.(sock);
+    }
+  });
+
+  sock.ev.on('creds.update', saveCreds);
+
+  return sock;
+}

package/src/gateway/config.ts

@@ -0,0 +1,81 @@
+/**
+ * Gateway configuration loader.
+ *
+ * Reads from ~/.brownian/gateway.json if it exists,
+ * otherwise returns sensible defaults.
+ */
+import { readFileSync, existsSync, mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+export interface WhatsAppChannelConfig {
+  enabled: boolean;
+  /** Phone numbers allowed to interact (e.g. ["+1234567890"]). Empty = allow all. */
+  allowFrom: string[];
+}
+
+export interface GatewayConfig {
+  channels: {
+    whatsapp: WhatsAppChannelConfig;
+  };
+}
+
+const BROWNIAN_DIR = join(homedir(), '.brownian');
+const CONFIG_PATH = join(BROWNIAN_DIR, 'gateway.json');
+
+const DEFAULT_CONFIG: GatewayConfig = {
+  channels: {
+    whatsapp: {
+      enabled: true,
+      allowFrom: [],
+    },
+  },
+};
+
+/**
+ * Ensure the ~/.brownian directory exists.
+ */
+export function ensureBrownianDir(): string {
+  if (!existsSync(BROWNIAN_DIR)) {
+    mkdirSync(BROWNIAN_DIR, { recursive: true });
+  }
+  return BROWNIAN_DIR;
+}
+
+/**
+ * Load gateway config from ~/.brownian/gateway.json.
+ * Creates the file with defaults if it doesn't exist.
+ */
+export function loadGatewayConfig(): GatewayConfig {
+  ensureBrownianDir();
+
+  if (!existsSync(CONFIG_PATH)) {
+    writeFileSync(CONFIG_PATH, JSON.stringify(DEFAULT_CONFIG, null, 2) + '\n');
+    return DEFAULT_CONFIG;
+  }
+
+  try {
+    const raw = readFileSync(CONFIG_PATH, 'utf-8');
+    const parsed = JSON.parse(raw) as Partial<GatewayConfig>;
+    return {
+      channels: {
+        whatsapp: {
+          ...DEFAULT_CONFIG.channels.whatsapp,
+          ...parsed.channels?.whatsapp,
+        },
+      },
+    };
+  } catch {
+    console.warn(`Failed to parse ${CONFIG_PATH}, using defaults.`);
+    return DEFAULT_CONFIG;
+  }
+}
+
+/** Path to WhatsApp credential storage. */
+export function getCredentialsDir(): string {
+  const dir = join(ensureBrownianDir(), 'credentials');
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  return dir;
+}