@tuttiai/core 0.8.0 → 0.9.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { TuttiHooks, TelemetryConfig, TuttiEventType, TuttiEvent, ScoreConfig, AgentResult, Session, LLMProvider, SessionStore, AgentConfig, ChatMessage, Voice, Permission, BudgetConfig, ChatRequest, ChatResponse, StreamChunk } from '@tuttiai/types';
+import { TuttiHooks, TokenUsage, ScoreConfig, TelemetryConfig, TuttiEventType, TuttiEvent, AgentResult, Session, LLMProvider, SessionStore, AgentConfig, ChatMessage, Voice, Permission, BudgetConfig, ChatRequest, ChatResponse, StreamChunk } from '@tuttiai/types';
 export { AgentConfig, AgentMemoryConfig, AgentResult, BudgetConfig, ChatMessage, ChatRequest, ChatResponse, ContentBlock, HookContext, LLMProvider, MemoryConfig, Permission, ScoreConfig, Session, SessionStore, StopReason, StreamChunk, TelemetryConfig, TextBlock, TokenUsage, Tool, ToolContext, ToolDefinition, ToolMemoryHelpers, ToolResult, ToolResultBlock, ToolUseBlock, TuttiEvent, TuttiEventHandler, TuttiEventType, TuttiHooks, Voice, VoiceContext } from '@tuttiai/types';
 import pino from 'pino';
 
@@ -84,6 +84,76 @@ declare function createBlocklistHook(blockedTools: string[]): TuttiHooks;
  */
 declare function createMaxCostHook(maxUsd: number): TuttiHooks;
 
+/** Evaluation framework types. */
+
+interface EvalAssertion {
+    type: "contains" | "not_contains" | "matches_regex" | "tool_called" | "tool_not_called" | "turns_lte" | "cost_lte";
+    value: string | number;
+    description?: string;
+}
+interface EvalCase {
+    id: string;
+    name: string;
+    agent_id: string;
+    input: string;
+    assertions: EvalAssertion[];
+}
+interface EvalSuite {
+    name: string;
+    cases: EvalCase[];
+}
+interface AssertionResult {
+    assertion: EvalAssertion;
+    passed: boolean;
+    actual: string | number;
+}
+interface EvalResult {
+    case_id: string;
+    case_name: string;
+    passed: boolean;
+    score: number;
+    output: string;
+    turns: number;
+    usage: TokenUsage;
+    cost_usd: number;
+    duration_ms: number;
+    assertions: AssertionResult[];
+    error?: string;
+}
+interface EvalSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    avg_score: number;
+    total_cost_usd: number;
+    total_duration_ms: number;
+}
+interface EvalReport {
+    suite_name: string;
+    results: EvalResult[];
+    summary: EvalSummary;
+}
+
+/** Evaluation runner — executes test suites against a score. */
+
+declare class EvalRunner {
+    private runtime;
+    constructor(score: ScoreConfig);
+    run(suite: EvalSuite): Promise<EvalReport>;
+    private runCase;
+    private checkAssertion;
+    private summarize;
+}
+
+/** Evaluation report formatting — table, JSON, Markdown. */
+
+/** Print a formatted results table to stdout. */
+declare function printTable(report: EvalReport): void;
+/** Convert report to a plain JSON object for storage or CI. */
+declare function toJSON(report: EvalReport): string;
+/** Convert report to a GitHub-friendly markdown table. */
+declare function toMarkdown(report: EvalReport): string;
+
 declare const createLogger: (name: string) => pino.Logger<never, boolean>;
 declare const logger: pino.Logger<never, boolean>;
 
@@ -336,4 +406,4 @@ declare class GeminiProvider implements LLMProvider {
     stream(request: ChatRequest): AsyncGenerator<StreamChunk>;
 }
 
-export { AgentNotFoundError, AgentRouter, AgentRunner, AnthropicProvider, type AnthropicProviderOptions, AuthenticationError, BudgetExceededError, ContextWindowError, EventBus, GeminiProvider, type GeminiProviderOptions, InMemorySemanticStore, InMemorySessionStore, type MemoryEntry, OpenAIProvider, type OpenAIProviderOptions, PathTraversalError, PermissionError, PermissionGuard, PostgresSessionStore, PromptGuard, ProviderError, RateLimitError, ScoreLoader, ScoreValidationError, SecretsManager, type SemanticMemoryStore, TokenBudget, ToolTimeoutError, TuttiError, TuttiRuntime, TuttiTracer, UrlValidationError, VoiceError, createBlocklistHook, createCacheHook, createLogger, createLoggingHook, createMaxCostHook, defineScore, initTelemetry, logger, shutdownTelemetry, validateScore };
+export { AgentNotFoundError, AgentRouter, AgentRunner, AnthropicProvider, type AnthropicProviderOptions, AuthenticationError, BudgetExceededError, ContextWindowError, type EvalAssertion, type EvalCase, type EvalReport, type EvalResult, EvalRunner, type EvalSuite, type EvalSummary, EventBus, GeminiProvider, type GeminiProviderOptions, InMemorySemanticStore, InMemorySessionStore, type MemoryEntry, OpenAIProvider, type OpenAIProviderOptions, PathTraversalError, PermissionError, PermissionGuard, PostgresSessionStore, PromptGuard, ProviderError, RateLimitError, ScoreLoader, ScoreValidationError, SecretsManager, type SemanticMemoryStore, TokenBudget, ToolTimeoutError, TuttiError, TuttiRuntime, TuttiTracer, UrlValidationError, VoiceError, createBlocklistHook, createCacheHook, createLogger, createLoggingHook, createMaxCostHook, defineScore, toJSON as evalToJSON, toMarkdown as evalToMarkdown, initTelemetry, logger, printTable as printEvalTable, shutdownTelemetry, validateScore };

package/dist/index.js

@@ -122,20 +122,21 @@ Only http:// and https:// URLs to public hosts are allowed.`,
 // src/hooks/index.ts
 function createLoggingHook(log) {
   return {
-    async beforeLLMCall(ctx, request) {
+    beforeLLMCall(ctx, request) {
       log.info({ agent: ctx.agent_name, turn: ctx.turn, model: request.model }, "LLM call");
-      return request;
+      return Promise.resolve(request);
     },
-    async afterLLMCall(ctx, response) {
+    afterLLMCall(ctx, response) {
       log.info({ agent: ctx.agent_name, turn: ctx.turn, usage: response.usage }, "LLM response");
+      return Promise.resolve();
     },
-    async beforeToolCall(ctx, tool, input) {
+    beforeToolCall(ctx, tool, input) {
       log.info({ agent: ctx.agent_name, tool, input }, "Tool call");
-      return input;
+      return Promise.resolve(input);
     },
-    async afterToolCall(ctx, tool, result) {
+    afterToolCall(ctx, tool, result) {
       log.info({ agent: ctx.agent_name, tool, is_error: result.is_error }, "Tool result");
-      return result;
+      return Promise.resolve(result);
     }
   };
 }
@@ -144,150 +145,47 @@ function createCacheHook(store) {
     return tool + ":" + JSON.stringify(input);
   }
   return {
-    async beforeToolCall(_ctx, tool, input) {
+    beforeToolCall(_ctx, tool, input) {
       const cached = store.get(cacheKey(tool, input));
-      if (cached) return cached;
-      return input;
+      if (cached) return Promise.resolve(cached);
+      return Promise.resolve(input);
     },
-    async afterToolCall(_ctx, tool, result) {
+    afterToolCall(_ctx, tool, result) {
       if (!result.is_error) {
         store.set(cacheKey(tool, result.content), result.content);
       }
-      return result;
+      return Promise.resolve(result);
     }
   };
 }
 function createBlocklistHook(blockedTools) {
   const blocked = new Set(blockedTools);
   return {
-    async beforeToolCall(_ctx, tool) {
-      return !blocked.has(tool);
+    beforeToolCall(_ctx, tool) {
+      return Promise.resolve(!blocked.has(tool));
     }
   };
 }
 function createMaxCostHook(maxUsd) {
   let totalCost = 0;
-  const INPUT_PER_M = 3;
-  const OUTPUT_PER_M = 15;
+  const INPUT_PER_M2 = 3;
+  const OUTPUT_PER_M2 = 15;
   return {
-    async afterLLMCall(_ctx, response) {
-      totalCost += response.usage.input_tokens / 1e6 * INPUT_PER_M + response.usage.output_tokens / 1e6 * OUTPUT_PER_M;
+    afterLLMCall(_ctx, response) {
+      totalCost += response.usage.input_tokens / 1e6 * INPUT_PER_M2 + response.usage.output_tokens / 1e6 * OUTPUT_PER_M2;
+      return Promise.resolve();
     },
-    async beforeLLMCall(ctx, request) {
+    beforeLLMCall(ctx, request) {
       if (totalCost >= maxUsd) {
-        throw new Error(
+        return Promise.reject(new Error(
           "Max cost hook: $" + totalCost.toFixed(4) + " exceeds limit $" + maxUsd.toFixed(2) + " for agent " + ctx.agent_name
-        );
+        ));
       }
-      return request;
+      return Promise.resolve(request);
     }
   };
 }
 
-// src/logger.ts
-import pino from "pino";
-var createLogger = (name) => pino({
-  name,
-  level: process.env.TUTTI_LOG_LEVEL ?? "info",
-  transport: process.env.NODE_ENV === "production" ? void 0 : {
-    target: "pino-pretty",
-    options: {
-      colorize: true,
-      translateTime: "HH:MM:ss",
-      ignore: "pid,hostname"
-    }
-  }
-});
-var logger = createLogger("tutti");
-
-// src/telemetry.ts
-import { trace, SpanStatusCode } from "@opentelemetry/api";
-var tracer = trace.getTracer("tutti", "1.0.0");
-var TuttiTracer = {
-  agentRun(agentName, sessionId, fn) {
-    return tracer.startActiveSpan("agent.run", async (span) => {
-      span.setAttribute("agent.name", agentName);
-      span.setAttribute("session.id", sessionId);
-      try {
-        const result = await fn();
-        span.setStatus({ code: SpanStatusCode.OK });
-        return result;
-      } catch (err) {
-        span.setStatus({
-          code: SpanStatusCode.ERROR,
-          message: err instanceof Error ? err.message : String(err)
-        });
-        throw err;
-      } finally {
-        span.end();
-      }
-    });
-  },
-  llmCall(model, fn) {
-    return tracer.startActiveSpan("llm.call", async (span) => {
-      span.setAttribute("llm.model", model);
-      try {
-        const result = await fn();
-        span.setStatus({ code: SpanStatusCode.OK });
-        return result;
-      } catch (err) {
-        span.setStatus({
-          code: SpanStatusCode.ERROR,
-          message: err instanceof Error ? err.message : String(err)
-        });
-        throw err;
-      } finally {
-        span.end();
-      }
-    });
-  },
-  toolCall(toolName, fn) {
-    return tracer.startActiveSpan("tool.call", async (span) => {
-      span.setAttribute("tool.name", toolName);
-      try {
-        const result = await fn();
-        span.setStatus({ code: SpanStatusCode.OK });
-        return result;
-      } catch (err) {
-        span.setStatus({
-          code: SpanStatusCode.ERROR,
-          message: err instanceof Error ? err.message : String(err)
-        });
-        throw err;
-      } finally {
-        span.end();
-      }
-    });
-  }
-};
-
-// src/telemetry-setup.ts
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-var sdk;
-function initTelemetry(config) {
-  if (!config.enabled || sdk) return;
-  const endpoint = config.endpoint ?? "http://localhost:4318";
-  const exporter = new OTLPTraceExporter({
-    url: `${endpoint}/v1/traces`,
-    headers: config.headers
-  });
-  sdk = new NodeSDK({
-    traceExporter: exporter,
-    instrumentations: [getNodeAutoInstrumentations({ "@opentelemetry/instrumentation-fs": { enabled: false } })],
-    serviceName: process.env.OTEL_SERVICE_NAME ?? "tutti"
-  });
-  sdk.start();
-  logger.info({ endpoint }, "OpenTelemetry tracing enabled");
-}
-async function shutdownTelemetry() {
-  if (sdk) {
-    await sdk.shutdown();
-    sdk = void 0;
-  }
-}
-
 // src/agent-runner.ts
 import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
@@ -415,6 +313,83 @@ var TokenBudget = class {
   }
 };
 
+// src/logger.ts
+import pino from "pino";
+var createLogger = (name) => pino({
+  name,
+  level: process.env.TUTTI_LOG_LEVEL ?? "info",
+  transport: process.env.NODE_ENV === "production" ? void 0 : {
+    target: "pino-pretty",
+    options: {
+      colorize: true,
+      translateTime: "HH:MM:ss",
+      ignore: "pid,hostname"
+    }
+  }
+});
+var logger = createLogger("tutti");
+
+// src/telemetry.ts
+import { trace, SpanStatusCode } from "@opentelemetry/api";
+var tracer = trace.getTracer("tutti", "1.0.0");
+var TuttiTracer = {
+  agentRun(agentName, sessionId, fn) {
+    return tracer.startActiveSpan("agent.run", async (span) => {
+      span.setAttribute("agent.name", agentName);
+      span.setAttribute("session.id", sessionId);
+      try {
+        const result = await fn();
+        span.setStatus({ code: SpanStatusCode.OK });
+        return result;
+      } catch (err) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: err instanceof Error ? err.message : String(err)
+        });
+        throw err;
+      } finally {
+        span.end();
+      }
+    });
+  },
+  llmCall(model, fn) {
+    return tracer.startActiveSpan("llm.call", async (span) => {
+      span.setAttribute("llm.model", model);
+      try {
+        const result = await fn();
+        span.setStatus({ code: SpanStatusCode.OK });
+        return result;
+      } catch (err) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: err instanceof Error ? err.message : String(err)
+        });
+        throw err;
+      } finally {
+        span.end();
+      }
+    });
+  },
+  toolCall(toolName, fn) {
+    return tracer.startActiveSpan("tool.call", async (span) => {
+      span.setAttribute("tool.name", toolName);
+      try {
+        const result = await fn();
+        span.setStatus({ code: SpanStatusCode.OK });
+        return result;
+      } catch (err) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: err instanceof Error ? err.message : String(err)
+        });
+        throw err;
+      } finally {
+        span.end();
+      }
+    });
+  }
+};
+
 // src/agent-runner.ts
 var DEFAULT_MAX_TURNS = 10;
 var DEFAULT_MAX_TOOL_CALLS = 20;
@@ -1106,6 +1081,33 @@ var PermissionGuard = class {
   }
 };
 
+// src/telemetry-setup.ts
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+var sdk;
+function initTelemetry(config) {
+  if (!config.enabled || sdk) return;
+  const endpoint = config.endpoint ?? "http://localhost:4318";
+  const exporter = new OTLPTraceExporter({
+    url: `${endpoint}/v1/traces`,
+    headers: config.headers
+  });
+  sdk = new NodeSDK({
+    traceExporter: exporter,
+    instrumentations: [getNodeAutoInstrumentations({ "@opentelemetry/instrumentation-fs": { enabled: false } })],
+    serviceName: process.env.OTEL_SERVICE_NAME ?? "tutti"
+  });
+  sdk.start();
+  logger.info({ endpoint }, "OpenTelemetry tracing enabled");
+}
+async function shutdownTelemetry() {
+  if (sdk) {
+    await sdk.shutdown();
+    sdk = void 0;
+  }
+}
+
 // src/runtime.ts
 var TuttiRuntime = class _TuttiRuntime {
   events;
@@ -1196,6 +1198,209 @@ Supported: "in-memory", "postgres"`,
   }
 };
 
+// src/eval/runner.ts
+var INPUT_PER_M = 3;
+var OUTPUT_PER_M = 15;
+function estimateCost(inputTokens, outputTokens) {
+  return inputTokens / 1e6 * INPUT_PER_M + outputTokens / 1e6 * OUTPUT_PER_M;
+}
+var EvalRunner = class {
+  runtime;
+  constructor(score) {
+    this.runtime = new TuttiRuntime(score);
+  }
+  async run(suite) {
+    const results = [];
+    for (const testCase of suite.cases) {
+      const result = await this.runCase(testCase);
+      results.push(result);
+    }
+    const summary = this.summarize(results);
+    return { suite_name: suite.name, results, summary };
+  }
+  async runCase(testCase) {
+    const toolsCalled = [];
+    const unsubscribeToolStart = this.runtime.events.on("tool:start", (e) => {
+      toolsCalled.push(e.tool_name);
+    });
+    const start = Date.now();
+    let output = "";
+    let turns = 0;
+    let usage = { input_tokens: 0, output_tokens: 0 };
+    let error;
+    try {
+      const result = await this.runtime.run(testCase.agent_id, testCase.input);
+      output = result.output;
+      turns = result.turns;
+      usage = result.usage;
+    } catch (err) {
+      error = err instanceof Error ? err.message : String(err);
+      output = "[error] " + error;
+    }
+    unsubscribeToolStart();
+    const durationMs = Date.now() - start;
+    const costUsd = estimateCost(usage.input_tokens, usage.output_tokens);
+    const assertionResults = testCase.assertions.map(
+      (assertion) => this.checkAssertion(assertion, output, toolsCalled, turns, costUsd)
+    );
+    const passedCount = assertionResults.filter((a) => a.passed).length;
+    const score = testCase.assertions.length > 0 ? passedCount / testCase.assertions.length : error ? 0 : 1;
+    return {
+      case_id: testCase.id,
+      case_name: testCase.name,
+      passed: assertionResults.every((a) => a.passed) && !error,
+      score,
+      output,
+      turns,
+      usage,
+      cost_usd: costUsd,
+      duration_ms: durationMs,
+      assertions: assertionResults,
+      error
+    };
+  }
+  checkAssertion(assertion, output, toolsCalled, turns, costUsd) {
+    const val = assertion.value;
+    switch (assertion.type) {
+      case "contains":
+        return {
+          assertion,
+          passed: output.toLowerCase().includes(String(val).toLowerCase()),
+          actual: output.slice(0, 200)
+        };
+      case "not_contains":
+        return {
+          assertion,
+          passed: !output.toLowerCase().includes(String(val).toLowerCase()),
+          actual: output.slice(0, 200)
+        };
+      case "matches_regex": {
+        const regex = new RegExp(String(val), "i");
+        return {
+          assertion,
+          passed: regex.test(output),
+          actual: output.slice(0, 200)
+        };
+      }
+      case "tool_called":
+        return {
+          assertion,
+          passed: toolsCalled.includes(String(val)),
+          actual: toolsCalled.join(", ") || "(none)"
+        };
+      case "tool_not_called":
+        return {
+          assertion,
+          passed: !toolsCalled.includes(String(val)),
+          actual: toolsCalled.join(", ") || "(none)"
+        };
+      case "turns_lte":
+        return {
+          assertion,
+          passed: turns <= Number(val),
+          actual: turns
+        };
+      case "cost_lte":
+        return {
+          assertion,
+          passed: costUsd <= Number(val),
+          actual: Number(costUsd.toFixed(4))
+        };
+      default:
+        logger.warn({ type: assertion.type }, "Unknown assertion type");
+        return { assertion, passed: false, actual: "unknown assertion type" };
+    }
+  }
+  summarize(results) {
+    const passed = results.filter((r) => r.passed).length;
+    const scores = results.map((r) => r.score);
+    const avgScore = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+    const totalCost = results.reduce((s, r) => s + r.cost_usd, 0);
+    const totalDuration = results.reduce((s, r) => s + r.duration_ms, 0);
+    return {
+      total: results.length,
+      passed,
+      failed: results.length - passed,
+      avg_score: Number(avgScore.toFixed(2)),
+      total_cost_usd: Number(totalCost.toFixed(4)),
+      total_duration_ms: totalDuration
+    };
+  }
+};
+
+// src/eval/report.ts
+function pad(str, len) {
+  return str.length >= len ? str.slice(0, len) : str + " ".repeat(len - str.length);
+}
+function printTable(report) {
+  const { results, summary } = report;
+  console.log();
+  console.log("  Eval suite: " + report.suite_name + " (" + summary.total + " cases)");
+  console.log();
+  for (const r of results) {
+    const icon = r.passed ? "\x1B[32m\u2714\x1B[0m" : "\x1B[31m\u2717\x1B[0m";
+    const score = r.score.toFixed(2);
+    const cost = "$" + r.cost_usd.toFixed(3);
+    const line = "  " + icon + " " + pad(r.case_id, 10) + " " + pad(r.case_name, 28) + " " + pad(score, 6) + " " + r.turns + " turns  " + cost;
+    console.log(line);
+    if (!r.passed) {
+      for (const a of r.assertions) {
+        if (!a.passed) {
+          const desc = a.assertion.description ?? a.assertion.type + ": " + String(a.assertion.value);
+          console.log("    \x1B[31m\u21B3 FAIL: " + desc + " (actual: " + String(a.actual).slice(0, 60) + ")\x1B[0m");
+        }
+      }
+      if (r.error) {
+        console.log("    \x1B[31m\u21B3 ERROR: " + r.error.slice(0, 80) + "\x1B[0m");
+      }
+    }
+  }
+  const pct = summary.total > 0 ? Math.round(summary.passed / summary.total * 100) : 0;
+  console.log();
+  console.log(
+    "  Results: " + summary.passed + "/" + summary.total + " passed (" + pct + "%) | Avg: " + summary.avg_score.toFixed(2) + " | Total: $" + summary.total_cost_usd.toFixed(3)
+  );
+  console.log();
+}
+function toJSON(report) {
+  return JSON.stringify(report, null, 2);
+}
+function toMarkdown(report) {
+  const { results, summary } = report;
+  const lines = [];
+  lines.push("## Eval: " + report.suite_name);
+  lines.push("");
+  lines.push("| Status | ID | Name | Score | Turns | Cost |");
+  lines.push("|--------|-----|------|-------|-------|------|");
+  for (const r of results) {
+    const icon = r.passed ? "pass" : "FAIL";
+    lines.push(
+      "| " + icon + " | " + r.case_id + " | " + r.case_name + " | " + r.score.toFixed(2) + " | " + r.turns + " | $" + r.cost_usd.toFixed(3) + " |"
+    );
+  }
+  lines.push("");
+  const pct = summary.total > 0 ? Math.round(summary.passed / summary.total * 100) : 0;
+  lines.push(
+    "**Results:** " + summary.passed + "/" + summary.total + " passed (" + pct + "%) | Avg score: " + summary.avg_score.toFixed(2) + " | Total cost: $" + summary.total_cost_usd.toFixed(3)
+  );
+  const failed = results.filter((r) => !r.passed);
+  if (failed.length > 0) {
+    lines.push("");
+    lines.push("### Failures");
+    lines.push("");
+    for (const r of failed) {
+      lines.push("**" + r.case_id + "** \u2014 " + r.case_name);
+      for (const a of r.assertions.filter((x) => !x.passed)) {
+        const desc = a.assertion.description ?? a.assertion.type + ": " + String(a.assertion.value);
+        lines.push("- " + desc + " (actual: `" + String(a.actual).slice(0, 80) + "`)");
+      }
+      if (r.error) lines.push("- Error: " + r.error);
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+
 // src/agent-router.ts
 import { z as z2 } from "zod";
 var AgentRouter = class {
@@ -2050,6 +2255,7 @@ export {
   AuthenticationError,
   BudgetExceededError,
   ContextWindowError,
+  EvalRunner,
   EventBus,
   GeminiProvider,
   InMemorySemanticStore,
@@ -2078,8 +2284,11 @@ export {
   createLoggingHook,
   createMaxCostHook,
   defineScore,
+  toJSON as evalToJSON,
+  toMarkdown as evalToMarkdown,
   initTelemetry,
   logger,
+  printTable as printEvalTable,
   shutdownTelemetry,
   validateScore
 };