@princetheprogrammerbtw/husk 0.1.0 → 0.2.0

package/dist/index.d.ts

@@ -525,6 +525,48 @@ declare class OpenAIProvider implements Provider {
     chat(request: ChatRequest): Promise<ChatResponse>;
 }
 
+/**
+ * Husk — Ollama provider adapter.
+ *
+ * Wraps Ollama's OpenAI-compatible Chat Completions API. Because Ollama
+ * exposes the exact same wire format as OpenAI, we can reuse the OpenAI
+ * adapter internally — only the default model name, base URL, and the
+ * provider 'name' field differ.
+ *
+ * Why this exists: local models (llama3.2, deepseek-r1, qwen2.5, etc.)
+ * are a first-class use case. Privacy, cost, and offline-ability all
+ * matter. Ollama is the dominant local-model runtime and uses the
+ * OpenAI API surface, so the adapter is a thin shell.
+ *
+ * Defaults:
+ *   - model: 'llama3.2' (override via constructor)
+ *   - baseURL: 'http://localhost:11434/v1' (override for remote Ollama)
+ *   - apiKey: 'ollama' (Ollama ignores the value but the OpenAI SDK
+ *     requires a non-empty string)
+ *
+ * Usage:
+ *   const agent = new Agent({ model: new OllamaProvider() });
+ *   const result = await agent.run('Explain quantum entanglement');
+ *
+ * For a list of models: `ollama list` (in your terminal).
+ */
+
+interface OllamaProviderOptions {
+    /** Model id (run `ollama list` to see what's pulled locally). Default: 'llama3.2'. */
+    readonly model?: string;
+    /** Ollama server URL. Default: 'http://localhost:11434/v1'. */
+    readonly baseURL?: string;
+    /** API key — Ollama ignores this but the OpenAI SDK requires it. Default: 'ollama'. */
+    readonly apiKey?: string;
+}
+declare class OllamaProvider implements Provider {
+    readonly name = "ollama";
+    readonly model: string;
+    private readonly inner;
+    constructor(options?: OllamaProviderOptions);
+    chat(request: Parameters<Provider['chat']>[0]): ReturnType<Provider['chat']>;
+}
+
 /**
  * Husk — tool registry helpers.
  *
@@ -688,6 +730,232 @@ interface GrepInput {
 }
 declare const Grep: ToolDefinition<GrepInput>;
 
+/**
+ * Husk — eval runner types and API.
+ *
+ * The eval runner lets users assert that an agent's output meets
+ * expectations. Three primitives:
+ *
+ *   1. EvalCase — an input + the expected outcome (an assertion or a set of them)
+ *   2. Assertion — a function that takes the agent's result and returns pass/fail
+ *   3. EvalSuite — a named collection of eval cases, runnable as a unit
+ *
+ * The design choice: assertions are plain async functions, not a DSL.
+ * Users can use the 4 built-ins (equals, contains, matches, fn) or
+ * write their own. The DSL is intentionally tiny — a heavy DSL
+ * (think Jest matchers) is a maintainability trap.
+ *
+ * Example:
+ *
+ *   const suite = defineSuite({
+ *     name: 'hello-agent',
+ *     cases: [
+ *       {
+ *         name: 'answers geography',
+ *         input: 'What is the capital of France? Answer in one word.',
+ *         assertions: [
+ *           contains('Paris'),
+ *           matches(/^[A-Z][a-z]+$/),  // single capitalized word
+ *         ],
+ *       },
+ *     ],
+ *   });
+ *
+ *   const results = await runSuite(suite, () => new Agent({ model: ... }));
+ *   console.log(`${results.passed}/${results.total} passed`);
+ */
+
+/**
+ * A function that checks whether an agent's output meets a criterion.
+ * Returns a pass/fail with an optional message explaining the failure.
+ */
+type Assertion = (result: AgentResult) => AssertionResult | Promise<AssertionResult>;
+interface AssertionResult {
+    /** Whether the assertion passed. */
+    readonly pass: boolean;
+    /** Human-readable name shown in eval reports. */
+    readonly name: string;
+    /** Optional message — required when pass is false to explain why. */
+    readonly message?: string;
+}
+/** Output exactly equals the expected string. */
+declare function equals(expected: string): Assertion;
+/** Output contains the expected substring (case-sensitive). */
+declare function contains(needle: string): Assertion;
+/** Output matches the expected regex. */
+declare function matches(pattern: RegExp): Assertion;
+/** Output passes a custom predicate. Use this for shape-based checks. */
+declare function fn(name: string, predicate: (output: string) => boolean, message?: string): Assertion;
+/** Output does NOT contain the given substring. */
+declare function notContains(needle: string): Assertion;
+/** Output length is within bounds. */
+declare function lengthBetween(min: number, max: number): Assertion;
+interface EvalCase {
+    /** Human-readable name shown in eval reports. */
+    readonly name: string;
+    /** The input to pass to agent.run(). */
+    readonly input: string;
+    /** Assertions to run on the result. All must pass for the case to pass. */
+    readonly assertions: readonly Assertion[];
+    /**
+     * Optional max iterations override. Lets you cap runaway agents per-case
+     * without affecting other cases in the suite.
+     */
+    readonly maxIterations?: number;
+}
+interface EvalSuite {
+    /** Suite name shown in reports. */
+    readonly name: string;
+    /** Cases in this suite, run sequentially. */
+    readonly cases: readonly EvalCase[];
+}
+interface CaseResult {
+    readonly caseName: string;
+    readonly passed: boolean;
+    readonly assertionResults: readonly AssertionResult[];
+    readonly agentResult: AgentResult;
+    readonly durationMs: number;
+}
+interface SuiteResult {
+    readonly suiteName: string;
+    readonly results: readonly CaseResult[];
+    readonly passed: number;
+    readonly total: number;
+    readonly durationMs: number;
+}
+
+/**
+ * Husk — eval runner.
+ *
+ * Takes an EvalSuite + a factory that returns an Agent, runs each
+ * case sequentially, applies the assertions, and reports results.
+ *
+ * Why a factory (not an Agent instance): each case might want its
+ * own agent configuration. The factory pattern gives the user full
+ * control without forcing a specific shape.
+ *
+ * Why sequential (not parallel): LLM calls compete for rate limits
+ * and cost $$$. Sequential gives predictable billing and easier
+ * debugging. Parallel mode is a v0.3.0 addition.
+ *
+ * Failure handling: an agent run that throws an error is reported
+ * as a case failure (not a runner crash). The error message is
+ * included in the assertion results so the user can see what broke.
+ */
+
+/**
+ * A factory that produces a fresh Agent per case. Called once per
+ * case so each case can have isolated memory, config, etc.
+ */
+type AgentFactory = () => Agent | Promise<Agent>;
+interface RunSuiteOptions {
+    /** Stop on first failing case. Default: false (run all cases regardless). */
+    readonly failFast?: boolean;
+    /** Custom logger for runner-level events. Default: silent. */
+    readonly onCaseStart?: (caseName: string) => void;
+    readonly onCaseEnd?: (result: CaseResult) => void;
+}
+declare function runSuite(suite: EvalSuite, factory: AgentFactory, options?: RunSuiteOptions): Promise<SuiteResult>;
+/**
+ * Build a suite with less boilerplate. Equivalent to constructing
+ * the object inline, but reads more clearly at the call site.
+ */
+declare function defineSuite(suite: {
+    name: string;
+    cases: readonly EvalCase[];
+}): EvalSuite;
+
+/**
+ * Husk — observability types (tracer interface).
+ *
+ * A minimal, OTel-inspired tracer interface. Husk's events are mapped
+ * to spans by the mapper in ./tracer.ts. Users can plug in the real
+ * @opentelemetry/api tracer via the adapter (see ./otel-adapter.ts)
+ * or any other compatible backend.
+ *
+ * Design choice: we don't depend on @opentelemetry/api directly. The
+ * interface here is a strict subset of OTel's Span interface (just
+ * what's needed for agent observability). Keeping the dep out of
+ * Husk's core means users who don't need OTel pay nothing for it.
+ *
+ * For users who want full OTel:
+ *   import { trace } from '@opentelemetry/api';
+ *   import { toOtelTracer } from '@princetheprogrammerbtw/husk/otel-adapter';
+ *   agent.onAny(toOtelTracer(trace.getTracer('husk')).onEvent);
+ */
+type SpanKind = 'internal' | 'client' | 'server';
+interface SpanContext {
+    /** Unique trace id (all spans in one agent.run share this). */
+    readonly traceId: string;
+    /** Unique span id. */
+    readonly spanId: string;
+    /** Parent span id, if any. */
+    readonly parentSpanId?: string;
+}
+interface SpanOptions {
+    readonly name: string;
+    readonly kind?: SpanKind;
+    readonly attributes?: Readonly<Record<string, unknown>>;
+    readonly startTimeNs?: bigint;
+}
+interface Span {
+    readonly context: SpanContext;
+    /** Record an event (timestamped annotation) on the span. */
+    addEvent(name: string, attributes?: Record<string, unknown>): void;
+    /** Set or update an attribute on the span. */
+    setAttribute(key: string, value: string | number | boolean | null): void;
+    /** Record an exception. */
+    recordException(err: Error): void;
+    /** Mark the span as failed. */
+    setStatus(status: 'ok' | 'error', message?: string): void;
+    /** End the span. Must be called exactly once. */
+    end(endTimeNs?: bigint): void;
+}
+interface Tracer {
+    /**
+     * Start a new span. If parent is provided, the new span becomes a
+     * child of it. Returns the new span; caller is responsible for
+     * calling .end() on it.
+     */
+    startSpan(options: SpanOptions, parent?: SpanContext): Span;
+}
+/**
+ * A tracer that does nothing. Used when no real tracer is configured.
+ * Zero overhead — every method is a no-op, so the cost is one virtual
+ * call per event.
+ */
+declare class NoopTracer implements Tracer {
+    startSpan(_options: SpanOptions, _parent?: SpanContext): Span;
+}
+
+/**
+ * Husk — agent event → tracer mapper.
+ *
+ * Translates the typed AgentEvent stream into tracer spans. The top-
+ * level 'agent:start' begins a trace, each iteration becomes a child
+ * span, and tool calls become their own spans under the iteration.
+ *
+ * Design: spans are created in startSpanOrder. Tool spans nest under
+ * the iteration span. The end of the agent run ends the trace span.
+ *
+ * Usage:
+ *   const mapper = new EventTracer(myTracer);
+ *   agent.onAny(mapper.onEvent.bind(mapper));
+ *   await agent.run(...);  // emits spans to myTracer
+ */
+
+declare class EventTracer {
+    private readonly tracer;
+    private traceSpan;
+    private iterationSpan;
+    private toolSpans;
+    constructor(tracer: Tracer);
+    /**
+     * Bind as an event handler: `agent.onAny(tracer.onEvent.bind(tracer))`
+     */
+    onEvent: AgentEventHandler;
+}
+
 /**
  * Husk — public API entry point.
  *
@@ -697,6 +965,6 @@ declare const Grep: ToolDefinition<GrepInput>;
  *
  * Re-exports are added incrementally as features land (see commit history).
  */
-declare const VERSION = "0.0.1";
+declare const VERSION = "0.1.0";
 
-export { Agent, type AgentConfig, type AgentEvent, AgentEventEmitter, type AgentEventHandler, type AgentResult, AnthropicProvider, type AnthropicProviderOptions, Bash, type BashInput, type ChatChunk, type ChatRequest, type ChatResponse, ConsoleLogger, type ContentBlock, Edit, type EditInput, type Example, FileStore, type FileStoreOptions, Grep, type GrepInput, InMemoryStore, type JSONSchema, type JSONSchemaField, type LogLevel, type Logger, type MemoryStore, type Message, type MessageContent, OpenAIProvider, type OpenAIProviderOptions, type Provider, Read, type ReadInput, type Role, type SteeringConfig, type StopReason, type TextBlock, type TokenUsage, type ToolContext, type ToolDefinition, type ToolResult, type ToolResultBlock, type ToolUseBlock, VERSION, Write, type WriteInput, arrayField, booleanField, buildExampleMessages, buildSystemPrompt, defineTool, integerField, logEventsTo, numberField, objectField, objectSchema, stringField };
+export { Agent, type AgentConfig, type AgentEvent, AgentEventEmitter, type AgentEventHandler, type AgentFactory, type AgentResult, AnthropicProvider, type AnthropicProviderOptions, type Assertion, type AssertionResult, Bash, type BashInput, type CaseResult, type ChatChunk, type ChatRequest, type ChatResponse, ConsoleLogger, type ContentBlock, Edit, type EditInput, type EvalCase, type EvalSuite, EventTracer, type Example, FileStore, type FileStoreOptions, Grep, type GrepInput, InMemoryStore, type JSONSchema, type JSONSchemaField, type LogLevel, type Logger, type MemoryStore, type Message, type MessageContent, NoopTracer, OllamaProvider, type OllamaProviderOptions, OpenAIProvider, type OpenAIProviderOptions, type Provider, Read, type ReadInput, type Role, type RunSuiteOptions, type Span, type SpanContext, type SpanKind, type SpanOptions, type SteeringConfig, type StopReason, type SuiteResult, type TextBlock, type TokenUsage, type ToolContext, type ToolDefinition, type ToolResult, type ToolResultBlock, type ToolUseBlock, type Tracer, VERSION, Write, type WriteInput, arrayField, booleanField, buildExampleMessages, buildSystemPrompt, contains, defineSuite, defineTool, equals, fn, integerField, lengthBetween, logEventsTo, matches, notContains, numberField, objectField, objectSchema, runSuite, stringField };

package/dist/index.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 import { promises } from 'fs';
 import { join, resolve, dirname } from 'path';
 import Anthropic from '@anthropic-ai/sdk';
@@ -820,6 +821,27 @@ function mapStopReason2(reason) {
   }
 }
 
+// src/providers/ollama.ts
+var DEFAULT_BASE_URL = "http://localhost:11434/v1";
+var DEFAULT_MODEL = "llama3.2";
+var PLACEHOLDER_API_KEY = "ollama";
+var OllamaProvider = class {
+  name = "ollama";
+  model;
+  inner;
+  constructor(options = {}) {
+    this.model = options.model ?? DEFAULT_MODEL;
+    this.inner = new OpenAIProvider({
+      apiKey: options.apiKey ?? PLACEHOLDER_API_KEY,
+      model: this.model,
+      baseURL: options.baseURL ?? DEFAULT_BASE_URL
+    });
+  }
+  chat(request) {
+    return this.inner.chat(request);
+  }
+};
+
 // src/tools/registry.ts
 function defineTool(tool) {
   return {
@@ -1102,9 +1124,293 @@ function truncateOutput(output, limit) {
 ... (${lines.length - limit} more matches truncated)`;
 }
 
+// src/evals/types.ts
+function equals(expected) {
+  return (result) => {
+    const pass = result.output === expected;
+    return pass ? { name: `equals(${JSON.stringify(expected).slice(0, 40)})`, pass: true } : {
+      name: `equals(${JSON.stringify(expected).slice(0, 40)})`,
+      pass: false,
+      message: `Expected ${JSON.stringify(expected)}, got ${JSON.stringify(result.output).slice(0, 200)}`
+    };
+  };
+}
+function contains(needle) {
+  return (result) => {
+    const pass = result.output.includes(needle);
+    return pass ? { name: `contains(${JSON.stringify(needle).slice(0, 40)})`, pass: true } : {
+      name: `contains(${JSON.stringify(needle).slice(0, 40)})`,
+      pass: false,
+      message: `Expected output to contain ${JSON.stringify(needle)}, got ${JSON.stringify(result.output).slice(0, 200)}`
+    };
+  };
+}
+function matches(pattern) {
+  return (result) => {
+    const m = pattern.exec(result.output);
+    return {
+      name: `matches(${pattern})`,
+      pass: m !== null,
+      ...m === null ? {
+        message: `Output did not match ${pattern}: ${JSON.stringify(result.output).slice(0, 200)}`
+      } : {}
+    };
+  };
+}
+function fn(name, predicate, message) {
+  return (result) => {
+    const pass = predicate(result.output);
+    return {
+      name,
+      pass,
+      ...pass ? {} : { message: message ?? `Predicate ${name} failed` }
+    };
+  };
+}
+function notContains(needle) {
+  return (result) => {
+    const pass = !result.output.includes(needle);
+    return pass ? { name: `notContains(${JSON.stringify(needle).slice(0, 40)})`, pass: true } : {
+      name: `notContains(${JSON.stringify(needle).slice(0, 40)})`,
+      pass: false,
+      message: `Output should not contain ${JSON.stringify(needle)} but did: ${JSON.stringify(result.output).slice(0, 200)}`
+    };
+  };
+}
+function lengthBetween(min, max) {
+  return (result) => {
+    const len = result.output.length;
+    const pass = len >= min && len <= max;
+    return pass ? { name: `lengthBetween(${min}, ${max})`, pass: true } : {
+      name: `lengthBetween(${min}, ${max})`,
+      pass: false,
+      message: `Output length ${len} not in [${min}, ${max}]`
+    };
+  };
+}
+
+// src/evals/runner.ts
+async function runSuite(suite, factory, options = {}) {
+  const start = Date.now();
+  const results = [];
+  let passed = 0;
+  for (const c of suite.cases) {
+    options.onCaseStart?.(c.name);
+    const caseResult = await runCase(c, factory);
+    results.push(caseResult);
+    if (caseResult.passed) passed += 1;
+    options.onCaseEnd?.(caseResult);
+    if (options.failFast && !caseResult.passed) {
+      break;
+    }
+  }
+  return {
+    suiteName: suite.name,
+    results,
+    passed,
+    total: suite.cases.length,
+    durationMs: Date.now() - start
+  };
+}
+async function runCase(c, factory) {
+  const start = Date.now();
+  const agent = await factory();
+  let agentResult;
+  try {
+    agentResult = await agent.run(c.input);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const errorAssertionResult = {
+      pass: false,
+      name: "agent.run",
+      message: `agent.run threw: ${message}`
+    };
+    return {
+      caseName: c.name,
+      passed: false,
+      assertionResults: [errorAssertionResult],
+      agentResult: {
+        output: "",
+        messages: [],
+        iterations: 0,
+        usage: { inputTokens: 0, outputTokens: 0 },
+        durationMs: Date.now() - start
+      },
+      durationMs: Date.now() - start
+    };
+  }
+  const assertionResults = [];
+  for (const a of c.assertions) {
+    const r = await a(agentResult);
+    assertionResults.push(r);
+  }
+  const allPassed = assertionResults.every((r) => r.pass);
+  return {
+    caseName: c.name,
+    passed: allPassed,
+    assertionResults,
+    agentResult,
+    durationMs: Date.now() - start
+  };
+}
+function defineSuite(suite) {
+  return {
+    name: suite.name,
+    cases: suite.cases
+  };
+}
+
+// src/obs/tracer.ts
+var NoopTracer = class {
+  startSpan(_options, _parent) {
+    const ctx = {
+      traceId: "0",
+      spanId: "0"
+    };
+    return {
+      context: ctx,
+      addEvent: () => {
+      },
+      setAttribute: () => {
+      },
+      recordException: () => {
+      },
+      setStatus: () => {
+      },
+      end: () => {
+      }
+    };
+  }
+};
+
+// src/obs/mapper.ts
+var EventTracer = class {
+  tracer;
+  traceSpan = null;
+  iterationSpan = null;
+  toolSpans = /* @__PURE__ */ new Map();
+  constructor(tracer) {
+    this.tracer = tracer;
+  }
+  /**
+   * Bind as an event handler: `agent.onAny(tracer.onEvent.bind(tracer))`
+   */
+  onEvent = (event) => {
+    switch (event.type) {
+      case "agent:start": {
+        this.traceSpan = this.tracer.startSpan({
+          name: "agent.run",
+          kind: "internal",
+          attributes: {
+            "husk.input": event.input,
+            "husk.session_id": event.sessionId
+          }
+        });
+        break;
+      }
+      case "agent:iteration": {
+        this.iterationSpan?.end();
+        this.iterationSpan = this.tracer.startSpan(
+          {
+            name: `iteration.${event.iteration}`,
+            kind: "internal",
+            attributes: { "husk.iteration": event.iteration }
+          },
+          this.traceSpan?.context
+        );
+        break;
+      }
+      case "provider:request": {
+        this.iterationSpan?.addEvent("provider.request", {
+          "provider.model": event.request.model
+        });
+        break;
+      }
+      case "provider:response": {
+        if (this.iterationSpan) {
+          this.iterationSpan.setAttribute(
+            "provider.input_tokens",
+            event.response.usage.inputTokens
+          );
+          this.iterationSpan.setAttribute(
+            "provider.output_tokens",
+            event.response.usage.outputTokens
+          );
+          this.iterationSpan.setAttribute("provider.stop_reason", event.response.stopReason);
+          this.iterationSpan.setAttribute("provider.duration_ms", event.durationMs);
+        }
+        break;
+      }
+      case "tool:call": {
+        const span = this.tracer.startSpan(
+          {
+            name: `tool.${event.name}`,
+            kind: "internal",
+            attributes: {
+              "tool.name": event.name,
+              "tool.input": JSON.stringify(event.input)
+            }
+          },
+          this.iterationSpan?.context ?? this.traceSpan?.context
+        );
+        this.toolSpans.set(event.id, span);
+        break;
+      }
+      case "tool:result": {
+        const span = this.toolSpans.get(event.id);
+        if (span) {
+          span.setAttribute("tool.is_error", event.result.isError ?? false);
+          span.setAttribute("tool.duration_ms", event.durationMs);
+          if (event.result.isError) {
+            span.setStatus("error", event.result.output);
+          } else {
+            span.setStatus("ok");
+          }
+          span.end();
+          this.toolSpans.delete(event.id);
+        }
+        break;
+      }
+      case "agent:end": {
+        this.iterationSpan?.end();
+        this.iterationSpan = null;
+        if (this.traceSpan) {
+          this.traceSpan.setAttribute("husk.iterations", event.iterations);
+          this.traceSpan.setAttribute("husk.duration_ms", event.durationMs);
+          this.traceSpan.setStatus("ok");
+          this.traceSpan.end();
+          this.traceSpan = null;
+        }
+        break;
+      }
+      case "agent:error": {
+        if (this.traceSpan) {
+          this.traceSpan.recordException(event.error);
+          this.traceSpan.setStatus("error", event.error.message);
+          this.traceSpan.end();
+          this.traceSpan = null;
+        }
+        this.iterationSpan?.end();
+        this.iterationSpan = null;
+        for (const span of this.toolSpans.values()) {
+          span.end();
+        }
+        this.toolSpans.clear();
+        break;
+      }
+      case "agent:message": {
+        this.iterationSpan?.addEvent("message", {
+          "message.role": event.message.role
+        });
+        break;
+      }
+    }
+  };
+};
+
 // src/index.ts
-var VERSION = "0.0.1";
+var VERSION = "0.1.0";
 
-export { Agent, AgentEventEmitter, AnthropicProvider, Bash, ConsoleLogger, Edit, FileStore, Grep, InMemoryStore, OpenAIProvider, Read, VERSION, Write, arrayField, booleanField, buildExampleMessages, buildSystemPrompt, defineTool, integerField, logEventsTo, numberField, objectField, objectSchema, stringField };
+export { Agent, AgentEventEmitter, AnthropicProvider, Bash, ConsoleLogger, Edit, EventTracer, FileStore, Grep, InMemoryStore, NoopTracer, OllamaProvider, OpenAIProvider, Read, VERSION, Write, arrayField, booleanField, buildExampleMessages, buildSystemPrompt, contains, defineSuite, defineTool, equals, fn, integerField, lengthBetween, logEventsTo, matches, notContains, numberField, objectField, objectSchema, runSuite, stringField };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map