@opengroundplan/core 0.1.1

package/templates/layers/evals/files/evals/runner.py

@@ -0,0 +1,114 @@
+"""Suite orchestration: load, run, assert, trace, gate."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+from . import assertions, budget
+from .budget import BudgetExceeded, RunBudget, Usage
+from .config import EvalConfig
+from .dataset import Case, load_cases
+from .targets import TargetError, build_target
+from .tracing import Tracer
+
+
+@dataclass(frozen=True, slots=True)
+class CaseResult:
+    id: str
+    status: str  # pass | fail | error
+    failures: tuple[str, ...]
+    usage: Usage
+
+
+@dataclass(slots=True)
+class RunReport:
+    results: list[CaseResult] = field(default_factory=list)
+    tokens_spent: int = 0
+    aborted: str | None = None
+
+    @property
+    def passed(self) -> int:
+        return sum(1 for r in self.results if r.status == "pass")
+
+    @property
+    def pass_rate(self) -> float:
+        return 100.0 * self.passed / len(self.results) if self.results else 0.0
+
+    def ok(self, min_pass_rate: float) -> bool:
+        return self.aborted is None and bool(self.results) and self.pass_rate >= min_pass_rate
+
+
+def run_suite(config: EvalConfig) -> RunReport:
+    cases = load_cases(config.dataset_path, config.tags)[: config.max_cases]
+    target = build_target(config)
+    run_budget = RunBudget(config.max_total_tokens)
+    report = RunReport()
+
+    with Tracer(config.trace_path, config.trace_enabled) as tracer:
+        tracer.emit("run.start", target=target.name, cases=len(cases), dataset=config.dataset_path)
+
+        for case in cases:
+            result = _run_case(case, target, config)
+            report.results.append(result)
+            tracer.emit(
+                "case",
+                case=result.id,
+                status=result.status,
+                failures=list(result.failures),
+                inputTokens=result.usage.input_tokens,
+                outputTokens=result.usage.output_tokens,
+                turns=result.usage.turns,
+                durationMs=round(result.usage.duration_ms, 2),
+            )
+
+            try:
+                run_budget.charge(result.usage, case.id)
+            except BudgetExceeded as error:
+                report.aborted = str(error)
+                break
+
+            if config.fail_fast and result.status != "pass":
+                report.aborted = f"fail-fast on {case.id!r}"
+                break
+
+        report.tokens_spent = run_budget.spent
+        tracer.emit(
+            "run.summary",
+            passed=report.passed,
+            total=len(report.results),
+            passRate=round(report.pass_rate, 2),
+            tokensSpent=report.tokens_spent,
+            tokenBudget=run_budget.limit,
+            aborted=report.aborted,
+        )
+
+    return report
+
+
+def _run_case(case: Case, target, config: EvalConfig) -> CaseResult:
+    cap = int(case.expect.get("maxOutputTokens", config.max_output_tokens))
+    started = time.perf_counter()
+    try:
+        response = target.run(case, cap)
+    except TargetError as error:
+        elapsed = (time.perf_counter() - started) * 1000
+        return CaseResult(case.id, "error", (str(error),), Usage(0, 0, 0, elapsed))
+
+    usage = Usage(
+        input_tokens=response.input_tokens,
+        output_tokens=response.output_tokens,
+        turns=response.turns,
+        duration_ms=(time.perf_counter() - started) * 1000,
+    )
+
+    failures = [
+        *assertions.check(case.expect, response.text),
+        *budget.check_case(
+            usage,
+            max_output_tokens=cap,
+            max_turns=config.max_turns,
+            timeout_seconds=config.timeout_seconds,
+        ),
+    ]
+    return CaseResult(case.id, "pass" if not failures else "fail", tuple(failures), usage)

package/templates/layers/evals/files/evals/runner.ts

@@ -0,0 +1,125 @@
+/** Suite orchestration: load, run, assert, trace, gate. */
+
+import { check } from './assertions.js';
+import { BudgetExceeded, checkCase, RunBudget, type Usage } from './budget.js';
+import type { EvalConfig } from './config.js';
+import { type Case, loadCases } from './dataset.js';
+import { buildTarget, type Target, TargetError } from './target.js';
+import { Tracer } from './tracing.js';
+
+export type CaseStatus = 'pass' | 'fail' | 'error';
+
+export interface CaseResult {
+  readonly id: string;
+  readonly status: CaseStatus;
+  readonly failures: readonly string[];
+  readonly usage: Usage;
+}
+
+export interface RunReport {
+  readonly results: readonly CaseResult[];
+  readonly tokensSpent: number;
+  readonly aborted: string | null;
+  readonly passed: number;
+  readonly passRate: number;
+}
+
+export async function runSuite(config: EvalConfig): Promise<RunReport> {
+  const cases = loadCases(config.datasetPath, config.tags).slice(0, config.maxCases);
+  const target = buildTarget(config);
+  const budget = new RunBudget(config.maxTotalTokens);
+  const tracer = new Tracer(config.tracePath, config.traceEnabled);
+  const results: CaseResult[] = [];
+  let aborted: string | null = null;
+
+  try {
+    tracer.emit('run.start', {
+      target: target.name,
+      cases: cases.length,
+      dataset: config.datasetPath,
+    });
+
+    for (const testCase of cases) {
+      const result = await runCase(testCase, target, config);
+      results.push(result);
+      tracer.emit('case', {
+        case: result.id,
+        status: result.status,
+        failures: result.failures,
+        inputTokens: result.usage.inputTokens,
+        outputTokens: result.usage.outputTokens,
+        turns: result.usage.turns,
+        durationMs: Number(result.usage.durationMs.toFixed(2)),
+      });
+
+      try {
+        budget.charge(result.usage, testCase.id);
+      } catch (error) {
+        if (!(error instanceof BudgetExceeded)) throw error;
+        aborted = error.message;
+        break;
+      }
+
+      if (config.failFast && result.status !== 'pass') {
+        aborted = `fail-fast on "${testCase.id}"`;
+        break;
+      }
+    }
+
+    const passed = results.filter((r) => r.status === 'pass').length;
+    const passRate = results.length ? (100 * passed) / results.length : 0;
+    tracer.emit('run.summary', {
+      passed,
+      total: results.length,
+      passRate: Number(passRate.toFixed(2)),
+      tokensSpent: budget.spent,
+      tokenBudget: budget.limit,
+      aborted,
+    });
+
+    return { results, tokensSpent: budget.spent, aborted, passed, passRate };
+  } finally {
+    tracer.close();
+  }
+}
+
+export function isGreen(report: RunReport, minPassRate: number): boolean {
+  return report.aborted === null && report.results.length > 0 && report.passRate >= minPassRate;
+}
+
+async function runCase(testCase: Case, target: Target, config: EvalConfig): Promise<CaseResult> {
+  const cap = testCase.expect.maxOutputTokens ?? config.maxOutputTokens;
+  const started = performance.now();
+
+  let response: Awaited<ReturnType<Target['run']>>;
+  try {
+    response = await target.run(testCase, cap);
+  } catch (error) {
+    if (!(error instanceof TargetError)) throw error;
+    const usage: Usage = {
+      inputTokens: 0,
+      outputTokens: 0,
+      turns: 0,
+      durationMs: performance.now() - started,
+    };
+    return { id: testCase.id, status: 'error', failures: [error.message], usage };
+  }
+
+  const usage: Usage = {
+    inputTokens: response.inputTokens,
+    outputTokens: response.outputTokens,
+    turns: response.turns,
+    durationMs: performance.now() - started,
+  };
+
+  const failures = [
+    ...check(testCase.expect, response.text),
+    ...checkCase(usage, {
+      maxOutputTokens: cap,
+      maxTurns: config.maxTurns,
+      timeoutSeconds: config.timeoutSeconds,
+    }),
+  ];
+
+  return { id: testCase.id, status: failures.length ? 'fail' : 'pass', failures, usage };
+}

package/templates/layers/evals/files/evals/target.ts

@@ -0,0 +1,95 @@
+/**
+ * What the harness calls. The default target is deterministic and offline, so the suite is
+ * green before a single credential exists. Point `EVAL_TARGET=http` at the real service.
+ */
+
+import type { EvalConfig } from './config.js';
+import type { Case } from './dataset.js';
+
+export class TargetError extends Error {}
+
+export interface Response {
+  readonly text: string;
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly turns: number;
+}
+
+export interface Target {
+  readonly name: string;
+  run(testCase: Case, maxOutputTokens: number): Promise<Response>;
+}
+
+/** Whitespace words, not a real tokenizer - budgets only need a stable proxy. */
+export function countTokens(text: string): number {
+  return text.trim() ? text.trim().split(/\s+/).length : 0;
+}
+
+/** Returns the prompt verbatim. Proves the wiring without spending anything. */
+class EchoTarget implements Target {
+  readonly name = 'echo';
+
+  async run(testCase: Case, maxOutputTokens: number): Promise<Response> {
+    const text = testCase.prompt.trim().split(/\s+/).slice(0, maxOutputTokens).join(' ');
+    return {
+      text,
+      inputTokens: countTokens(testCase.prompt),
+      outputTokens: countTokens(text),
+      turns: testCase.turns,
+    };
+  }
+}
+
+/** POSTs a case to the service and reads back text plus reported usage. */
+class HttpTarget implements Target {
+  readonly name = 'http';
+  readonly #url: string;
+  readonly #apiKey: string;
+  readonly #timeoutMs: number;
+
+  constructor(url: string, apiKey: string, timeoutSeconds: number) {
+    this.#url = url;
+    this.#apiKey = apiKey;
+    this.#timeoutMs = timeoutSeconds * 1000;
+  }
+
+  async run(testCase: Case, maxOutputTokens: number): Promise<Response> {
+    const headers: Record<string, string> = { 'content-type': 'application/json' };
+    if (this.#apiKey) headers.authorization = `Bearer ${this.#apiKey}`;
+
+    let payload: Record<string, unknown>;
+    try {
+      const response = await fetch(this.#url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          id: testCase.id,
+          prompt: testCase.prompt,
+          turns: testCase.turns,
+          maxOutputTokens,
+        }),
+        signal: AbortSignal.timeout(this.#timeoutMs),
+      });
+      if (!response.ok) throw new Error(`HTTP ${response.status}`);
+      payload = (await response.json()) as Record<string, unknown>;
+    } catch (error) {
+      throw new TargetError(`${this.#url}: ${(error as Error).message}`);
+    }
+
+    const text = String(payload.text ?? '');
+    return {
+      text,
+      inputTokens: Number(payload.inputTokens ?? countTokens(testCase.prompt)),
+      outputTokens: Number(payload.outputTokens ?? countTokens(text)),
+      turns: Number(payload.turns ?? testCase.turns),
+    };
+  }
+}
+
+export function buildTarget(config: EvalConfig): Target {
+  if (config.target === 'echo') return new EchoTarget();
+  if (config.target === 'http') {
+    return new HttpTarget(config.targetUrl, config.targetApiKey, config.timeoutSeconds);
+  }
+  throw new TargetError(`Unknown target "${config.target}". Use 'echo' or 'http'.`);
+}

package/templates/layers/evals/files/evals/targets.py

@@ -0,0 +1,103 @@
+"""What the harness calls.
+
+The default target is deterministic and offline, so the suite is green before a single
+credential exists. Point ``EVAL_TARGET=http`` at the real service when there is one.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Protocol
+
+from .config import EvalConfig
+from .dataset import Case
+
+
+class TargetError(RuntimeError):
+    """The system under evaluation failed to answer. Distinct from a failed assertion."""
+
+
+@dataclass(frozen=True, slots=True)
+class Response:
+    text: str
+    input_tokens: int
+    output_tokens: int
+    turns: int
+
+
+class Target(Protocol):
+    name: str
+
+    def run(self, case: Case, max_output_tokens: int) -> Response: ...
+
+
+def count_tokens(text: str) -> int:
+    """Whitespace words, not a real tokenizer - budgets only need a stable proxy."""
+    return len(text.split())
+
+
+class EchoTarget:
+    """Returns the prompt verbatim. Proves the wiring without spending anything."""
+
+    name = "echo"
+
+    def run(self, case: Case, max_output_tokens: int) -> Response:
+        words = case.prompt.split()
+        text = " ".join(words[:max_output_tokens])
+        return Response(
+            text=text,
+            input_tokens=count_tokens(case.prompt),
+            output_tokens=count_tokens(text),
+            turns=case.turns,
+        )
+
+
+class HttpTarget:
+    """POSTs a case to the service and reads back text plus reported usage."""
+
+    name = "http"
+
+    def __init__(self, url: str, api_key: str, timeout_seconds: float) -> None:
+        self._url = url
+        self._api_key = api_key
+        self._timeout = timeout_seconds
+
+    def run(self, case: Case, max_output_tokens: int) -> Response:
+        body = json.dumps(
+            {
+                "id": case.id,
+                "prompt": case.prompt,
+                "turns": case.turns,
+                "maxOutputTokens": max_output_tokens,
+            }
+        ).encode("utf-8")
+
+        headers = {"content-type": "application/json"}
+        if self._api_key:
+            headers["authorization"] = f"Bearer {self._api_key}"
+
+        request = urllib.request.Request(self._url, data=body, headers=headers, method="POST")
+        try:
+            with urllib.request.urlopen(request, timeout=self._timeout) as response:
+                payload = json.loads(response.read().decode("utf-8"))
+        except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error:
+            raise TargetError(f"{self._url}: {error}") from error
+
+        text = str(payload.get("text", ""))
+        return Response(
+            text=text,
+            input_tokens=int(payload.get("inputTokens", count_tokens(case.prompt))),
+            output_tokens=int(payload.get("outputTokens", count_tokens(text))),
+            turns=int(payload.get("turns", case.turns)),
+        )
+
+
+def build_target(config: EvalConfig) -> Target:
+    if config.target == EchoTarget.name:
+        return EchoTarget()
+    if config.target == HttpTarget.name:
+        return HttpTarget(config.target_url, config.target_api_key, config.timeout_seconds)
+    raise TargetError(f"Unknown target {config.target!r}. Use 'echo' or 'http'.")

package/templates/layers/evals/files/evals/tracing.py

@@ -0,0 +1,49 @@
+"""Run tracing.
+
+One JSON line per case plus a summary line. A failing run is then diffable against the last
+green one, which is the fastest way to see *what* changed rather than *that* something did.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from types import TracebackType
+from typing import Any, TextIO
+
+
+class Tracer:
+    """Appends to an open file for the duration of one run. Truncates on open."""
+
+    def __init__(self, path: str | Path, enabled: bool) -> None:
+        self._enabled = enabled
+        self._path = Path(path)
+        self._file: TextIO | None = None
+
+    def __enter__(self) -> Tracer:
+        if self._enabled:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            self._file = self._path.open("w", encoding="utf-8")
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def emit(self, kind: str, **fields: Any) -> None:
+        if self._file is None:
+            return
+        record = {"ts": time.time(), "kind": kind, **fields}
+        self._file.write(json.dumps(record, sort_keys=True, default=str) + "\n")
+        self._file.flush()

package/templates/layers/evals/files/evals/tracing.ts

@@ -0,0 +1,31 @@
+/**
+ * Run tracing. One JSON line per case plus a summary line, so a failing run is diffable
+ * against the last green one — which shows *what* changed, not just *that* something did.
+ */
+
+import { closeSync, mkdirSync, openSync, writeSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+export class Tracer {
+  #fd: number | null = null;
+  readonly path: string;
+
+  constructor(path: string, enabled: boolean) {
+    this.path = path;
+    if (!enabled) return;
+    mkdirSync(dirname(path), { recursive: true });
+    // Truncating open: each run replaces the previous trace rather than growing it.
+    this.#fd = openSync(path, 'w');
+  }
+
+  emit(kind: string, fields: Record<string, unknown>): void {
+    if (this.#fd === null) return;
+    writeSync(this.#fd, `${JSON.stringify({ ts: Date.now(), kind, ...fields })}\n`);
+  }
+
+  close(): void {
+    if (this.#fd === null) return;
+    closeSync(this.#fd);
+    this.#fd = null;
+  }
+}

package/templates/layers/evals/files/evals/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "es2023",
+    "module": "nodenext",
+    "moduleResolution": "nodenext",
+    "types": ["node"],
+    "noEmit": true,
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "verbatimModuleSyntax": true,
+    "erasableSyntaxOnly": true,
+    "skipLibCheck": true
+  },
+  "include": ["."]
+}