@opengroundplan/core 0.1.1

package/templates/layers/evals/files/evals/budget.ts

@@ -0,0 +1,61 @@
+/**
+ * Budget caps. An eval suite with no ceiling is a bill, not a test. Per-case caps fail the
+ * case; the run-wide token cap aborts the suite, because past it every further call is waste.
+ */
+
+export interface Usage {
+  readonly inputTokens: number;
+  readonly outputTokens: number;
+  readonly turns: number;
+  readonly durationMs: number;
+}
+
+export class BudgetExceeded extends Error {}
+
+export function totalTokens(usage: Usage): number {
+  return usage.inputTokens + usage.outputTokens;
+}
+
+export class RunBudget {
+  #spent = 0;
+  readonly limit: number;
+
+  // Plain assignment, not a parameter property: those are not erasable TypeScript syntax.
+  constructor(limit: number) {
+    this.limit = limit;
+  }
+
+  get spent(): number {
+    return this.#spent;
+  }
+
+  charge(usage: Usage, caseId: string): void {
+    this.#spent += totalTokens(usage);
+    if (this.#spent > this.limit) {
+      throw new BudgetExceeded(
+        `run budget exhausted at "${caseId}": ${this.#spent} > ${this.limit} tokens`,
+      );
+    }
+  }
+}
+
+export interface CaseCaps {
+  readonly maxOutputTokens: number;
+  readonly maxTurns: number;
+  readonly timeoutSeconds: number;
+}
+
+export function checkCase(usage: Usage, caps: CaseCaps): string[] {
+  const failures: string[] = [];
+  if (usage.outputTokens > caps.maxOutputTokens) {
+    failures.push(`budget: ${usage.outputTokens} output tokens > ${caps.maxOutputTokens}`);
+  }
+  if (usage.turns > caps.maxTurns) {
+    failures.push(`budget: ${usage.turns} turns > ${caps.maxTurns}`);
+  }
+  const limitMs = caps.timeoutSeconds * 1000;
+  if (usage.durationMs > limitMs) {
+    failures.push(`budget: ${usage.durationMs.toFixed(0)}ms > ${limitMs.toFixed(0)}ms`);
+  }
+  return failures;
+}

package/templates/layers/evals/files/evals/cli.ts

@@ -0,0 +1,76 @@
+/**
+ * CLI entry point. Flags are written into the environment before the config is built, so
+ * `config.ts` stays the only place that knows a default or reads a variable.
+ */
+
+import { parseArgs } from 'node:util';
+import { ENV_PREFIX, loadConfig } from './config.js';
+import { DatasetError } from './dataset.js';
+import { isGreen, runSuite } from './runner.js';
+import { TargetError } from './target.js';
+
+const FLAG_TO_ENV: Record<string, string> = {
+  dataset: 'DATASET',
+  target: 'TARGET',
+  'target-url': 'TARGET_URL',
+  tags: 'TAGS',
+  'max-cases': 'MAX_CASES',
+  'min-pass-rate': 'MIN_PASS_RATE',
+  'trace-path': 'TRACE_PATH',
+};
+
+function applyFlags(): void {
+  const { values } = parseArgs({
+    options: {
+      dataset: { type: 'string' },
+      target: { type: 'string' },
+      'target-url': { type: 'string' },
+      tags: { type: 'string' },
+      'max-cases': { type: 'string' },
+      'min-pass-rate': { type: 'string' },
+      'trace-path': { type: 'string' },
+      'fail-fast': { type: 'boolean' },
+    },
+  });
+
+  const parsed: Record<string, string | boolean | undefined> = values;
+  for (const [flag, env] of Object.entries(FLAG_TO_ENV)) {
+    const value = parsed[flag];
+    if (typeof value === 'string') process.env[ENV_PREFIX + env] = value;
+  }
+  if (parsed['fail-fast']) process.env[`${ENV_PREFIX}FAIL_FAST`] = 'true';
+}
+
+async function main(): Promise<number> {
+  applyFlags();
+  const config = loadConfig();
+
+  try {
+    const report = await runSuite(config);
+
+    for (const result of report.results) {
+      const mark = { pass: 'PASS', fail: 'FAIL', error: 'ERR ' }[result.status];
+      const ms = result.usage.durationMs.toFixed(0);
+      console.log(`${mark}  ${result.id}  (${result.usage.outputTokens} out, ${ms}ms)`);
+      for (const failure of result.failures) console.log(`        ${failure}`);
+    }
+
+    // ASCII only: this line is read in Windows consoles that are not UTF-8.
+    console.log(
+      `\n${report.passed}/${report.results.length} passed ` +
+        `(${report.passRate.toFixed(1)}%, gate ${config.minPassRate.toFixed(1)}%) | ` +
+        `${report.tokensSpent} tokens | trace: ${config.tracePath}`,
+    );
+    if (report.aborted) console.log(`aborted: ${report.aborted}`);
+
+    return isGreen(report, config.minPassRate) ? 0 : 1;
+  } catch (error) {
+    if (error instanceof DatasetError || error instanceof TargetError) {
+      console.error(`eval setup failed: ${error.message}`);
+      return 2;
+    }
+    throw error;
+  }
+}
+
+process.exitCode = await main();

package/templates/layers/evals/files/evals/config.py

@@ -0,0 +1,83 @@
+"""The one config surface for the eval harness: environment first, one typed default table.
+
+No other module under ``evals/`` reads the environment or holds a limit, URL, or path.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+
+ENV_PREFIX = "EVAL_"
+
+_TRUTHY = frozenset({"1", "true", "yes", "on"})
+
+
+def _env(name: str) -> str | None:
+    value = os.getenv(ENV_PREFIX + name)
+    return value.strip() if value and value.strip() else None
+
+
+def _str(name: str, default: str) -> str:
+    return _env(name) or default
+
+
+def _int(name: str, default: int) -> int:
+    raw = _env(name)
+    return default if raw is None else int(raw)
+
+
+def _float(name: str, default: float) -> float:
+    raw = _env(name)
+    return default if raw is None else float(raw)
+
+
+def _bool(name: str, default: bool) -> bool:
+    raw = _env(name)
+    return default if raw is None else raw.lower() in _TRUTHY
+
+
+def _csv(name: str, default: tuple[str, ...]) -> tuple[str, ...]:
+    raw = _env(name)
+    return default if raw is None else tuple(p.strip() for p in raw.split(",") if p.strip())
+
+
+@dataclass(frozen=True, slots=True)
+class EvalConfig:
+    """Every field is overridable by ``EVAL_<FIELD>`` in screaming snake case."""
+
+    dataset_path: str
+    target: str
+    target_url: str
+    target_api_key: str
+    timeout_seconds: float
+    max_output_tokens: int
+    max_total_tokens: int
+    max_turns: int
+    max_cases: int
+    min_pass_rate: float
+    trace_enabled: bool
+    trace_path: str
+    fail_fast: bool
+    tags: tuple[str, ...]
+
+
+@lru_cache(maxsize=1)
+def get_config() -> EvalConfig:
+    return EvalConfig(
+        dataset_path=_str("DATASET", "{{ evalDatasetPath }}"),
+        target=_str("TARGET", "{{ evalTarget }}"),
+        target_url=_str("TARGET_URL", "{{ evalTargetUrl }}"),
+        target_api_key=_str("TARGET_API_KEY", ""),
+        timeout_seconds=_float("TIMEOUT_SECONDS", {{ evalTimeoutSeconds }}),
+        max_output_tokens=_int("MAX_OUTPUT_TOKENS", {{ evalMaxOutputTokens }}),
+        max_total_tokens=_int("MAX_TOTAL_TOKENS", {{ evalMaxTotalTokens }}),
+        max_turns=_int("MAX_TURNS", {{ evalMaxTurns }}),
+        max_cases=_int("MAX_CASES", {{ evalMaxCases }}),
+        min_pass_rate=_float("MIN_PASS_RATE", {{ evalMinPassRate }}),
+        trace_enabled=_bool("TRACE", True),
+        trace_path=_str("TRACE_PATH", "{{ evalTracePath }}"),
+        fail_fast=_bool("FAIL_FAST", False),
+        tags=_csv("TAGS", ()),
+    )

package/templates/layers/evals/files/evals/config.ts

@@ -0,0 +1,77 @@
+/**
+ * The one config surface for the eval harness: environment first, one typed default table.
+ * No other module under `evals/` reads `process.env` or holds a limit, URL, or path.
+ */
+
+const PREFIX = 'EVAL_';
+const TRUTHY = new Set(['1', 'true', 'yes', 'on']);
+
+function raw(name: string): string | undefined {
+  const value = process.env[PREFIX + name]?.trim();
+  return value ? value : undefined;
+}
+
+function str(name: string, fallback: string): string {
+  return raw(name) ?? fallback;
+}
+
+function num(name: string, fallback: number): number {
+  const value = raw(name);
+  if (value === undefined) return fallback;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) throw new Error(`${PREFIX}${name} is not a number: ${value}`);
+  return parsed;
+}
+
+function bool(name: string, fallback: boolean): boolean {
+  const value = raw(name);
+  return value === undefined ? fallback : TRUTHY.has(value.toLowerCase());
+}
+
+function csv(name: string, fallback: readonly string[]): readonly string[] {
+  const value = raw(name);
+  if (value === undefined) return fallback;
+  return value
+    .split(',')
+    .map((part) => part.trim())
+    .filter(Boolean);
+}
+
+export interface EvalConfig {
+  readonly datasetPath: string;
+  readonly target: string;
+  readonly targetUrl: string;
+  readonly targetApiKey: string;
+  readonly timeoutSeconds: number;
+  readonly maxOutputTokens: number;
+  readonly maxTotalTokens: number;
+  readonly maxTurns: number;
+  readonly maxCases: number;
+  readonly minPassRate: number;
+  readonly traceEnabled: boolean;
+  readonly tracePath: string;
+  readonly failFast: boolean;
+  readonly tags: readonly string[];
+}
+
+/** Read once per process; flags in `cli.ts` write the environment before this is called. */
+export function loadConfig(): EvalConfig {
+  return Object.freeze({
+    datasetPath: str('DATASET', '{{ evalDatasetPath }}'),
+    target: str('TARGET', '{{ evalTarget }}'),
+    targetUrl: str('TARGET_URL', '{{ evalTargetUrl }}'),
+    targetApiKey: str('TARGET_API_KEY', ''),
+    timeoutSeconds: num('TIMEOUT_SECONDS', {{ evalTimeoutSeconds }}),
+    maxOutputTokens: num('MAX_OUTPUT_TOKENS', {{ evalMaxOutputTokens }}),
+    maxTotalTokens: num('MAX_TOTAL_TOKENS', {{ evalMaxTotalTokens }}),
+    maxTurns: num('MAX_TURNS', {{ evalMaxTurns }}),
+    maxCases: num('MAX_CASES', {{ evalMaxCases }}),
+    minPassRate: num('MIN_PASS_RATE', {{ evalMinPassRate }}),
+    traceEnabled: bool('TRACE', true),
+    tracePath: str('TRACE_PATH', '{{ evalTracePath }}'),
+    failFast: bool('FAIL_FAST', false),
+    tags: csv('TAGS', []),
+  });
+}
+
+export const ENV_PREFIX = PREFIX;

package/templates/layers/evals/files/evals/dataset.py

@@ -0,0 +1,90 @@
+"""Golden dataset loader.
+
+The dataset is JSONL in git: one case per line, so adding a case is a one-line diff and a
+changed expectation is visible in review rather than buried in a reformatted blob.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+ASSERTION_KEYS = frozenset(
+    ["equals", "contains", "notContains", "regex", "jsonEquals", "maxOutputTokens"],
+)
+
+
+class DatasetError(ValueError):
+    """Malformed dataset. Never tolerated: a silently skipped case is a silent regression."""
+
+
+@dataclass(frozen=True, slots=True)
+class Case:
+    id: str
+    prompt: str
+    turns: int
+    tags: tuple[str, ...]
+    expect: dict[str, Any]
+
+
+def load_cases(path: str | Path, tags: tuple[str, ...] = ()) -> list[Case]:
+    file = Path(path)
+    if not file.is_file():
+        raise DatasetError(f"Dataset not found: {file}")
+
+    seen: set[str] = set()
+    cases: list[Case] = []
+    for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), start=1):
+        text = line.strip()
+        # `//` lets a case be parked with a reason instead of deleted.
+        if not text or text.startswith("//"):
+            continue
+        cases.append(_parse(text, f"{file}:{number}", seen))
+
+    if not cases:
+        raise DatasetError(f"Dataset has no cases: {file}")
+
+    wanted = set(tags)
+    selected = [c for c in cases if not wanted or wanted & set(c.tags)]
+    # Stable order keeps two runs of the same dataset byte-comparable.
+    return sorted(selected, key=lambda c: c.id)
+
+
+def _parse(line: str, where: str, seen: set[str]) -> Case:
+    try:
+        raw = json.loads(line)
+    except json.JSONDecodeError as error:
+        raise DatasetError(f"{where}: not valid JSON - {error}") from error
+
+    if not isinstance(raw, dict):
+        raise DatasetError(f"{where}: expected an object")
+
+    case_id = raw.get("id")
+    if not isinstance(case_id, str) or not case_id:
+        raise DatasetError(f"{where}: missing 'id'")
+    if case_id in seen:
+        raise DatasetError(f"{where}: duplicate id {case_id!r}")
+    seen.add(case_id)
+
+    payload = raw.get("input")
+    if not isinstance(payload, dict) or not isinstance(payload.get("prompt"), str):
+        raise DatasetError(f"{where}: 'input.prompt' must be a string")
+
+    expect = raw.get("expect")
+    if not isinstance(expect, dict) or not expect:
+        raise DatasetError(f"{where}: 'expect' must be a non-empty object")
+
+    # A typo'd assertion key would otherwise make the case pass by asserting nothing.
+    unknown = sorted(set(expect) - ASSERTION_KEYS)
+    if unknown:
+        raise DatasetError(f"{where}: unknown assertion(s) {', '.join(unknown)}")
+
+    return Case(
+        id=case_id,
+        prompt=payload["prompt"],
+        turns=int(payload.get("turns", 1)),
+        tags=tuple(raw.get("tags", ())),
+        expect=expect,
+    )

package/templates/layers/evals/files/evals/dataset.ts

@@ -0,0 +1,101 @@
+/**
+ * Golden dataset loader. The dataset is JSONL in git: one case per line, so adding a case is
+ * a one-line diff and a changed expectation is visible in review.
+ */
+
+import { readFileSync } from 'node:fs';
+
+const ASSERTION_KEYS = new Set([
+  'equals',
+  'contains',
+  'notContains',
+  'regex',
+  'jsonEquals',
+  'maxOutputTokens',
+]);
+
+export class DatasetError extends Error {}
+
+export interface Expectation {
+  readonly equals?: string;
+  readonly contains?: readonly string[];
+  readonly notContains?: readonly string[];
+  readonly regex?: string;
+  readonly jsonEquals?: unknown;
+  readonly maxOutputTokens?: number;
+}
+
+export interface Case {
+  readonly id: string;
+  readonly prompt: string;
+  readonly turns: number;
+  readonly tags: readonly string[];
+  readonly expect: Expectation;
+}
+
+export function loadCases(path: string, tags: readonly string[] = []): Case[] {
+  let text: string;
+  try {
+    text = readFileSync(path, 'utf8');
+  } catch {
+    throw new DatasetError(`Dataset not found: ${path}`);
+  }
+
+  const seen = new Set<string>();
+  const cases: Case[] = [];
+  text.split(/\r?\n/).forEach((line, index) => {
+    const trimmed = line.trim();
+    // `//` lets a case be parked with a reason instead of deleted.
+    if (!trimmed || trimmed.startsWith('//')) return;
+    cases.push(parseCase(trimmed, `${path}:${index + 1}`, seen));
+  });
+
+  if (cases.length === 0) throw new DatasetError(`Dataset has no cases: ${path}`);
+
+  const wanted = new Set(tags);
+  const selected = wanted.size ? cases.filter((c) => c.tags.some((tag) => wanted.has(tag))) : cases;
+  // Stable order keeps two runs of the same dataset byte-comparable.
+  return selected.sort((a, b) => a.id.localeCompare(b.id));
+}
+
+function parseCase(line: string, where: string, seen: Set<string>): Case {
+  let raw: unknown;
+  try {
+    raw = JSON.parse(line);
+  } catch (error) {
+    throw new DatasetError(`${where}: not valid JSON - ${(error as Error).message}`);
+  }
+  if (typeof raw !== 'object' || raw === null) {
+    throw new DatasetError(`${where}: expected an object`);
+  }
+
+  const record = raw as Record<string, unknown>;
+  const id = record.id;
+  if (typeof id !== 'string' || !id) throw new DatasetError(`${where}: missing 'id'`);
+  if (seen.has(id)) throw new DatasetError(`${where}: duplicate id "${id}"`);
+  seen.add(id);
+
+  const input = record.input as Record<string, unknown> | undefined;
+  if (!input || typeof input.prompt !== 'string') {
+    throw new DatasetError(`${where}: 'input.prompt' must be a string`);
+  }
+
+  const expect = record.expect as Record<string, unknown> | undefined;
+  if (!expect || Object.keys(expect).length === 0) {
+    throw new DatasetError(`${where}: 'expect' must be a non-empty object`);
+  }
+
+  // A typo'd assertion key would otherwise make the case pass by asserting nothing.
+  const unknown = Object.keys(expect).filter((key) => !ASSERTION_KEYS.has(key));
+  if (unknown.length) {
+    throw new DatasetError(`${where}: unknown assertion(s) ${unknown.join(', ')}`);
+  }
+
+  return {
+    id,
+    prompt: input.prompt,
+    turns: typeof input.turns === 'number' ? input.turns : 1,
+    tags: Array.isArray(record.tags) ? (record.tags as string[]) : [],
+    expect: expect as Expectation,
+  };
+}

package/templates/layers/evals/files/evals/datasets/case.schema.json

@@ -0,0 +1,67 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://example.invalid/eval-case.schema.json",
+  "title": "Eval case",
+  "description": "One line of a golden dataset JSONL file. Every assertion is deterministic.",
+  "type": "object",
+  "required": ["id", "input", "expect"],
+  "additionalProperties": false,
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$",
+      "description": "Stable, unique. Renaming an id loses the case's history."
+    },
+    "tags": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Selector for partial runs, e.g. only the smoke subset on every push."
+    },
+    "input": {
+      "type": "object",
+      "required": ["prompt"],
+      "additionalProperties": false,
+      "properties": {
+        "prompt": { "type": "string" },
+        "turns": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Turns the target may take. Capped by the run-wide turn budget."
+        }
+      }
+    },
+    "expect": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": false,
+      "properties": {
+        "equals": {
+          "type": "string",
+          "description": "Exact match after collapsing whitespace."
+        },
+        "contains": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "All substrings must be present, case-insensitively."
+        },
+        "notContains": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "None may be present, case-insensitively."
+        },
+        "regex": {
+          "type": "string",
+          "description": "Must find a match somewhere in the output."
+        },
+        "jsonEquals": {
+          "description": "Output parses as JSON and deep-equals this value."
+        },
+        "maxOutputTokens": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Per-case override of the run-wide output-token cap."
+        }
+      }
+    }
+  }
+}

package/templates/layers/evals/files/evals/datasets/golden.jsonl

@@ -0,0 +1,4 @@
+{"id": "contains-and-forbidden", "tags": ["smoke", "content"], "input": {"prompt": "Summarize: Ada Lovelace published the first algorithm in 1843.", "turns": 1}, "expect": {"contains": ["Ada Lovelace", "algorithm"], "notContains": ["I cannot", "as an AI"]}}
+{"id": "regex-iso-date", "tags": ["smoke", "format"], "input": {"prompt": "Order 4711 shipped on 2026-03-01 from the Berlin warehouse.", "turns": 1}, "expect": {"regex": "\\b\\d{4}-\\d{2}-\\d{2}\\b"}}
+{"id": "json-shape", "tags": ["smoke", "structured"], "input": {"prompt": "{\"status\": \"ok\", \"items\": 3}", "turns": 1}, "expect": {"jsonEquals": {"status": "ok", "items": 3}}}
+{"id": "budget-short-answer", "tags": ["smoke", "budget"], "input": {"prompt": "Reply with one short line.", "turns": 1}, "expect": {"contains": ["short"], "maxOutputTokens": 16}}

package/templates/layers/evals/files/evals/run.py

@@ -0,0 +1,85 @@
+"""CLI entry point: ``python -m evals.run``.
+
+Flags are written into the environment before the config is built, so the config module
+stays the only place that knows a default or reads a variable.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+from .config import ENV_PREFIX, get_config
+from .dataset import DatasetError
+from .runner import RunReport, run_suite
+from .targets import TargetError
+
+_FLAG_TO_ENV = {
+    "dataset": "DATASET",
+    "target": "TARGET",
+    "target_url": "TARGET_URL",
+    "tags": "TAGS",
+    "max_cases": "MAX_CASES",
+    "min_pass_rate": "MIN_PASS_RATE",
+    "trace_path": "TRACE_PATH",
+}
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(prog="evals", description="Run the golden eval suite.")
+    parser.add_argument("--dataset", help="Path to the golden JSONL dataset")
+    parser.add_argument("--target", choices=["echo", "http"], help="System under evaluation")
+    parser.add_argument("--target-url", help="Endpoint for the http target")
+    parser.add_argument("--tags", help="Comma-separated tag filter")
+    parser.add_argument("--max-cases", help="Stop after this many cases")
+    parser.add_argument("--min-pass-rate", help="Percent of cases that must pass")
+    parser.add_argument("--trace-path", help="Where the JSONL trace is written")
+    parser.add_argument("--fail-fast", action="store_true", help="Stop at the first failure")
+    return parser.parse_args(argv)
+
+
+def _apply(args: argparse.Namespace) -> None:
+    for flag, env in _FLAG_TO_ENV.items():
+        value = getattr(args, flag)
+        if value is not None:
+            os.environ[ENV_PREFIX + env] = str(value)
+    if args.fail_fast:
+        os.environ[ENV_PREFIX + "FAIL_FAST"] = "true"
+
+
+def _report(report: RunReport, min_pass_rate: float, trace_path: str) -> None:
+    for result in report.results:
+        mark = {"pass": "PASS", "fail": "FAIL", "error": "ERR "}[result.status]
+        ms = f"{result.usage.duration_ms:.0f}ms"
+        print(f"{mark}  {result.id}  ({result.usage.output_tokens} out, {ms})")
+        for failure in result.failures:
+            print(f"        {failure}")
+
+    # ASCII only: this line is read in Windows consoles that are not UTF-8.
+    print(
+        f"\n{report.passed}/{len(report.results)} passed "
+        f"({report.pass_rate:.1f}%, gate {min_pass_rate:.1f}%) | "
+        f"{report.tokens_spent} tokens | trace: {trace_path}"
+    )
+    if report.aborted:
+        print(f"aborted: {report.aborted}")
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse_args(sys.argv[1:] if argv is None else argv)
+    _apply(args)
+
+    config = get_config()
+    try:
+        report = run_suite(config)
+    except (DatasetError, TargetError) as error:
+        print(f"eval setup failed: {error}", file=sys.stderr)
+        return 2
+
+    _report(report, config.min_pass_rate, config.trace_path)
+    return 0 if report.ok(config.min_pass_rate) else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())