@opengroundplan/core 0.1.1

package/templates/layers/evals/blueprint.yaml

@@ -0,0 +1,211 @@
+# Golden-dataset eval harness.
+#
+# Paths under `files/` mirror the generated repo: plan.ts strips only the first `files/`
+# segment, so `files/evals/runner.py` lands at `evals/runner.py`. Both stacks share the
+# `evals/` directory and the dataset; only the implementation files are stack-gated.
+id: evals
+kind: layer
+name: LLM eval harness
+description: >-
+  A golden dataset committed to git, deterministic assertions (no model grading another
+  model), enforced token/turn/latency budgets, and a JSONL trace per run. Ships an offline
+  `echo` target so the suite is green on a fresh clone, and an `http` target that evaluates
+  a running service over one JSON endpoint — the harness never enters the app's import graph.
+
+tags:
+  - evals
+  - llm
+  - ai
+  - golden-dataset
+  - regression-testing
+  - tracing
+  - budget
+  - quality-gate
+
+# The natural dependency — "an AI base" — is not expressible: `requires` names blueprint ids,
+# and the only base that provides `llm-runtime` today is python-ai-agent, so naming it would
+# force that base on every stack. Instead the harness is target-agnostic: it talks to the
+# system under evaluation over one HTTP contract and imports nothing from it. That also keeps
+# it usable against a staging deployment, which is where evals actually earn their keep.
+requires: []
+
+# No rival eval harness exists to conflict with, and the layer touches no file another layer
+# owns: everything lands under `evals/` or at `.github/workflows/evals-*.yml`. A future
+# alternative (evals-promptfoo, evals-braintrust) must declare the conflict from its own
+# blueprint — naming an id that does not exist makes the whole catalog fail to load.
+conflicts: []
+
+# Capability names other blueprints key off instead of naming this layer directly.
+# `eval-tracing` rather than `tracing`: observability-otel already provides that, and these
+# are different things — request spans versus a per-run quality record.
+provides:
+  - llm-evals
+  - eval-harness
+  - golden-dataset
+  - eval-runner
+  - eval-budget-caps
+  - eval-tracing
+  - eval-gate
+
+# Quality gating is cross-cutting — it constrains no folder layout.
+supportsArchitectures: []
+
+variables:
+  - name: evalDatasetPath
+    prompt: Path to the golden dataset (JSONL, one case per line, reviewed in git)
+    type: string
+    default: evals/datasets/golden.jsonl
+    required: false
+
+  - name: evalTarget
+    prompt: What the suite evaluates by default
+    type: select
+    default: echo
+    required: false
+    choices:
+      - { value: echo, label: 'echo (offline, deterministic — proves the harness works)' }
+      - { value: http, label: 'http (POST each case to a running service)' }
+
+  - name: evalTargetUrl
+    prompt: >-
+      Endpoint the http target POSTs to — match the port your service listens on (Node bases
+      usually 3000, Python bases usually 8000)
+    type: string
+    default: http://127.0.0.1:8000/evals/run
+    required: false
+
+  - name: evalTimeoutSeconds
+    prompt: Per-case wall-clock cap in seconds
+    type: number
+    default: 30
+    required: false
+
+  - name: evalMaxOutputTokens
+    prompt: Default per-case output-token cap (an individual case may lower it)
+    type: number
+    default: 512
+    required: false
+
+  - name: evalMaxTotalTokens
+    prompt: Run-wide token ceiling — the run aborts the moment it is crossed
+    type: number
+    default: 200000
+    required: false
+
+  - name: evalMaxTurns
+    prompt: Turns a single case may take before it fails
+    type: number
+    default: 4
+    required: false
+
+  - name: evalMaxCases
+    prompt: Hard cap on cases per run, so a dataset that grows overnight cannot blow the budget
+    type: number
+    default: 500
+    required: false
+
+  - name: evalMinPassRate
+    prompt: Percent of cases that must pass for the run to exit 0
+    type: number
+    default: 100
+    required: false
+
+  - name: evalTracePath
+    prompt: Where the per-run JSONL trace is written (disposable output — gitignore it)
+    type: string
+    default: .evals/last-run.jsonl
+    required: false
+
+  # ── CI workflow ─────────────────────────────────────────────────────────────
+  # Deliberately the same names ci-github-actions uses, so selecting both layers asks each
+  # question once instead of twice with two answers that then disagree.
+  - name: ciDefaultBranch
+    prompt: Branch that push builds and pull-request targets are gated on
+    type: string
+    default: main
+    required: false
+
+  - name: ciRunner
+    prompt: Runner the eval job uses
+    type: select
+    default: ubuntu-latest
+    required: false
+    choices:
+      - { value: ubuntu-latest, label: ubuntu-latest }
+      - { value: ubuntu-24.04, label: 'ubuntu-24.04 (pinned)' }
+
+  - name: ciTimeoutMinutes
+    prompt: Per-job timeout in minutes
+    type: number
+    default: 15
+    required: false
+
+  - name: ciNodeVersion
+    prompt: Node version the eval job runs on
+    type: string
+    default: '24'
+    required: false
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+
+  - name: ciPythonVersion
+    prompt: Python version the eval job runs on
+    type: string
+    default: '3.13'
+    required: false
+    when: "stack == 'python'"
+
+files:
+  # ── shared: docs, dataset ───────────────────────────────────────────────────
+  # Gated on the covered stacks so a dotnet or go base is not handed a README for a harness
+  # it never receives.
+  - from: files/docs/evals.md
+    to: .
+    when: "stack == 'python' || stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+  - from: files/evals/README.md
+    to: .
+    when: "stack == 'python' || stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+
+  # Deliberately not `raw`. Raw only skips plan-time token validation; apply.ts still renders
+  # every non-binary file, so a fixture containing a brace pair fails either way — and failing
+  # in `groundplan plan`, before anything is written, is the failure worth having. JSON cannot
+  # produce `{{` on its own; a prompt that must literally contain one escapes it as `\{{`.
+  - from: files/evals/datasets/*
+    to: .
+    when: "stack == 'python' || stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+
+  # ── python ──────────────────────────────────────────────────────────────────
+  # Standard library only, so the harness composes onto any Python base regardless of which
+  # packaging tool or settings library that base chose. `evals/` is an implicit namespace
+  # package (PEP 420): no `__init__.py`, and therefore no `__init__` path token to warn about.
+  - from: files/evals/*.py
+    to: .
+    when: "stack == 'python'"
+  - from: files/.github/workflows/evals-python.yml
+    to: .
+    when: "stack == 'python'"
+
+  # ── typescript / node / polyglot ────────────────────────────────────────────
+  # node: builtins only — no test framework, no HTTP client, no assertion library.
+  - from: files/evals/*.ts
+    to: .
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+  # Its own tsconfig: the bases scope `rootDir`/`include` to `src`, and widening that to pick
+  # up `evals/` would drag the harness into the shipped build output.
+  - from: files/evals/tsconfig.json
+    to: .
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+  - from: files/.github/workflows/evals-node.yml
+    to: .
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+
+postApply:
+  # Manifest-only, like the other layers: nothing installs here, so a pnpm or yarn repo never
+  # gains a stray npm lockfile. docs/evals.md states the install step.
+  - run: npm pkg set scripts.eval="tsx evals/cli.ts" "scripts.eval:typecheck"="tsc -p evals/tsconfig.json"
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+    optional: true
+  - run: npm pkg set devDependencies.tsx=^4.23.1
+    when: "stack == 'typescript' || stack == 'node' || stack == 'polyglot'"
+    optional: true
+
+  # Python needs no dependency and no script entry: `python -m evals.run` works as shipped.

package/templates/layers/evals/files/.github/workflows/evals-node.yml

@@ -0,0 +1,55 @@
+name: Evals
+
+on:
+  push:
+    branches: [{{ ciDefaultBranch }}]
+  pull_request:
+    branches: [{{ ciDefaultBranch }}]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: evals-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+# The only place a version is written in this file.
+env:
+  NODE_VERSION: '{{ ciNodeVersion }}'
+  TSX_VERSION: '4.23.1'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  evals:
+    name: Golden eval suite
+    runs-on: {{ ciRunner }}
+    timeout-minutes: {{ ciTimeoutMinutes }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-node@v7
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      # The harness imports nothing but node: builtins, so `npx tsx` is the whole toolchain —
+      # no install step, and no assumption about which package manager the repo uses. It runs
+      # against the offline `echo` target unless repository variables point it elsewhere;
+      # unset variables arrive empty and fall through to the defaults in evals/config.ts.
+      - name: Run evals
+        env:
+          EVAL_TARGET: ${{ vars.EVAL_TARGET }}
+          EVAL_TARGET_URL: ${{ vars.EVAL_TARGET_URL }}
+          EVAL_TARGET_API_KEY: ${{ secrets.EVAL_TARGET_API_KEY }}
+        run: npx --yes "tsx@$TSX_VERSION" evals/cli.ts
+
+      # Uploaded even on failure: the trace is how you find which case moved.
+      - name: Upload trace
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: eval-trace
+          path: {{ evalTracePath }}
+          if-no-files-found: warn

package/templates/layers/evals/files/.github/workflows/evals-python.yml

@@ -0,0 +1,54 @@
+name: Evals
+
+on:
+  push:
+    branches: [{{ ciDefaultBranch }}]
+  pull_request:
+    branches: [{{ ciDefaultBranch }}]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: evals-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+# The only place a version or command is written in this file.
+env:
+  PYTHON_VERSION: '{{ ciPythonVersion }}'
+  EVAL_COMMAND: python -m evals.run
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  evals:
+    name: Golden eval suite
+    runs-on: {{ ciRunner }}
+    timeout-minutes: {{ ciTimeoutMinutes }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      # The harness is standard-library only, so there is nothing to install. It runs against
+      # the offline `echo` target unless repository variables point it elsewhere; unset
+      # variables arrive empty and fall through to the defaults in evals/config.py.
+      - name: Run evals
+        env:
+          EVAL_TARGET: ${{ vars.EVAL_TARGET }}
+          EVAL_TARGET_URL: ${{ vars.EVAL_TARGET_URL }}
+          EVAL_TARGET_API_KEY: ${{ secrets.EVAL_TARGET_API_KEY }}
+        run: $EVAL_COMMAND
+
+      # Uploaded even on failure: the trace is how you find which case moved.
+      - name: Upload trace
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: eval-trace
+          path: {{ evalTracePath }}
+          if-no-files-found: warn

package/templates/layers/evals/files/docs/evals.md

@@ -0,0 +1,68 @@
+# Evaluating this service
+
+An LLM feature has no compiler. The eval suite is the only thing standing between a prompt
+edit and a silent quality regression, so it is wired the same way as any other test gate:
+it runs on every pull request, it fails the build, and its inputs are reviewed in git.
+
+## Running it
+
+Python:
+
+```bash
+python -m evals.run
+python -m evals.run --tags smoke --fail-fast
+python -m evals.run --target http --target-url http://127.0.0.1:8000/evals/run
+```
+
+Node / TypeScript:
+
+```bash
+npm run eval
+npm run eval -- --tags smoke --fail-fast
+npm run eval -- --target http --target-url http://127.0.0.1:3000/evals/run
+```
+
+Exit codes: `0` green, `1` the pass-rate gate was missed or the run aborted on budget,
+`2` the dataset or target is misconfigured.
+
+## Configuration
+
+Every tunable is an environment variable with a typed default in one place
+(`evals/config.py` or `evals/config.ts`). Nothing else in the harness reads the
+environment, and no limit is written at a call site.
+
+| Variable | Meaning |
+|---|---|
+| `EVAL_DATASET` | Path to the golden JSONL dataset |
+| `EVAL_TARGET` | `echo` (offline, deterministic) or `http` |
+| `EVAL_TARGET_URL` | Endpoint the `http` target POSTs to |
+| `EVAL_TARGET_API_KEY` | Sent as `Authorization: Bearer …` when set |
+| `EVAL_TIMEOUT_SECONDS` | Per-case wall-clock cap |
+| `EVAL_MAX_OUTPUT_TOKENS` | Default per-case output cap; a case may lower it |
+| `EVAL_MAX_TOTAL_TOKENS` | Run-wide ceiling; the run aborts when it is crossed |
+| `EVAL_MAX_TURNS` | Per-case turn cap |
+| `EVAL_MAX_CASES` | Hard limit on cases per run |
+| `EVAL_MIN_PASS_RATE` | Percent of cases that must pass for exit code `0` |
+| `EVAL_TAGS` | Comma-separated tag filter |
+| `EVAL_TRACE` | `false` disables trace output |
+| `EVAL_TRACE_PATH` | Where the JSONL trace is written |
+| `EVAL_FAIL_FAST` | Stop at the first failing case |
+
+The trace directory is disposable output — add it to `.gitignore`.
+
+## Wiring the real target
+
+The `http` target keeps the harness out of your application's import graph: it speaks JSON
+over one endpoint, so the same suite can evaluate a local process, a staging deployment, or
+a container in CI without a code change. Add an endpoint that takes
+`{id, prompt, turns, maxOutputTokens}` and returns `{text, inputTokens, outputTokens, turns}`,
+and forward `maxOutputTokens` to the provider call — otherwise the budget caps only measure
+overruns instead of preventing them.
+
+## Working the suite
+
+1. A bug report becomes a case **before** it becomes a fix. That is the regression test.
+2. Land prompt and model changes with the eval diff in the same pull request. A moved
+   pass-rate with no dataset change is a real signal; both moving at once is not.
+3. Keep the gate at the level you actually hold. A gate nobody believes gets bypassed, and a
+   bypassed gate is worse than none.

package/templates/layers/evals/files/evals/README.md

@@ -0,0 +1,73 @@
+# Golden eval suite
+
+The dataset lives in git. Every assertion is deterministic. Every run is capped.
+
+```
+evals/
+├── datasets/golden.jsonl     the cases — one JSON object per line
+├── datasets/case.schema.json the shape of a line, for editors and review
+├── config.*                  the only place a default, limit, path, or URL lives
+├── dataset.*                 loader + validation (unknown assertion keys are fatal)
+├── assertions.*              equals / contains / notContains / regex / jsonEquals
+├── budget.*                  per-case caps and the run-wide token ceiling
+├── tracing.*                 one JSON line per case, plus a run summary
+├── target.*                  what gets evaluated: `echo` (offline) or `http`
+└── runner.*                  load → run → assert → trace → gate
+```
+
+## Adding a case
+
+Append one line to `datasets/golden.jsonl`:
+
+```json
+{"id": "refund-policy-cited", "tags": ["policy"], "input": {"prompt": "Can I return an opened item?"}, "expect": {"contains": ["14 days"], "notContains": ["I cannot"]}}
+```
+
+Rules that keep the suite trustworthy:
+
+- **Ids are permanent.** Renaming one loses its history in the traces.
+- **Assert on invariants, not on prose.** `contains` a required fact, `regex` a required
+  format, `jsonEquals` a required shape. Never assert on a whole model answer — it will be
+  reworded and the test will fail for the wrong reason.
+- **One behaviour per case.** A case that checks five things reports one bit.
+- **No LLM judge.** A grader that is itself stochastic cannot distinguish a regression from
+  noise. If a property genuinely needs judgement, encode the observable part of it here and
+  review the rest by hand.
+
+Unknown keys under `expect` are fatal rather than ignored, so a typo like `contians` fails
+the run instead of quietly turning the case into an assertion of nothing.
+
+## Budgets
+
+Caps are enforced, not advisory:
+
+| Cap | Scope | Effect when exceeded |
+|---|---|---|
+| `maxOutputTokens` | per case (case value wins over the default) | case fails |
+| `maxTurns` | per case | case fails |
+| `timeoutSeconds` | per case | case fails |
+| `maxTotalTokens` | whole run | run aborts immediately |
+| `maxCases` | whole run | extra cases are not run |
+
+The run-wide ceiling is the one that matters: past it, every further call is money spent on
+a result nobody will read.
+
+## Targets
+
+`echo` returns the prompt verbatim and counts whitespace words as tokens. It costs nothing
+and makes the harness self-verifying, which is why it is the default and why the shipped
+cases pass on a fresh clone.
+
+`http` POSTs each case to the configured endpoint and expects back:
+
+```json
+{"text": "...", "inputTokens": 12, "outputTokens": 40, "turns": 1}
+```
+
+Only `text` is required; the harness counts tokens itself when the others are absent.
+
+## Traces
+
+Each run truncates and rewrites the trace file: one `run.start` line, one `case` line per
+case, one `run.summary` line. Diff a red run against the last green one to see which case
+moved and by how much.

package/templates/layers/evals/files/evals/assertions.py

@@ -0,0 +1,56 @@
+"""Deterministic assertions only.
+
+No model grades another model here. A judge that is itself stochastic cannot tell a
+regression from noise, and a suite that cannot do that is decoration.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Mapping
+from typing import Any
+
+
+def normalize(text: str) -> str:
+    """Collapse whitespace so a reflowed answer is not a diff."""
+    return " ".join(text.split())
+
+
+def check(expect: Mapping[str, Any], output: str) -> list[str]:
+    """Return one message per violated expectation; empty means the case passed."""
+    failures: list[str] = []
+    actual = normalize(output)
+    haystack = actual.lower()
+
+    if "equals" in expect:
+        wanted = normalize(str(expect["equals"]))
+        if actual != wanted:
+            failures.append(f"equals: expected {wanted!r}, got {actual!r}")
+
+    for needle in expect.get("contains", ()):
+        if str(needle).lower() not in haystack:
+            failures.append(f"contains: {needle!r} missing")
+
+    for needle in expect.get("notContains", ()):
+        if str(needle).lower() in haystack:
+            failures.append(f"notContains: {needle!r} present")
+
+    pattern = expect.get("regex")
+    if pattern is not None and re.search(str(pattern), actual) is None:
+        failures.append(f"regex: {pattern!r} did not match")
+
+    if "jsonEquals" in expect:
+        failures.extend(_check_json(expect["jsonEquals"], output))
+
+    return failures
+
+
+def _check_json(wanted: Any, output: str) -> list[str]:
+    try:
+        parsed = json.loads(output)
+    except json.JSONDecodeError as error:
+        return [f"jsonEquals: output is not JSON - {error}"]
+    if parsed != wanted:
+        return [f"jsonEquals: expected {json.dumps(wanted, sort_keys=True)}, got {output.strip()}"]
+    return []

package/templates/layers/evals/files/evals/assertions.ts

@@ -0,0 +1,66 @@
+/**
+ * Deterministic assertions only. No model grades another model here: a judge that is itself
+ * stochastic cannot tell a regression from noise.
+ */
+
+import type { Expectation } from './dataset.js';
+
+/** Collapse whitespace so a reflowed answer is not a diff. */
+export function normalize(text: string): string {
+  return text.trim().split(/\s+/).join(' ');
+}
+
+/** One message per violated expectation; empty means the case passed. */
+export function check(expect: Expectation, output: string): string[] {
+  const failures: string[] = [];
+  const actual = normalize(output);
+  const haystack = actual.toLowerCase();
+
+  if (expect.equals !== undefined) {
+    const wanted = normalize(expect.equals);
+    if (actual !== wanted) failures.push(`equals: expected "${wanted}", got "${actual}"`);
+  }
+
+  for (const needle of expect.contains ?? []) {
+    if (!haystack.includes(needle.toLowerCase())) failures.push(`contains: "${needle}" missing`);
+  }
+
+  for (const needle of expect.notContains ?? []) {
+    if (haystack.includes(needle.toLowerCase())) failures.push(`notContains: "${needle}" present`);
+  }
+
+  if (expect.regex !== undefined && !new RegExp(expect.regex).test(actual)) {
+    failures.push(`regex: /${expect.regex}/ did not match`);
+  }
+
+  if ('jsonEquals' in expect) failures.push(...checkJson(expect.jsonEquals, output));
+
+  return failures;
+}
+
+function checkJson(wanted: unknown, output: string): string[] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(output);
+  } catch (error) {
+    return [`jsonEquals: output is not JSON - ${(error as Error).message}`];
+  }
+  if (!deepEqual(parsed, wanted)) {
+    return [`jsonEquals: expected ${stable(wanted)}, got ${stable(parsed)}`];
+  }
+  return [];
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  return stable(a) === stable(b);
+}
+
+/** Key-order-insensitive serialization - two equal objects must compare equal. */
+function stable(value: unknown): string {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
+  if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
+  const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) =>
+    a.localeCompare(b),
+  );
+  return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stable(v)}`).join(',')}}`;
+}

package/templates/layers/evals/files/evals/budget.py

@@ -0,0 +1,55 @@
+"""Budget caps.
+
+An eval suite with no ceiling is a bill, not a test. Per-case caps fail the case;
+the run-wide token cap aborts the whole run, because past it every further call is waste.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+class BudgetExceeded(RuntimeError):
+    """The run-wide token budget is spent. Raised once, aborts the suite."""
+
+
+@dataclass(frozen=True, slots=True)
+class Usage:
+    input_tokens: int
+    output_tokens: int
+    turns: int
+    duration_ms: float
+
+    @property
+    def total_tokens(self) -> int:
+        return self.input_tokens + self.output_tokens
+
+
+class RunBudget:
+    """Cumulative token spend across the suite."""
+
+    def __init__(self, limit: int) -> None:
+        self.limit = limit
+        self.spent = 0
+
+    def charge(self, usage: Usage, case_id: str) -> None:
+        self.spent += usage.total_tokens
+        if self.spent > self.limit:
+            raise BudgetExceeded(f"token budget spent at {case_id!r}: {self.spent} > {self.limit}")
+
+
+def check_case(
+    usage: Usage,
+    *,
+    max_output_tokens: int,
+    max_turns: int,
+    timeout_seconds: float,
+) -> list[str]:
+    failures: list[str] = []
+    if usage.output_tokens > max_output_tokens:
+        failures.append(f"budget: {usage.output_tokens} output tokens > {max_output_tokens}")
+    if usage.turns > max_turns:
+        failures.append(f"budget: {usage.turns} turns > {max_turns}")
+    if usage.duration_ms > timeout_seconds * 1000:
+        failures.append(f"budget: {usage.duration_ms:.0f}ms > {timeout_seconds * 1000:.0f}ms")
+    return failures