@stackbone/cli 0.2.8 → 0.3.0

package/migrations/0038_evals.sql

@@ -0,0 +1,272 @@
+-- 0038_evals.sql — EVALS in the box (STA-408, PRD STA-393): the six tables that
+-- turn "is the agent any good?" from an opinion into a number, plus the `eval`
+-- run trigger.
+--
+-- Widening a named CHECK REPLACES IT WHOLESALE. `runs_trigger_check` was
+-- re-listed by hand in 0001 (manual/webhook/cron/event), 0016 (+job) and 0017
+-- (+workflow/chat); a migration that only remembers its own value silently
+-- revokes the rest, and nothing conflicts because they are different files. All
+-- eight are re-listed below, and `evals-migration.spec.ts` now pins the complete
+-- set so the next widening breaks loudly.
+ALTER TABLE stackbone_platform.runs DROP CONSTRAINT IF EXISTS runs_trigger_check;
+ALTER TABLE stackbone_platform.runs
+    ADD CONSTRAINT runs_trigger_check
+    CHECK (trigger IN (
+        'manual', 'webhook', 'cron', 'event', 'job', 'workflow', 'chat', 'eval'
+    ));
+
+-- --------------------------------------------------------------------------
+-- The case list: a header row plus one immutable version per edit.
+-- --------------------------------------------------------------------------
+--
+-- Same shape as prompts (0013), and for the same reason: a run has to keep
+-- saying WHICH cases it measured. Editing the list copies its cases into a new
+-- version, so a run from last month still reads its exact cases even after three
+-- edits. The copy costs one INSERT per case; for the expected size — tens or
+-- hundreds of cases per list — that is irrelevant next to losing the ability to
+-- compare two runs at all.
+--
+-- Both header tables carry `deleted_at` and are SOFT-deleted by the store,
+-- because a run points at a dataset version (and a suite) and a sealed run that
+-- lost its subject is not evidence of anything. The uniqueness of the operator's
+-- key therefore has to be PARTIAL — a full UNIQUE would let a deleted list squat
+-- its name forever, so an operator who removes "smoke tests" could never make
+-- another one.
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_datasets (
+    id              uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    -- Operator-facing handle, unique among the ACTIVE lists (index below).
+    key             text        NOT NULL,
+    name            text        NOT NULL,
+    description     text,
+    -- Points at the live row in `eval_dataset_versions` for this list.
+    current_version integer     NOT NULL DEFAULT 1 CHECK (current_version >= 1),
+    created_at      timestamptz NOT NULL DEFAULT now(),
+    updated_at      timestamptz NOT NULL DEFAULT now(),
+    -- The acting user, as the verified JWT / signed actor header reported them.
+    -- NULL on the un-authenticated `stackbone dev` datapath.
+    created_by      text,
+    updated_by      text,
+    -- Soft delete. Reads filter on it; the row survives so the runs that
+    -- measured against it keep their subject.
+    deleted_at      timestamptz
+);
+
+-- Unique among ACTIVE lists only: see the note above on squatting.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_datasets_active_key_idx
+    ON stackbone_platform.eval_datasets (key)
+    WHERE deleted_at IS NULL;
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_dataset_versions (
+    id         uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    dataset_id uuid        NOT NULL
+               REFERENCES stackbone_platform.eval_datasets (id) ON DELETE CASCADE,
+    version    integer     NOT NULL CHECK (version >= 1),
+    -- Free text an operator can leave describing what this edit changed.
+    note       text,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    created_by text
+);
+
+-- The race backstop. The store assigns MAX(version) + 1 under a per-dataset
+-- advisory lock inside the same transaction, exactly like prompts; this index is
+-- what stops two writers both landing version 1 if that lock is ever lost.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_dataset_versions_version_idx
+    ON stackbone_platform.eval_dataset_versions (dataset_id, version);
+
+-- Drives the version-history list (newest first).
+CREATE INDEX IF NOT EXISTS eval_dataset_versions_version_desc_idx
+    ON stackbone_platform.eval_dataset_versions (dataset_id, version DESC);
+
+-- A case belongs to ONE version. Editing the list copies the cases forward.
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_cases (
+    id                 uuid    PRIMARY KEY DEFAULT gen_random_uuid(),
+    dataset_version_id uuid    NOT NULL
+                       REFERENCES stackbone_platform.eval_dataset_versions (id) ON DELETE CASCADE,
+    name               text    NOT NULL,
+    input              jsonb   NOT NULL,
+    -- Optional on purpose: most cases are judged by a rubric, and demanding a
+    -- reference answer would force the operator to invent one.
+    expected           jsonb,
+    tags               text[]  NOT NULL DEFAULT '{}',
+    -- Which run this case was captured from, if it was captured. Deliberately a
+    -- BARE uuid with NO foreign key: a captured case has to outlive run
+    -- retention, and a case that vanished when its source run was pruned would
+    -- silently shrink the list an operator built. Do not "fix" this into an FK.
+    source_run_id      uuid,
+    -- Explicit order within the version, so the list reads the way it was built.
+    position           integer NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS eval_cases_version_position_idx
+    ON stackbone_platform.eval_cases (dataset_version_id, position);
+
+-- --------------------------------------------------------------------------
+-- The suite, its runs, and the per-case verdicts.
+-- --------------------------------------------------------------------------
+--
+-- Two conventions inherited from guardrails (0034), and they are the reason this
+-- table looks under-constrained:
+--
+--   1. The DISCRIMINATOR columns — `target_kind`, `status` — carry NO value
+--      CHECK. A box running an older build has to be able to STORE a kind or a
+--      state it cannot execute rather than reject the write and lose the
+--      operator's configuration. An unknown value fails CLOSED at evaluation
+--      time, which is the safe place for that decision.
+--   2. No PAYLOAD default lives here. The documented defaults live in
+--      `@stackbone/validators` → `studio-evals.ts`, so the runtime, the facade
+--      and the Studio form all read one source of truth. The defaults below are
+--      structural only: identity, timestamps, the progress counters whose
+--      absence would make a half-written run row unreadable, and `repeats`,
+--      whose 1 is the neutral value (run each case once) rather than a
+--      configuration choice.
+--
+-- `scorers` and `variants` are free-form jsonb validated against the CATALOG at
+-- the handler, never by the DDL. A CHECK here would be a second description of
+-- each scorer that drifts from the one the screen renders its form from. Neither
+-- gets an empty-array default: a suite that measures nothing, and a run with no
+-- variant, are not rows worth making writable.
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_suites (
+    id          uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    -- Operator-facing name, unique among the ACTIVE suites (index below), and
+    -- what `stackbone eval <suite>` is given on the command line.
+    name        text        NOT NULL,
+    description text,
+    -- `agent` | `workflow`. Deliberately unconstrained — see note 1 above.
+    target_kind text        NOT NULL,
+    -- The agent or workflow name, resolved against the catalog at launch.
+    target_name text        NOT NULL,
+    dataset_id  uuid        NOT NULL REFERENCES stackbone_platform.eval_datasets (id),
+    -- A pinned version number, or NULL for "the latest". NULL by default is what
+    -- the operator wants: the case captured yesterday joins today's run.
+    dataset_pin integer,
+    -- A list of { type, params }, validated against the scorer catalog.
+    scorers     jsonb       NOT NULL,
+    -- The pass mark. Left without a recommended value on purpose: inventing one
+    -- before there is data from a real suite would be a guess dressed as advice.
+    threshold   numeric     NOT NULL,
+    -- How many times each case runs. More than one is how a case that only
+    -- passes half the time stops sneaking through as a pass.
+    repeats     integer     NOT NULL DEFAULT 1 CHECK (repeats >= 1),
+    -- The spend brake. Every case costs real tokens out of the customer's own
+    -- budget, so a run that would exceed this truncates and SAYS SO.
+    max_cases   integer     NOT NULL CHECK (max_cases >= 1),
+    created_at  timestamptz NOT NULL DEFAULT now(),
+    updated_at  timestamptz NOT NULL DEFAULT now(),
+    created_by  text,
+    updated_by  text,
+    deleted_at  timestamptz
+);
+
+-- Unique among ACTIVE suites only, for the same anti-squatting reason as the
+-- dataset key above.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_suites_active_name_idx
+    ON stackbone_platform.eval_suites (name)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS eval_suites_active_dataset_idx
+    ON stackbone_platform.eval_suites (dataset_id)
+    WHERE deleted_at IS NULL;
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_runs (
+    id                 uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    suite_id           uuid        NOT NULL
+                       REFERENCES stackbone_platform.eval_suites (id) ON DELETE CASCADE,
+    -- The EXACT version measured, never "whatever is current now". Without this
+    -- pin, "this run scored 90 and last month's scored 70" is not a comparison.
+    dataset_version_id uuid        NOT NULL
+                       REFERENCES stackbone_platform.eval_dataset_versions (id),
+    -- Version seal: what was actually evaluated, so an improvement can be
+    -- attributed to a change instead of guessed at.
+    git_sha            text,
+    agent_version      text,
+    -- One or two variants, each with its configuration and its SEALED
+    -- aggregates. Sealed at close so opening the comparison screen does not
+    -- recompute — and so the numbers cannot drift under a reader.
+    variants           jsonb       NOT NULL,
+    -- `running` | `done` | `failed` | `cancelled`. Unconstrained — see note 1.
+    -- Also the CANCEL channel: the engine checks this row between cases and
+    -- aborts clean, sealing what it already measured. There is no live run
+    -- cancellation in the platform to lean on.
+    status             text        NOT NULL,
+    -- DISTINCT CASES this run meant to measure, after the spend brake. Not
+    -- evaluations: `truncation_notice` reads this number as cases ("stopped this
+    -- run at 5 cases"), and the plan LIMITs by it.
+    case_count         integer     NOT NULL,
+    -- A run that measured fewer cases than the suite asked for has to say so,
+    -- not present partial coverage as complete.
+    truncated          boolean     NOT NULL DEFAULT false,
+    -- How many times each case ran, SEALED from the suite at launch beside the
+    -- dataset version and the variants — and for the same reason: editing the
+    -- suite next week must not change what an old run says it measured. It is
+    -- also the missing factor in the progress denominator (cases × repeats ×
+    -- variants); without it on the row, a screen counting attempts against
+    -- cases reports a run of 30 evaluations as finished after 5.
+    repeats            integer     NOT NULL DEFAULT 1 CHECK (repeats >= 1),
+    -- Progress, counted in EVALUATIONS (one row of `eval_case_results` per case
+    -- × variant × repeat), polled by the screen while the run is in flight.
+    completed_count    integer     NOT NULL DEFAULT 0,
+    passed_count       integer     NOT NULL DEFAULT 0,
+    -- Sealed aggregates: NULL until the run closes.
+    score_avg          numeric,
+    input_tokens       integer,
+    output_tokens      integer,
+    p95_duration_ms    integer,
+    created_at         timestamptz NOT NULL DEFAULT now(),
+    created_by         text,
+    started_at         timestamptz,
+    finished_at        timestamptz
+);
+
+-- Drives both the suite's run history and the "diff against the previous run"
+-- read. NULLS LAST keeps a queued run (no `started_at` yet) off the top.
+CREATE INDEX IF NOT EXISTS eval_runs_suite_started_idx
+    ON stackbone_platform.eval_runs (suite_id, started_at DESC NULLS LAST);
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_case_results (
+    id            uuid    PRIMARY KEY DEFAULT gen_random_uuid(),
+    eval_run_id   uuid    NOT NULL
+                  REFERENCES stackbone_platform.eval_runs (id) ON DELETE CASCADE,
+    -- BARE uuid with NO foreign key, beside a denormalised `case_name`: the case
+    -- row dies with its dataset version, and the verdict has to stay readable
+    -- afterwards. Do not "fix" this into an FK.
+    case_id       uuid,
+    case_name     text    NOT NULL,
+    -- Which of the (at most two) variants produced this verdict.
+    variant_label text    NOT NULL,
+    attempt       integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
+    -- The hinge: from a failed case, jump to its full trace. ON DELETE SET NULL
+    -- and not CASCADE — when run retention eventually prunes the run, the
+    -- VERDICT survives and only the explanation is lost. The score is the
+    -- evidence; the trace is the commentary.
+    run_id        uuid    REFERENCES stackbone_platform.runs (id) ON DELETE SET NULL,
+    passed        boolean NOT NULL,
+    score         numeric,
+    -- One entry per scorer, so a failure names WHICH criterion rejected it.
+    scores        jsonb,
+    output        jsonb,
+    duration_ms   integer,
+    -- Structured failure: a case killed by a technical error counts as a failure
+    -- without taking the run down, and a case stopped by a guardrail or parked
+    -- on a human approval is distinguishable from one that simply answered badly.
+    error         jsonb,
+    created_at    timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS eval_case_results_run_idx
+    ON stackbone_platform.eval_case_results (eval_run_id);
+
+-- A note on visibility, since it is a consequence nobody chose explicitly:
+-- 0005_stackbone_viewer.sql set DEFAULT PRIVILEGES granting SELECT on new tables
+-- in this schema to `stackbone_viewer`, so the Studio DB Explorer can read these
+-- rows — including `eval_cases.input`, which will hold real captured customer
+-- data. That matches how `runs.input` already behaves. If a deployment ever
+-- needs these hidden, 0011_secrets_revoke_viewer.sql is the precedent for
+-- revoking; nothing is revoked here.
+--
+-- No pg_notify trigger is attached to any of these tables ON PURPOSE. Ten
+-- earlier migrations do attach one, so copying it in is the obvious reflex — but
+-- live progress for an eval run is done by POLLING the `eval_runs` row, which
+-- carries its own counters. Emitting eval frames would mean touching the
+-- trigger, the emulator bus and the client parser to paint a counter.

package/migrations/0039_eval_case_results_idempotency.sql

@@ -0,0 +1,22 @@
+-- 0039_eval_case_results_idempotency.sql — the constraint behind "a retry does
+-- not re-invoke the model" (STA-415, PRD STA-393).
+--
+-- The engine already reads which cases an eval run has verdicts for and skips
+-- them when it resumes. That read-then-write is a RACE on its own: two workers
+-- picking up the same interrupted run would each read "case 3 has no verdict"
+-- and each pay for case 3 in the customer's own token budget. Every eval case
+-- costs real money, so the skip needs a constraint under it and not just a
+-- convention.
+--
+-- A verdict is identified by WHICH case, under WHICH variant, on WHICH attempt.
+-- All four columns are needed: repeats are the whole reason `attempt` exists (a
+-- case that only passes half the time has to be run more than once), and the two
+-- variants of a comparison measure the same case on purpose.
+--
+-- NULLS NOT DISTINCT because `case_id` is nullable — a verdict outlives the case
+-- row it measured. Under the default (NULLS DISTINCT) two verdicts with a null
+-- case would both be allowed through the index and the append would stop being
+-- idempotent exactly when a list had been edited underneath a resumed run.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_case_results_attempt_idx
+    ON stackbone_platform.eval_case_results (eval_run_id, case_id, variant_label, attempt)
+    NULLS NOT DISTINCT;

package/migrations/0040_eval_case_results_run_idx.sql

@@ -0,0 +1,27 @@
+-- 0040_eval_case_results_run_idx.sql — the index behind the foreign key 0038 added
+-- from `eval_case_results` to `runs` (STA-415, PRD STA-393).
+--
+-- `eval_case_results.run_id` REFERENCES `runs (id) ON DELETE SET NULL`, and
+-- Postgres enforces that by running the equivalent of
+--   UPDATE eval_case_results SET run_id = NULL WHERE run_id = $1
+-- inside EVERY transaction that deletes a `runs` row. With no index on the
+-- referencing column that is a full sequential scan of what becomes the largest
+-- table this feature creates (cases x repeats x variants x runs, never pruned —
+-- 0038 says out loud that a verdict must outlive its run).
+--
+-- The cost lands on code that has nothing to do with evals: the workflow
+-- concurrency gate deletes a single `waiting` run row on every admission
+-- (`clearWaitingRun` / `clearOldestWaitingRun`), so admission latency would grow
+-- with the accumulated eval history of a box that merely ran nightly evals.
+--
+-- The repo already gets this right for the two other foreign keys of the same
+-- shape — `approvals_run_idx` next to 0002's ON DELETE SET NULL, and
+-- `run_steps_run_started_idx` next to 0018's cascade. This is the one that was
+-- left bare. `eval_case_results_run_idx` (0038) reads like it covers this, but it
+-- indexes `eval_run_id` — the EVAL run, not the platform run.
+--
+-- A separate migration rather than an edit to 0038: an applied migration's
+-- checksum is verified on every boot, so rewriting it would brick a box that
+-- already ran it.
+CREATE INDEX IF NOT EXISTS eval_case_results_platform_run_idx
+    ON stackbone_platform.eval_case_results (run_id);

package/migrations/0041_eval_criterion_workflow_name.sql

@@ -0,0 +1,68 @@
+-- 0041_eval_criterion_workflow_name.sql — write the name that used to be implicit
+-- onto every stored `custom_workflow` criterion (STA-521, PRD STA-393).
+--
+-- The criterion "judged by your own workflow" used to resolve ONE reserved workflow
+-- name and store nothing about it: the role WAS the workflow name, so a suite only
+-- had to say "custom_workflow" and the box knew where to go. STA-521 makes the
+-- judging workflow a setting picked from a dropdown of what the box holds, which
+-- means the name is now something a suite has to CARRY.
+--
+-- WHY THIS SHIPS IN THE SAME SLICE AS THE CATALOG CHANGE, not after it. The catalog
+-- rejects settings it does not declare, so a backfill landing FIRST would break
+-- every subsequent save; and once `workflowName` is required, an unmigrated suite
+-- can neither be saved nor judged. Either order on its own opens a window in which
+-- a stored suite is broken. Both halves land together, so there is no window.
+--
+-- THE LITERAL BELOW MUST MATCH `EVAL_CRITERION_ROLE` in
+-- libs/shared/evals/src/lib/custom/custom-criterion.ts. SQL cannot import the
+-- constant, and that constant now exists for exactly this reason: it is the value
+-- every suite meant while the name was implicit, and both sides have to mean the
+-- same thing by it. The TS side keeps a comment pointing back here.
+--
+-- ONLY WHERE THE SETTING IS ABSENT OR BLANK. A box that already had the picker has
+-- an operator's own choice in that field; overwriting it would silently re-point
+-- their judge at a workflow they never picked, and nothing on the screen would say
+-- so. `COALESCE(... ->> 'workflowName', '') = ''` covers all three ways of having
+-- said nothing — the key missing, a JSON null, and an empty string — and makes the
+-- statement idempotent: run it twice and the second pass matches nothing.
+--
+-- `scorers` is a jsonb ARRAY of `{type, params}`, so the rewrite goes element by
+-- element through `jsonb_array_elements` and reassembles the list with `jsonb_agg`.
+-- Element ORDER is preserved through `WITH ORDINALITY` … `ORDER BY`, because a
+-- criterion list is read positionally on the screen; the co-located spec pins that
+-- against a suite whose criterion is neither first nor alone. The `COALESCE(…,
+-- '[]')` around the aggregate is belt-and-braces only — `jsonb_agg` over no rows
+-- returns NULL, and the `EXISTS` guard below already means no row without criteria
+-- is ever updated. It stays because writing NULL into a NOT NULL column is the one
+-- mistake here that would take a boot down.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+UPDATE stackbone_platform.eval_suites AS s
+   SET scorers = (
+       SELECT COALESCE(jsonb_agg(rewritten.scorer ORDER BY rewritten.position), '[]'::jsonb)
+         FROM (
+              SELECT element.position,
+                     CASE
+                       WHEN element.scorer->>'type' = 'custom_workflow'
+                        AND COALESCE(element.scorer->'params'->>'workflowName', '') = ''
+                       THEN jsonb_set(
+                              element.scorer,
+                              '{params}',
+                              COALESCE(element.scorer->'params', '{}'::jsonb)
+                                || jsonb_build_object('workflowName', 'eval-criterion')
+                            )
+                       ELSE element.scorer
+                     END AS scorer
+                FROM jsonb_array_elements(s.scorers)
+                     WITH ORDINALITY AS element(scorer, position)
+              ) AS rewritten
+       )
+ WHERE jsonb_typeof(s.scorers) = 'array'
+   AND EXISTS (
+       SELECT 1
+         FROM jsonb_array_elements(s.scorers) AS scorer
+        WHERE scorer->>'type' = 'custom_workflow'
+          AND COALESCE(scorer->'params'->>'workflowName', '') = ''
+       );

package/migrations/0042_eval_case_results_ended.sql

@@ -0,0 +1,64 @@
+-- 0042_eval_case_results_ended.sql — record HOW a measured case ended, next to
+-- the free-form answer the verdict already keeps (STA-526, PRD STA-393).
+--
+-- 0038's own comment already promised this: "a case stopped by a guardrail or
+-- parked on a human approval is distinguishable from one that simply answered
+-- badly". It was only ever true by convention — the ending was a TypeScript
+-- union translated away before the insert, and the sole trace left behind was a
+-- string inside the `error` blob. Studio then had to guess the verdict by
+-- matching the front of that string, which is how a parked workflow ended up
+-- painted as an error.
+--
+-- A COLUMN rather than a code inside `error`, for two reasons:
+--   - an agent that correctly ESCALATED is not an error, and archiving it as one
+--     makes a suite of held cases read as a broken box;
+--   - "3 of 15 escalated" has to be one query, not a JSON read per row.
+--
+-- THE SIX VALUES, and why the two parks are separate:
+--   completed             reached a terminal state with an answer
+--   escalated             parked on an operator approval
+--   asked_user            parked on a resume hook, waiting on a person
+--   blocked_by_guardrail  refused at the border, never started
+--   errored               broke
+--   cancelled             stopped before it could finish
+-- `escalated` and `asked_user` are NOT folded together: telling "escalated to a
+-- human" from "asked the end user" is the whole distinction the pause criteria
+-- are being built to measure.
+--
+-- NULLABLE, and deliberately never backfilled. This is an APPEND-ONLY results
+-- table: every verdict written before today belongs to a sealed run, and there
+-- is no honest value to invent for it — `completed` would put words in that
+-- run's mouth. A NULL reads as "this run could not say", which is the truth.
+-- It also keeps a rolling upgrade safe: a box that applies this migration a
+-- moment before its runners learn to write the column still writes verdicts,
+-- rather than failing every one of them on a NOT NULL.
+--
+-- A CHECK and not free text, unlike `runs.entity_kind` (0037) or the suite's
+-- `target_kind`. Those two are discriminators a box on an older build must be
+-- able to STORE without executing, so rejecting an unknown value would lose an
+-- operator's configuration. This column is the opposite: the box WRITES it, one
+-- of six values decided in this repository, and a screen that groups on it must
+-- not have to defend against a seventh spelling invented by a typo.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD COLUMN IF NOT EXISTS ended text;
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_case_results
+    DROP CONSTRAINT IF EXISTS eval_case_results_ended_check;
+
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD CONSTRAINT eval_case_results_ended_check
+    CHECK (ended IS NULL OR ended IN (
+        'completed',
+        'escalated',
+        'asked_user',
+        'blocked_by_guardrail',
+        'errored',
+        'cancelled'
+    ));

package/migrations/0043_eval_suite_approval_policy.sql

@@ -0,0 +1,50 @@
+-- 0043_eval_suite_approval_policy.sql — what a suite does about an approval one
+-- of its cases parks on (STA-530, PRD STA-393).
+--
+-- A measured case has NOBODY waiting on it. The thread belongs to one attempt of
+-- one case and is never spoken to again, so an agent that correctly stops to ask
+-- an operator is scored on the stop and everything it would have said afterwards
+-- is never seen. Half of what an approval-bearing agent does is on the other side
+-- of that gate, and today a suite cannot measure any of it.
+--
+-- THE FOUR VALUES:
+--   leave_waiting   the case closes on the park and the request it raised is
+--                   taken back out of the inbox. What the product did before this
+--                   column existed.
+--   reject          answer it no, then keep watching the turn.
+--   approve         answer it yes, then keep watching the turn.
+--   model_decides   ask the judging model which of the two, then answer it.
+--
+-- NOT NULL with a DEFAULT, unlike 0042's `ended`. The opposite situation: that
+-- one is an append-only results column with no honest value to invent for a row
+-- written before it existed, while this is CONFIGURATION and the honest value for
+-- every suite already stored is precisely what they do today — leave the case
+-- waiting. Backfilling it is therefore not putting words in anybody's mouth.
+--
+-- A CHECK and not free text, for the same reason 0042's ending carries one: the
+-- BOX writes and reads this, one of four values decided in this repository, and
+-- the runner that branches on it must not have to defend against a fifth spelling
+-- invented by a typo. The suite's own `target_kind` is deliberately open text by
+-- contrast — that one is a discriminator a box on an older build has to be able
+-- to store without executing.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+ALTER TABLE stackbone_platform.eval_suites
+    ADD COLUMN IF NOT EXISTS approval_policy text NOT NULL DEFAULT 'leave_waiting';
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_suites
+    DROP CONSTRAINT IF EXISTS eval_suites_approval_policy_check;
+
+ALTER TABLE stackbone_platform.eval_suites
+    ADD CONSTRAINT eval_suites_approval_policy_check
+    CHECK (approval_policy IN (
+        'leave_waiting',
+        'reject',
+        'approve',
+        'model_decides'
+    ));

package/migrations/0044_eval_persona_simulator.sql

@@ -0,0 +1,88 @@
+-- 0044_eval_persona_simulator.sql — a case carries a persona, and the suite
+-- decides whether anybody plays it (STA-532, PRD STA-393).
+--
+-- The clarification loop is the part of the product no eval touched. An agent
+-- that asks before it acts parks on the first question, so what it would have
+-- done with the answer — the whole reason it asks at all — was never measured.
+-- 0043 opened the half of that story that faces an OPERATOR; this one opens the
+-- half that faces the END USER.
+--
+-- FOUR TABLES, and each column is on the table that owns the fact:
+--
+--   eval_cases.persona              WHO answers, in what words, how many times.
+--                                   Belongs to the case, so the same list can be
+--                                   run by a suite that measures the first
+--                                   question and one that measures the whole
+--                                   conversation.
+--   eval_suites.simulator           WHETHER anybody plays it, and with which
+--                                   model. Belongs to the suite, so a persona
+--                                   with the simulator off is simply ignored.
+--   eval_runs.simulator_model       WHO ACTUALLY played it, sealed at launch.
+--   eval_runs.reply_cap             HOW MANY TIMES they were allowed to.
+--   eval_runs.simulated_connectors  WHICH connectors were stood in for.
+--   eval_case_results.simulation    WHAT was said, turn by turn, marking which
+--                                   lines were scripted and which improvised.
+--
+-- WHY THE RUN SEALS THREE MORE THINGS. Two runs of the same suite a month apart
+-- have to stay comparable, and every one of these moves a score without the agent
+-- changing at all: a different model plays the user differently, a cap raised
+-- from three to five gives the agent two more chances, and a connector that was
+-- real last month and simulated today is a different world. A run that did not
+-- record them makes the next comparison read a change of setting as a change in
+-- the agent.
+--
+-- A NOTE ON GRANULARITY, so it is not rediscovered as a surprise. `git_sha` and
+-- `agent_version` on this table are already taken from the FIRST variant only,
+-- while a run may carry two. The three columns below are added at the SAME single
+-- value per run, deliberately — a per-variant seal for these alone would leave
+-- the row half one shape and half another. When the variant scoping is fixed it
+-- has to be fixed for all five together.
+--
+-- JSONB rather than columns for `persona` and `simulator`, matching `scorers` and
+-- `variants` beside them: they are configuration payloads whose documented
+-- defaults live in `@stackbone/validators` → `studio-evals.ts`, and a CHECK here
+-- would be a second description that drifts from the form an operator fills in.
+--
+-- `simulated_connectors` is `text[]` rather than jsonb, matching `eval_cases.tags`
+-- — it is a flat list of identifiers, and the array type is what makes "which
+-- runs simulated Outlook?" an ordinary query.
+--
+-- NULLABLE where the fact is HISTORICAL and defaulted where it is CONFIGURATION,
+-- exactly as 0042 and 0043 split it. A verdict written before this column existed
+-- has no honest simulation to invent, and a run sealed before it has no honest
+-- model to name — those read NULL, which is the truth. A suite, by contrast, has
+-- a perfectly honest current value: nobody answers, which is what every stored
+-- suite does today.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+
+ALTER TABLE stackbone_platform.eval_cases
+    ADD COLUMN IF NOT EXISTS persona jsonb;
+
+ALTER TABLE stackbone_platform.eval_suites
+    ADD COLUMN IF NOT EXISTS simulator jsonb NOT NULL
+    DEFAULT '{"enabled": false, "model": null}'::jsonb;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS simulator_model text;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS reply_cap integer;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS simulated_connectors text[] NOT NULL DEFAULT '{}';
+
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD COLUMN IF NOT EXISTS simulation jsonb;
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_runs
+    DROP CONSTRAINT IF EXISTS eval_runs_reply_cap_check;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD CONSTRAINT eval_runs_reply_cap_check
+    CHECK (reply_cap IS NULL OR reply_cap >= 0);

package/migrations/0045_eval_case_description.sql

@@ -0,0 +1,34 @@
+-- 0045_eval_case_description.sql — a case says, in prose, what it is for.
+
+-- A case list is read far more often than it is written, and until now the only
+-- thing a reader got was the `name`. A name has to be short and unique, so it
+-- ends up as a slug — `prof-asunto-y-cuerpo-discrepan` — and everything that
+-- makes the case worth having lives in whoever wrote it.
+--
+-- WHERE THIS WAS BEING KEPT INSTEAD, and why that is the actual argument for the
+-- column. Case lists in the wild already carry the explanation; they just have
+-- nowhere to put it, so it goes inside `expected` as an extra key. That field is
+-- the REFERENCE ANSWER — the thing scorers compare against — so an explanation
+-- parked there is prose sitting inside the ground truth, surviving only because
+-- each scorer is taught to skip that one key by name. A scorer that compares the
+-- whole object (`exact-match` does) fails every case that has one.
+--
+-- So this is not a new capability so much as moving a fact onto the row that owns
+-- it, which leaves `expected` holding only what is actually compared.
+--
+-- `text` and NULLABLE, not `NOT NULL DEFAULT ''`:
+--
+--   * Every case stored before today genuinely has no description, and NULL is
+--     what "nobody wrote one" looks like. A default of '' would claim all of them
+--     had been described with an empty sentence, and a screen cannot tell that
+--     apart from a description somebody cleared on purpose.
+--   * No length CHECK, matching `eval_cases.name` beside it: the bound is a form
+--     concern and lives with the form, in `@stackbone/validators` →
+--     `studio-evals.ts`. A number repeated here is a second bound that drifts.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+
+ALTER TABLE stackbone_platform.eval_cases
+    ADD COLUMN IF NOT EXISTS description text;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackbone/cli",
-  "version": "0.2.8",
+  "version": "0.3.0",
   "license": "UNLICENSED",
   "type": "module",
   "bin": {