@stackbone/cli 0.2.7 → 0.2.10

package/migrations/0038_evals.sql

@@ -0,0 +1,272 @@
+-- 0038_evals.sql — EVALS in the box (STA-408, PRD STA-393): the six tables that
+-- turn "is the agent any good?" from an opinion into a number, plus the `eval`
+-- run trigger.
+--
+-- Widening a named CHECK REPLACES IT WHOLESALE. `runs_trigger_check` was
+-- re-listed by hand in 0001 (manual/webhook/cron/event), 0016 (+job) and 0017
+-- (+workflow/chat); a migration that only remembers its own value silently
+-- revokes the rest, and nothing conflicts because they are different files. All
+-- eight are re-listed below, and `evals-migration.spec.ts` now pins the complete
+-- set so the next widening breaks loudly.
+ALTER TABLE stackbone_platform.runs DROP CONSTRAINT IF EXISTS runs_trigger_check;
+ALTER TABLE stackbone_platform.runs
+    ADD CONSTRAINT runs_trigger_check
+    CHECK (trigger IN (
+        'manual', 'webhook', 'cron', 'event', 'job', 'workflow', 'chat', 'eval'
+    ));
+
+-- --------------------------------------------------------------------------
+-- The case list: a header row plus one immutable version per edit.
+-- --------------------------------------------------------------------------
+--
+-- Same shape as prompts (0013), and for the same reason: a run has to keep
+-- saying WHICH cases it measured. Editing the list copies its cases into a new
+-- version, so a run from last month still reads its exact cases even after three
+-- edits. The copy costs one INSERT per case; for the expected size — tens or
+-- hundreds of cases per list — that is irrelevant next to losing the ability to
+-- compare two runs at all.
+--
+-- Both header tables carry `deleted_at` and are SOFT-deleted by the store,
+-- because a run points at a dataset version (and a suite) and a sealed run that
+-- lost its subject is not evidence of anything. The uniqueness of the operator's
+-- key therefore has to be PARTIAL — a full UNIQUE would let a deleted list squat
+-- its name forever, so an operator who removes "smoke tests" could never make
+-- another one.
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_datasets (
+    id              uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    -- Operator-facing handle, unique among the ACTIVE lists (index below).
+    key             text        NOT NULL,
+    name            text        NOT NULL,
+    description     text,
+    -- Points at the live row in `eval_dataset_versions` for this list.
+    current_version integer     NOT NULL DEFAULT 1 CHECK (current_version >= 1),
+    created_at      timestamptz NOT NULL DEFAULT now(),
+    updated_at      timestamptz NOT NULL DEFAULT now(),
+    -- The acting user, as the verified JWT / signed actor header reported them.
+    -- NULL on the un-authenticated `stackbone dev` datapath.
+    created_by      text,
+    updated_by      text,
+    -- Soft delete. Reads filter on it; the row survives so the runs that
+    -- measured against it keep their subject.
+    deleted_at      timestamptz
+);
+
+-- Unique among ACTIVE lists only: see the note above on squatting.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_datasets_active_key_idx
+    ON stackbone_platform.eval_datasets (key)
+    WHERE deleted_at IS NULL;
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_dataset_versions (
+    id         uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    dataset_id uuid        NOT NULL
+               REFERENCES stackbone_platform.eval_datasets (id) ON DELETE CASCADE,
+    version    integer     NOT NULL CHECK (version >= 1),
+    -- Free text an operator can leave describing what this edit changed.
+    note       text,
+    created_at timestamptz NOT NULL DEFAULT now(),
+    created_by text
+);
+
+-- The race backstop. The store assigns MAX(version) + 1 under a per-dataset
+-- advisory lock inside the same transaction, exactly like prompts; this index is
+-- what stops two writers both landing version 1 if that lock is ever lost.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_dataset_versions_version_idx
+    ON stackbone_platform.eval_dataset_versions (dataset_id, version);
+
+-- Drives the version-history list (newest first).
+CREATE INDEX IF NOT EXISTS eval_dataset_versions_version_desc_idx
+    ON stackbone_platform.eval_dataset_versions (dataset_id, version DESC);
+
+-- A case belongs to ONE version. Editing the list copies the cases forward.
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_cases (
+    id                 uuid    PRIMARY KEY DEFAULT gen_random_uuid(),
+    dataset_version_id uuid    NOT NULL
+                       REFERENCES stackbone_platform.eval_dataset_versions (id) ON DELETE CASCADE,
+    name               text    NOT NULL,
+    input              jsonb   NOT NULL,
+    -- Optional on purpose: most cases are judged by a rubric, and demanding a
+    -- reference answer would force the operator to invent one.
+    expected           jsonb,
+    tags               text[]  NOT NULL DEFAULT '{}',
+    -- Which run this case was captured from, if it was captured. Deliberately a
+    -- BARE uuid with NO foreign key: a captured case has to outlive run
+    -- retention, and a case that vanished when its source run was pruned would
+    -- silently shrink the list an operator built. Do not "fix" this into an FK.
+    source_run_id      uuid,
+    -- Explicit order within the version, so the list reads the way it was built.
+    position           integer NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS eval_cases_version_position_idx
+    ON stackbone_platform.eval_cases (dataset_version_id, position);
+
+-- --------------------------------------------------------------------------
+-- The suite, its runs, and the per-case verdicts.
+-- --------------------------------------------------------------------------
+--
+-- Two conventions inherited from guardrails (0034), and they are the reason this
+-- table looks under-constrained:
+--
+--   1. The DISCRIMINATOR columns — `target_kind`, `status` — carry NO value
+--      CHECK. A box running an older build has to be able to STORE a kind or a
+--      state it cannot execute rather than reject the write and lose the
+--      operator's configuration. An unknown value fails CLOSED at evaluation
+--      time, which is the safe place for that decision.
+--   2. No PAYLOAD default lives here. The documented defaults live in
+--      `@stackbone/validators` → `studio-evals.ts`, so the runtime, the facade
+--      and the Studio form all read one source of truth. The defaults below are
+--      structural only: identity, timestamps, the progress counters whose
+--      absence would make a half-written run row unreadable, and `repeats`,
+--      whose 1 is the neutral value (run each case once) rather than a
+--      configuration choice.
+--
+-- `scorers` and `variants` are free-form jsonb validated against the CATALOG at
+-- the handler, never by the DDL. A CHECK here would be a second description of
+-- each scorer that drifts from the one the screen renders its form from. Neither
+-- gets an empty-array default: a suite that measures nothing, and a run with no
+-- variant, are not rows worth making writable.
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_suites (
+    id          uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    -- Operator-facing name, unique among the ACTIVE suites (index below), and
+    -- what `stackbone eval <suite>` is given on the command line.
+    name        text        NOT NULL,
+    description text,
+    -- `agent` | `workflow`. Deliberately unconstrained — see note 1 above.
+    target_kind text        NOT NULL,
+    -- The agent or workflow name, resolved against the catalog at launch.
+    target_name text        NOT NULL,
+    dataset_id  uuid        NOT NULL REFERENCES stackbone_platform.eval_datasets (id),
+    -- A pinned version number, or NULL for "the latest". NULL by default is what
+    -- the operator wants: the case captured yesterday joins today's run.
+    dataset_pin integer,
+    -- A list of { type, params }, validated against the scorer catalog.
+    scorers     jsonb       NOT NULL,
+    -- The pass mark. Left without a recommended value on purpose: inventing one
+    -- before there is data from a real suite would be a guess dressed as advice.
+    threshold   numeric     NOT NULL,
+    -- How many times each case runs. More than one is how a case that only
+    -- passes half the time stops sneaking through as a pass.
+    repeats     integer     NOT NULL DEFAULT 1 CHECK (repeats >= 1),
+    -- The spend brake. Every case costs real tokens out of the customer's own
+    -- budget, so a run that would exceed this truncates and SAYS SO.
+    max_cases   integer     NOT NULL CHECK (max_cases >= 1),
+    created_at  timestamptz NOT NULL DEFAULT now(),
+    updated_at  timestamptz NOT NULL DEFAULT now(),
+    created_by  text,
+    updated_by  text,
+    deleted_at  timestamptz
+);
+
+-- Unique among ACTIVE suites only, for the same anti-squatting reason as the
+-- dataset key above.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_suites_active_name_idx
+    ON stackbone_platform.eval_suites (name)
+    WHERE deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS eval_suites_active_dataset_idx
+    ON stackbone_platform.eval_suites (dataset_id)
+    WHERE deleted_at IS NULL;
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_runs (
+    id                 uuid        PRIMARY KEY DEFAULT gen_random_uuid(),
+    suite_id           uuid        NOT NULL
+                       REFERENCES stackbone_platform.eval_suites (id) ON DELETE CASCADE,
+    -- The EXACT version measured, never "whatever is current now". Without this
+    -- pin, "this run scored 90 and last month's scored 70" is not a comparison.
+    dataset_version_id uuid        NOT NULL
+                       REFERENCES stackbone_platform.eval_dataset_versions (id),
+    -- Version seal: what was actually evaluated, so an improvement can be
+    -- attributed to a change instead of guessed at.
+    git_sha            text,
+    agent_version      text,
+    -- One or two variants, each with its configuration and its SEALED
+    -- aggregates. Sealed at close so opening the comparison screen does not
+    -- recompute — and so the numbers cannot drift under a reader.
+    variants           jsonb       NOT NULL,
+    -- `running` | `done` | `failed` | `cancelled`. Unconstrained — see note 1.
+    -- Also the CANCEL channel: the engine checks this row between cases and
+    -- aborts clean, sealing what it already measured. There is no live run
+    -- cancellation in the platform to lean on.
+    status             text        NOT NULL,
+    -- DISTINCT CASES this run meant to measure, after the spend brake. Not
+    -- evaluations: `truncation_notice` reads this number as cases ("stopped this
+    -- run at 5 cases"), and the plan LIMITs by it.
+    case_count         integer     NOT NULL,
+    -- A run that measured fewer cases than the suite asked for has to say so,
+    -- not present partial coverage as complete.
+    truncated          boolean     NOT NULL DEFAULT false,
+    -- How many times each case ran, SEALED from the suite at launch beside the
+    -- dataset version and the variants — and for the same reason: editing the
+    -- suite next week must not change what an old run says it measured. It is
+    -- also the missing factor in the progress denominator (cases × repeats ×
+    -- variants); without it on the row, a screen counting attempts against
+    -- cases reports a run of 30 evaluations as finished after 5.
+    repeats            integer     NOT NULL DEFAULT 1 CHECK (repeats >= 1),
+    -- Progress, counted in EVALUATIONS (one row of `eval_case_results` per case
+    -- × variant × repeat), polled by the screen while the run is in flight.
+    completed_count    integer     NOT NULL DEFAULT 0,
+    passed_count       integer     NOT NULL DEFAULT 0,
+    -- Sealed aggregates: NULL until the run closes.
+    score_avg          numeric,
+    input_tokens       integer,
+    output_tokens      integer,
+    p95_duration_ms    integer,
+    created_at         timestamptz NOT NULL DEFAULT now(),
+    created_by         text,
+    started_at         timestamptz,
+    finished_at        timestamptz
+);
+
+-- Drives both the suite's run history and the "diff against the previous run"
+-- read. NULLS LAST keeps a queued run (no `started_at` yet) off the top.
+CREATE INDEX IF NOT EXISTS eval_runs_suite_started_idx
+    ON stackbone_platform.eval_runs (suite_id, started_at DESC NULLS LAST);
+
+CREATE TABLE IF NOT EXISTS stackbone_platform.eval_case_results (
+    id            uuid    PRIMARY KEY DEFAULT gen_random_uuid(),
+    eval_run_id   uuid    NOT NULL
+                  REFERENCES stackbone_platform.eval_runs (id) ON DELETE CASCADE,
+    -- BARE uuid with NO foreign key, beside a denormalised `case_name`: the case
+    -- row dies with its dataset version, and the verdict has to stay readable
+    -- afterwards. Do not "fix" this into an FK.
+    case_id       uuid,
+    case_name     text    NOT NULL,
+    -- Which of the (at most two) variants produced this verdict.
+    variant_label text    NOT NULL,
+    attempt       integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
+    -- The hinge: from a failed case, jump to its full trace. ON DELETE SET NULL
+    -- and not CASCADE — when run retention eventually prunes the run, the
+    -- VERDICT survives and only the explanation is lost. The score is the
+    -- evidence; the trace is the commentary.
+    run_id        uuid    REFERENCES stackbone_platform.runs (id) ON DELETE SET NULL,
+    passed        boolean NOT NULL,
+    score         numeric,
+    -- One entry per scorer, so a failure names WHICH criterion rejected it.
+    scores        jsonb,
+    output        jsonb,
+    duration_ms   integer,
+    -- Structured failure: a case killed by a technical error counts as a failure
+    -- without taking the run down, and a case stopped by a guardrail or parked
+    -- on a human approval is distinguishable from one that simply answered badly.
+    error         jsonb,
+    created_at    timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS eval_case_results_run_idx
+    ON stackbone_platform.eval_case_results (eval_run_id);
+
+-- A note on visibility, since it is a consequence nobody chose explicitly:
+-- 0005_stackbone_viewer.sql set DEFAULT PRIVILEGES granting SELECT on new tables
+-- in this schema to `stackbone_viewer`, so the Studio DB Explorer can read these
+-- rows — including `eval_cases.input`, which will hold real captured customer
+-- data. That matches how `runs.input` already behaves. If a deployment ever
+-- needs these hidden, 0011_secrets_revoke_viewer.sql is the precedent for
+-- revoking; nothing is revoked here.
+--
+-- No pg_notify trigger is attached to any of these tables ON PURPOSE. Ten
+-- earlier migrations do attach one, so copying it in is the obvious reflex — but
+-- live progress for an eval run is done by POLLING the `eval_runs` row, which
+-- carries its own counters. Emitting eval frames would mean touching the
+-- trigger, the emulator bus and the client parser to paint a counter.

package/migrations/0039_eval_case_results_idempotency.sql

@@ -0,0 +1,22 @@
+-- 0039_eval_case_results_idempotency.sql — the constraint behind "a retry does
+-- not re-invoke the model" (STA-415, PRD STA-393).
+--
+-- The engine already reads which cases an eval run has verdicts for and skips
+-- them when it resumes. That read-then-write is a RACE on its own: two workers
+-- picking up the same interrupted run would each read "case 3 has no verdict"
+-- and each pay for case 3 in the customer's own token budget. Every eval case
+-- costs real money, so the skip needs a constraint under it and not just a
+-- convention.
+--
+-- A verdict is identified by WHICH case, under WHICH variant, on WHICH attempt.
+-- All four columns are needed: repeats are the whole reason `attempt` exists (a
+-- case that only passes half the time has to be run more than once), and the two
+-- variants of a comparison measure the same case on purpose.
+--
+-- NULLS NOT DISTINCT because `case_id` is nullable — a verdict outlives the case
+-- row it measured. Under the default (NULLS DISTINCT) two verdicts with a null
+-- case would both be allowed through the index and the append would stop being
+-- idempotent exactly when a list had been edited underneath a resumed run.
+CREATE UNIQUE INDEX IF NOT EXISTS eval_case_results_attempt_idx
+    ON stackbone_platform.eval_case_results (eval_run_id, case_id, variant_label, attempt)
+    NULLS NOT DISTINCT;

package/migrations/0040_eval_case_results_run_idx.sql

@@ -0,0 +1,27 @@
+-- 0040_eval_case_results_run_idx.sql — the index behind the foreign key 0038 added
+-- from `eval_case_results` to `runs` (STA-415, PRD STA-393).
+--
+-- `eval_case_results.run_id` REFERENCES `runs (id) ON DELETE SET NULL`, and
+-- Postgres enforces that by running the equivalent of
+--   UPDATE eval_case_results SET run_id = NULL WHERE run_id = $1
+-- inside EVERY transaction that deletes a `runs` row. With no index on the
+-- referencing column that is a full sequential scan of what becomes the largest
+-- table this feature creates (cases x repeats x variants x runs, never pruned —
+-- 0038 says out loud that a verdict must outlive its run).
+--
+-- The cost lands on code that has nothing to do with evals: the workflow
+-- concurrency gate deletes a single `waiting` run row on every admission
+-- (`clearWaitingRun` / `clearOldestWaitingRun`), so admission latency would grow
+-- with the accumulated eval history of a box that merely ran nightly evals.
+--
+-- The repo already gets this right for the two other foreign keys of the same
+-- shape — `approvals_run_idx` next to 0002's ON DELETE SET NULL, and
+-- `run_steps_run_started_idx` next to 0018's cascade. This is the one that was
+-- left bare. `eval_case_results_run_idx` (0038) reads like it covers this, but it
+-- indexes `eval_run_id` — the EVAL run, not the platform run.
+--
+-- A separate migration rather than an edit to 0038: an applied migration's
+-- checksum is verified on every boot, so rewriting it would brick a box that
+-- already ran it.
+CREATE INDEX IF NOT EXISTS eval_case_results_platform_run_idx
+    ON stackbone_platform.eval_case_results (run_id);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackbone/cli",
-  "version": "0.2.7",
+  "version": "0.2.10",
   "license": "UNLICENSED",
   "type": "module",
   "bin": {
@@ -26,6 +26,7 @@
     "drizzle-kit": "^0.31.10",
     "drizzle-orm": "^0.45.2",
     "hono": "^4.12.15",
+    "jose": "^6.2.3",
     "libsodium-wrappers": "^0.8.4",
     "open": "^11.0.0",
     "openai": "^6.42.0",