@stackbone/cli 0.2.10 → 0.3.0

package/migrations/0041_eval_criterion_workflow_name.sql

@@ -0,0 +1,68 @@
+-- 0041_eval_criterion_workflow_name.sql — write the name that used to be implicit
+-- onto every stored `custom_workflow` criterion (STA-521, PRD STA-393).
+--
+-- The criterion "judged by your own workflow" used to resolve ONE reserved workflow
+-- name and store nothing about it: the role WAS the workflow name, so a suite only
+-- had to say "custom_workflow" and the box knew where to go. STA-521 makes the
+-- judging workflow a setting picked from a dropdown of what the box holds, which
+-- means the name is now something a suite has to CARRY.
+--
+-- WHY THIS SHIPS IN THE SAME SLICE AS THE CATALOG CHANGE, not after it. The catalog
+-- rejects settings it does not declare, so a backfill landing FIRST would break
+-- every subsequent save; and once `workflowName` is required, an unmigrated suite
+-- can neither be saved nor judged. Either order on its own opens a window in which
+-- a stored suite is broken. Both halves land together, so there is no window.
+--
+-- THE LITERAL BELOW MUST MATCH `EVAL_CRITERION_ROLE` in
+-- libs/shared/evals/src/lib/custom/custom-criterion.ts. SQL cannot import the
+-- constant, and that constant now exists for exactly this reason: it is the value
+-- every suite meant while the name was implicit, and both sides have to mean the
+-- same thing by it. The TS side keeps a comment pointing back here.
+--
+-- ONLY WHERE THE SETTING IS ABSENT OR BLANK. A box that already had the picker has
+-- an operator's own choice in that field; overwriting it would silently re-point
+-- their judge at a workflow they never picked, and nothing on the screen would say
+-- so. `COALESCE(... ->> 'workflowName', '') = ''` covers all three ways of having
+-- said nothing — the key missing, a JSON null, and an empty string — and makes the
+-- statement idempotent: run it twice and the second pass matches nothing.
+--
+-- `scorers` is a jsonb ARRAY of `{type, params}`, so the rewrite goes element by
+-- element through `jsonb_array_elements` and reassembles the list with `jsonb_agg`.
+-- Element ORDER is preserved through `WITH ORDINALITY` … `ORDER BY`, because a
+-- criterion list is read positionally on the screen; the co-located spec pins that
+-- against a suite whose criterion is neither first nor alone. The `COALESCE(…,
+-- '[]')` around the aggregate is belt-and-braces only — `jsonb_agg` over no rows
+-- returns NULL, and the `EXISTS` guard below already means no row without criteria
+-- is ever updated. It stays because writing NULL into a NOT NULL column is the one
+-- mistake here that would take a boot down.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+UPDATE stackbone_platform.eval_suites AS s
+   SET scorers = (
+       SELECT COALESCE(jsonb_agg(rewritten.scorer ORDER BY rewritten.position), '[]'::jsonb)
+         FROM (
+              SELECT element.position,
+                     CASE
+                       WHEN element.scorer->>'type' = 'custom_workflow'
+                        AND COALESCE(element.scorer->'params'->>'workflowName', '') = ''
+                       THEN jsonb_set(
+                              element.scorer,
+                              '{params}',
+                              COALESCE(element.scorer->'params', '{}'::jsonb)
+                                || jsonb_build_object('workflowName', 'eval-criterion')
+                            )
+                       ELSE element.scorer
+                     END AS scorer
+                FROM jsonb_array_elements(s.scorers)
+                     WITH ORDINALITY AS element(scorer, position)
+              ) AS rewritten
+       )
+ WHERE jsonb_typeof(s.scorers) = 'array'
+   AND EXISTS (
+       SELECT 1
+         FROM jsonb_array_elements(s.scorers) AS scorer
+        WHERE scorer->>'type' = 'custom_workflow'
+          AND COALESCE(scorer->'params'->>'workflowName', '') = ''
+       );

package/migrations/0042_eval_case_results_ended.sql

@@ -0,0 +1,64 @@
+-- 0042_eval_case_results_ended.sql — record HOW a measured case ended, next to
+-- the free-form answer the verdict already keeps (STA-526, PRD STA-393).
+--
+-- 0038's own comment already promised this: "a case stopped by a guardrail or
+-- parked on a human approval is distinguishable from one that simply answered
+-- badly". It was only ever true by convention — the ending was a TypeScript
+-- union translated away before the insert, and the sole trace left behind was a
+-- string inside the `error` blob. Studio then had to guess the verdict by
+-- matching the front of that string, which is how a parked workflow ended up
+-- painted as an error.
+--
+-- A COLUMN rather than a code inside `error`, for two reasons:
+--   - an agent that correctly ESCALATED is not an error, and archiving it as one
+--     makes a suite of held cases read as a broken box;
+--   - "3 of 15 escalated" has to be one query, not a JSON read per row.
+--
+-- THE SIX VALUES, and why the two parks are separate:
+--   completed             reached a terminal state with an answer
+--   escalated             parked on an operator approval
+--   asked_user            parked on a resume hook, waiting on a person
+--   blocked_by_guardrail  refused at the border, never started
+--   errored               broke
+--   cancelled             stopped before it could finish
+-- `escalated` and `asked_user` are NOT folded together: telling "escalated to a
+-- human" from "asked the end user" is the whole distinction the pause criteria
+-- are being built to measure.
+--
+-- NULLABLE, and deliberately never backfilled. This is an APPEND-ONLY results
+-- table: every verdict written before today belongs to a sealed run, and there
+-- is no honest value to invent for it — `completed` would put words in that
+-- run's mouth. A NULL reads as "this run could not say", which is the truth.
+-- It also keeps a rolling upgrade safe: a box that applies this migration a
+-- moment before its runners learn to write the column still writes verdicts,
+-- rather than failing every one of them on a NOT NULL.
+--
+-- A CHECK and not free text, unlike `runs.entity_kind` (0037) or the suite's
+-- `target_kind`. Those two are discriminators a box on an older build must be
+-- able to STORE without executing, so rejecting an unknown value would lose an
+-- operator's configuration. This column is the opposite: the box WRITES it, one
+-- of six values decided in this repository, and a screen that groups on it must
+-- not have to defend against a seventh spelling invented by a typo.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD COLUMN IF NOT EXISTS ended text;
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_case_results
+    DROP CONSTRAINT IF EXISTS eval_case_results_ended_check;
+
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD CONSTRAINT eval_case_results_ended_check
+    CHECK (ended IS NULL OR ended IN (
+        'completed',
+        'escalated',
+        'asked_user',
+        'blocked_by_guardrail',
+        'errored',
+        'cancelled'
+    ));

package/migrations/0043_eval_suite_approval_policy.sql

@@ -0,0 +1,50 @@
+-- 0043_eval_suite_approval_policy.sql — what a suite does about an approval one
+-- of its cases parks on (STA-530, PRD STA-393).
+--
+-- A measured case has NOBODY waiting on it. The thread belongs to one attempt of
+-- one case and is never spoken to again, so an agent that correctly stops to ask
+-- an operator is scored on the stop and everything it would have said afterwards
+-- is never seen. Half of what an approval-bearing agent does is on the other side
+-- of that gate, and today a suite cannot measure any of it.
+--
+-- THE FOUR VALUES:
+--   leave_waiting   the case closes on the park and the request it raised is
+--                   taken back out of the inbox. What the product did before this
+--                   column existed.
+--   reject          answer it no, then keep watching the turn.
+--   approve         answer it yes, then keep watching the turn.
+--   model_decides   ask the judging model which of the two, then answer it.
+--
+-- NOT NULL with a DEFAULT, unlike 0042's `ended`. The opposite situation: that
+-- one is an append-only results column with no honest value to invent for a row
+-- written before it existed, while this is CONFIGURATION and the honest value for
+-- every suite already stored is precisely what they do today — leave the case
+-- waiting. Backfilling it is therefore not putting words in anybody's mouth.
+--
+-- A CHECK and not free text, for the same reason 0042's ending carries one: the
+-- BOX writes and reads this, one of four values decided in this repository, and
+-- the runner that branches on it must not have to defend against a fifth spelling
+-- invented by a typo. The suite's own `target_kind` is deliberately open text by
+-- contrast — that one is a discriminator a box on an older build has to be able
+-- to store without executing.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+ALTER TABLE stackbone_platform.eval_suites
+    ADD COLUMN IF NOT EXISTS approval_policy text NOT NULL DEFAULT 'leave_waiting';
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_suites
+    DROP CONSTRAINT IF EXISTS eval_suites_approval_policy_check;
+
+ALTER TABLE stackbone_platform.eval_suites
+    ADD CONSTRAINT eval_suites_approval_policy_check
+    CHECK (approval_policy IN (
+        'leave_waiting',
+        'reject',
+        'approve',
+        'model_decides'
+    ));

package/migrations/0044_eval_persona_simulator.sql

@@ -0,0 +1,88 @@
+-- 0044_eval_persona_simulator.sql — a case carries a persona, and the suite
+-- decides whether anybody plays it (STA-532, PRD STA-393).
+--
+-- The clarification loop is the part of the product no eval touched. An agent
+-- that asks before it acts parks on the first question, so what it would have
+-- done with the answer — the whole reason it asks at all — was never measured.
+-- 0043 opened the half of that story that faces an OPERATOR; this one opens the
+-- half that faces the END USER.
+--
+-- FOUR TABLES, and each column is on the table that owns the fact:
+--
+--   eval_cases.persona              WHO answers, in what words, how many times.
+--                                   Belongs to the case, so the same list can be
+--                                   run by a suite that measures the first
+--                                   question and one that measures the whole
+--                                   conversation.
+--   eval_suites.simulator           WHETHER anybody plays it, and with which
+--                                   model. Belongs to the suite, so a persona
+--                                   with the simulator off is simply ignored.
+--   eval_runs.simulator_model       WHO ACTUALLY played it, sealed at launch.
+--   eval_runs.reply_cap             HOW MANY TIMES they were allowed to.
+--   eval_runs.simulated_connectors  WHICH connectors were stood in for.
+--   eval_case_results.simulation    WHAT was said, turn by turn, marking which
+--                                   lines were scripted and which improvised.
+--
+-- WHY THE RUN SEALS THREE MORE THINGS. Two runs of the same suite a month apart
+-- have to stay comparable, and every one of these moves a score without the agent
+-- changing at all: a different model plays the user differently, a cap raised
+-- from three to five gives the agent two more chances, and a connector that was
+-- real last month and simulated today is a different world. A run that did not
+-- record them makes the next comparison read a change of setting as a change in
+-- the agent.
+--
+-- A NOTE ON GRANULARITY, so it is not rediscovered as a surprise. `git_sha` and
+-- `agent_version` on this table are already taken from the FIRST variant only,
+-- while a run may carry two. The three columns below are added at the SAME single
+-- value per run, deliberately — a per-variant seal for these alone would leave
+-- the row half one shape and half another. When the variant scoping is fixed it
+-- has to be fixed for all five together.
+--
+-- JSONB rather than columns for `persona` and `simulator`, matching `scorers` and
+-- `variants` beside them: they are configuration payloads whose documented
+-- defaults live in `@stackbone/validators` → `studio-evals.ts`, and a CHECK here
+-- would be a second description that drifts from the form an operator fills in.
+--
+-- `simulated_connectors` is `text[]` rather than jsonb, matching `eval_cases.tags`
+-- — it is a flat list of identifiers, and the array type is what makes "which
+-- runs simulated Outlook?" an ordinary query.
+--
+-- NULLABLE where the fact is HISTORICAL and defaulted where it is CONFIGURATION,
+-- exactly as 0042 and 0043 split it. A verdict written before this column existed
+-- has no honest simulation to invent, and a run sealed before it has no honest
+-- model to name — those read NULL, which is the truth. A suite, by contrast, has
+-- a perfectly honest current value: nobody answers, which is what every stored
+-- suite does today.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+
+ALTER TABLE stackbone_platform.eval_cases
+    ADD COLUMN IF NOT EXISTS persona jsonb;
+
+ALTER TABLE stackbone_platform.eval_suites
+    ADD COLUMN IF NOT EXISTS simulator jsonb NOT NULL
+    DEFAULT '{"enabled": false, "model": null}'::jsonb;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS simulator_model text;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS reply_cap integer;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD COLUMN IF NOT EXISTS simulated_connectors text[] NOT NULL DEFAULT '{}';
+
+ALTER TABLE stackbone_platform.eval_case_results
+    ADD COLUMN IF NOT EXISTS simulation jsonb;
+
+-- Added separately from the column so the statement above stays re-runnable:
+-- `ADD COLUMN IF NOT EXISTS ... CHECK (...)` would skip the constraint entirely
+-- on a box where the column somehow already exists.
+ALTER TABLE stackbone_platform.eval_runs
+    DROP CONSTRAINT IF EXISTS eval_runs_reply_cap_check;
+
+ALTER TABLE stackbone_platform.eval_runs
+    ADD CONSTRAINT eval_runs_reply_cap_check
+    CHECK (reply_cap IS NULL OR reply_cap >= 0);

package/migrations/0045_eval_case_description.sql

@@ -0,0 +1,34 @@
+-- 0045_eval_case_description.sql — a case says, in prose, what it is for.
+
+-- A case list is read far more often than it is written, and until now the only
+-- thing a reader got was the `name`. A name has to be short and unique, so it
+-- ends up as a slug — `prof-asunto-y-cuerpo-discrepan` — and everything that
+-- makes the case worth having lives in whoever wrote it.
+--
+-- WHERE THIS WAS BEING KEPT INSTEAD, and why that is the actual argument for the
+-- column. Case lists in the wild already carry the explanation; they just have
+-- nowhere to put it, so it goes inside `expected` as an extra key. That field is
+-- the REFERENCE ANSWER — the thing scorers compare against — so an explanation
+-- parked there is prose sitting inside the ground truth, surviving only because
+-- each scorer is taught to skip that one key by name. A scorer that compares the
+-- whole object (`exact-match` does) fails every case that has one.
+--
+-- So this is not a new capability so much as moving a fact onto the row that owns
+-- it, which leaves `expected` holding only what is actually compared.
+--
+-- `text` and NULLABLE, not `NOT NULL DEFAULT ''`:
+--
+--   * Every case stored before today genuinely has no description, and NULL is
+--     what "nobody wrote one" looks like. A default of '' would claim all of them
+--     had been described with an empty sentence, and a screen cannot tell that
+--     apart from a description somebody cleared on purpose.
+--   * No length CHECK, matching `eval_cases.name` beside it: the bound is a form
+--     concern and lives with the form, in `@stackbone/validators` →
+--     `studio-evals.ts`. A number repeated here is a second bound that drifts.
+--
+-- A separate file rather than an edit to 0038: the migrator hashes each body and
+-- verifies it on every boot, so rewriting a landed migration bricks a box that
+-- already ran it. Comments count towards that hash too.
+
+ALTER TABLE stackbone_platform.eval_cases
+    ADD COLUMN IF NOT EXISTS description text;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stackbone/cli",
-  "version": "0.2.10",
+  "version": "0.3.0",
   "license": "UNLICENSED",
   "type": "module",
   "bin": {