@rulemetric/local 0.10.0 → 0.12.0

package/dist/supabase/migrations/00190_eval_targets_artifact_link.sql

@@ -0,0 +1,176 @@
+-- Give an eval target a real link to the artifact it measures.
+--
+-- THE STRUCTURAL DEFECT
+--
+-- `eval_targets` stored no reference to the thing under test: no
+-- `instruction_id`, no project. Just a COPY of the content and a `name` string.
+-- "Which artifact does this target measure?" was answered by parsing the name —
+-- if it starts with '/', treat it as a file path — a convention enforced
+-- nowhere that the actuator nonetheless depends on
+-- (`cron-instruction-evolution.ts`: `target.name.startsWith('/') && existsSync`).
+--
+-- When the name is not a path, the loop still measures the target and still
+-- "promotes" it; the promotion writes back to `eval_targets.content`, the
+-- target's own copy, which nothing downstream reads. Measured 2026-08-15:
+-- `TypeScript Best Practices` — a 129-character demo row seeded 2026-03-10 —
+-- was one of only 2 autorun-enabled targets out of 251, write-enabled, and had
+-- burned 102 eval runs producing edits no session could ever load.
+--
+-- The link was available and discarded. `sessions/linking.ts` auto-creates
+-- targets from captured instructions (169 of 197 instruction targets came from
+-- there); it SELECTs `instructions` by `inst.id` to read the content, then
+-- stores only `inst.name`. The session — and therefore the project — is in
+-- scope in the same function.
+--
+-- WHY project_path RATHER THAN project_id
+--
+-- The trial half of the system already models this correctly:
+-- `instruction_experiments` carries `instruction_id` + `project_path`, and its
+-- verdicts (`instruction_verdicts`) carry the same pair. Skills are globally
+-- defined (scope 'public') but their EFFECTIVENESS is per-project — the same
+-- skill helps in one repo and not another. Matching that table's shape keeps
+-- the two measurement modes joinable on the same keys. (Modes A and B are
+-- complementary, not redundant: a repo file cannot be withheld from real work,
+-- so CLAUDE.md is Mode A only, while a skill can be, so it is trialled.)
+--
+-- Both columns are NULLABLE and nothing is backfilled. 249 of 251 targets are
+-- not enrolled in anything; guessing their provenance from a name would be
+-- inventing data. The gate below applies only where promotions can happen.
+
+alter table public.eval_targets
+  add column if not exists instruction_id uuid references public.instructions(id) on delete set null,
+  add column if not exists project_path text;
+
+comment on column public.eval_targets.instruction_id is
+  'The instruction this target measures. Null for targets whose provenance was never recorded (everything created before 00190) and for prompt-type targets captured from a session. Populated at auto-creation, where the id is already in hand.';
+
+comment on column public.eval_targets.project_path is
+  'Project the target is measured FOR. Mirrors instruction_experiments.project_path so the eval (Mode A) and trial (Mode B) halves join on the same keys. A public skill is global; its effectiveness is not.';
+
+create index if not exists idx_eval_targets_instruction
+  on public.eval_targets (instruction_id) where instruction_id is not null;
+
+create index if not exists idx_eval_targets_project
+  on public.eval_targets (project_path) where project_path is not null;
+
+-- ---------------------------------------------------------------------------
+-- The gate, at the producer — the same place 00188 gated the training producer
+-- on the canonical checkout, and for the same reason: enrolment is a manual
+-- metadata toggle, so the producer is the only chokepoint every scheduled batch
+-- must pass through.
+--
+-- A target may only consume the autorun budget if the loop can name what a
+-- promotion would land on. Two ways to qualify:
+--
+--   (a) it carries a link (instruction_id or project_path), or
+--   (b) it is named by an absolute path — the legacy convention, kept so the
+--       163 file-anchored targets that predate this migration keep working
+--       without a backfill.
+--
+-- A target with neither is refused. That is exactly the demo row: no
+-- instruction, no project, and a name that is a label rather than a path.
+-- ---------------------------------------------------------------------------
+
+create or replace function public.enqueue_eval_autorun()
+returns integer
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+  queued integer;
+begin
+  with eligible as (
+    select t.id, t.user_id,
+      -- Current judge for this target = most recent stamped run, mirroring
+      -- judge-calibration's own resolution (created_at desc, model desc for a
+      -- stable tie-break).
+      (
+        select r.grading->>'judge_model'
+        from eval_runs r
+        where r.eval_target_id = t.id
+          and r.grading->>'judge_model' is not null
+        order by r.created_at desc, r.grading->>'judge_model' desc
+        limit 1
+      ) as current_judge
+    from eval_targets t
+    where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
+      -- Gate 0 (00190): the target must be anchored to something a promotion
+      -- could land on. Without this, a batch is two LLM executions plus a
+      -- judged grade spent to rewrite a row nothing reads.
+      and (
+        t.instruction_id is not null
+        or t.project_path is not null
+        or t.name like '/%'
+      )
+      -- Gate 2a: recent grading wasn't MOSTLY ungradeable (proportional, and
+      -- identical to the evolution producer's 2a since 00163).
+      and not exists (
+        select 1 from eval_runs r
+        where r.eval_target_id = t.id
+          and r.created_at > now() - interval '7 days'
+        group by r.eval_target_id
+        having count(*) >= 5
+           and count(*) filter (
+                 where jsonb_typeof(r.grading->'expectations') = 'array'
+                   and exists (
+                     select 1 from jsonb_array_elements(r.grading->'expectations') e
+                     where coalesce(e->>'errored', 'false') = 'true'
+                   )
+               ) * 4 >= count(*)
+      )
+      -- Gate 2b: no human anchors pointing at an ungradeable assertion.
+      and not exists (
+        select 1
+        from eval_grade_annotations a
+        join eval_runs r on r.id = a.eval_run_id
+        where a.eval_target_id = t.id
+          and jsonb_typeof(r.grading->'expectations') = 'array'
+          and coalesce(
+            r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
+          ) = 'true'
+      )
+      -- Gate 2c: last week's batch wasn't mostly failures.
+      and not exists (
+        select 1 from eval_runs r
+        where r.eval_target_id = t.id
+          and r.created_at > now() - interval '7 days'
+        group by r.eval_target_id
+        having count(*) >= 5
+           and count(*) filter (where r.status = 'failed') * 4 >= count(*)
+      )
+      -- Gate 3: the target has live cases to actually run.
+      and exists (
+        select 1 from evals ev
+        where ev.eval_target_id = t.id and ev.retired_at is null
+      )
+      -- One in flight at a time, per the 00083/00086 pattern.
+      and not exists (
+        select 1 from agent_jobs j
+        where j.task_kind = 'cron_eval_autorun'
+          and j.status in ('pending', 'claimed', 'running')
+          and j.payload->>'evalTargetId' = t.id::text
+      )
+  ), ins as (
+    insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
+    select e.user_id,
+           'cron_eval_autorun',
+           'pending',
+           jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
+             || case
+                  when e.current_judge is not null
+                    then jsonb_build_object('graderModel', e.current_judge)
+                  else '{}'::jsonb
+                end,
+           'eval-autorun:' || e.id::text
+    from eligible e
+    returning 1
+  )
+  select count(*)::int into queued from ins;
+
+  return queued;
+end;
+$$;
+
+comment on function public.enqueue_eval_autorun() is
+  'Nightly producer for cron_eval_autorun. Enqueues one bounded depth batch per OPTED-IN, ANCHORED, healthy, under-powered eval target, with the grader pinned to the target''s current judge (00148, #196). Gate 0 (00190) requires instruction_id, project_path, or an absolute-path name: a target anchored to nothing can be measured and "promoted" forever while rewriting only its own content column — measured 2026-08-15, a 129-char demo row had burned 102 runs that way. Health 2a is PROPORTIONAL as of 00169. Opt in with: update eval_targets set metadata = coalesce(metadata, ''{}''::jsonb) || ''{"autoRunEnabled":true}''::jsonb where id = ...';

package/dist/supabase/migrations/00191_recommendation_applications_effect_ledger.sql

@@ -0,0 +1,106 @@
+-- Turn `recommendation_applications` into the effect ledger.
+--
+-- 00187 shipped this table with the apply → measure → revert lifecycle, the
+-- before/after hashes, and a CAS transition guard — and then nothing ever wrote
+-- a row (0 in prod as of 2026-08-16). The multi-artifact actuator plan
+-- specified a second table, `applied_artifacts`, for the same job. Two empty
+-- ledgers and two dead read surfaces is the outcome nobody wanted, so the
+-- columns that plan needed are added here instead.
+--
+-- What was missing, and why each one is load-bearing:
+--
+--   boundary_class  Whether the actuator owned its targets. `inside` means an
+--                   exact inverse was possible and is therefore required;
+--                   `outside` means the user/git/another tool also writes there,
+--                   so the row owes a compensation instead. Declared per
+--                   artifact rather than rediscovered per actuator.
+--
+--   targets         EVERY path the apply path wrote. `target_path` (00187) holds
+--                   one, which is fine for a single-file recommendation and
+--                   wrong for a skill that lands in two locations. `target_path`
+--                   stays as the primary target so the existing unique index and
+--                   its callers keep working; `targets` is the complete set.
+--
+--   inverse         How to undo this row, RECORDED AT WRITE TIME. The system's
+--                   other undos are reconstructed — `removeHookByCommand`
+--                   searches a settings file for a command substring, and skill
+--                   retraction searches six directories by name. A search is not
+--                   an inverse: it misses silently when the thing was renamed
+--                   and removes the wrong entry when it matches twice.
+--
+--   requires        The coeffect declaration. Two rules make it load-bearing:
+--   provides        do not apply a dependent before its dependency is present,
+--                   and do not withdraw a provider while an applied dependent
+--                   names it. The second closes a live hole — archiving is a
+--                   soft delete, so archiving a skill today leaves every
+--                   instruction naming it pointing at nothing, silently.
+--
+--   apply_batch_id  The ordering. Not decoration: inverses must run LIFO,
+--   apply_seq       because a batch can install a provider and its dependent
+--                   moments apart, and undoing in application order withdraws
+--                   the provider while the dependent is still live. It is also
+--                   the only thing that can answer "undo everything last
+--                   night's loop applied", a question that currently has no
+--                   answer for any artifact type.
+--
+-- Every column is nullable or defaulted: the 00187 rows that do not exist yet,
+-- and any that appear between deploy and this migration, stay readable.
+
+alter table public.recommendation_applications
+  add column if not exists boundary_class text,
+  add column if not exists targets text[] not null default '{}',
+  add column if not exists inverse jsonb,
+  add column if not exists requires text[] not null default '{}',
+  add column if not exists provides text[] not null default '{}',
+  add column if not exists apply_batch_id uuid,
+  add column if not exists apply_seq integer;
+
+do $$
+begin
+  if not exists (
+    select 1 from pg_constraint
+     where conname = 'recommendation_applications_boundary_class_check'
+  ) then
+    alter table public.recommendation_applications
+      add constraint recommendation_applications_boundary_class_check
+      check (boundary_class is null or boundary_class in ('inside', 'outside'));
+  end if;
+end $$;
+
+-- A row that reached `applied` must carry the inverse it recorded at that
+-- moment. Enforced in the database rather than only in the actuator because the
+-- actuator is one of several writers-to-be (CLI accept, worker auto-apply, the
+-- API's own transition endpoint), and "we forgot to record the undo" is not a
+-- failure any of them would notice at the time — it surfaces months later when
+-- someone asks to revert and the row cannot say how.
+--
+-- Statuses before `applied` (proposed, ready) and the terminal `failed` are
+-- exempt: nothing was written, so there is nothing to undo.
+do $$
+begin
+  if not exists (
+    select 1 from pg_constraint
+     where conname = 'recommendation_applications_applied_has_inverse'
+  ) then
+    alter table public.recommendation_applications
+      add constraint recommendation_applications_applied_has_inverse
+      check (
+        status in ('proposed', 'ready', 'failed')
+        or inverse is not null
+      ) not valid;
+  end if;
+end $$;
+
+-- NOT VALID above, validated here: the constraint applies to every future write
+-- immediately, and the validation pass confirms the (currently empty) backlog
+-- without taking an ACCESS EXCLUSIVE lock for the scan.
+alter table public.recommendation_applications
+  validate constraint recommendation_applications_applied_has_inverse;
+
+-- The withdrawal guard's query: "which applied rows require any of these keys?"
+create index if not exists idx_recommendation_applications_requires
+  on public.recommendation_applications using gin (requires);
+
+-- The reversal query: one batch, newest first.
+create index if not exists idx_recommendation_applications_batch
+  on public.recommendation_applications (user_id, apply_batch_id, apply_seq desc);

package/dist/supabase/migrations/00192_cleanup_reaps_memberless_orgs.sql

@@ -0,0 +1,142 @@
+-- cleanup_stale_test_users cannot reach an org whose members are already gone.
+--
+-- The org sweep (00087, unchanged through 00121) elects orgs by joining
+-- `user_organizations` to the set of users it is about to delete:
+--
+--     and exists (select 1 from user_organizations uo
+--                  where uo.org_id = o.id and uo.user_id = any(v_user_ids))
+--
+-- That arm only ever matches while the membership row still exists. Anything
+-- that deletes the user first — the e2e suites' own `deleteTestUser`, an admin
+-- delete, a user deleting their account — cascades `user_organizations` away,
+-- and from that moment the org is invisible to every subsequent sweep. It has no
+-- members, so nobody can see it, join it, or delete it through the product; it
+-- is unreachable and permanent.
+--
+-- Measured in prod 2026-08-16: 30,276 memberless orgs of 30,307 total,
+-- accumulating since 2026-07-12 (30,261 of them named "…'s Workspace", the
+-- signup default; the other 15 are timestamp-suffixed org/team test fixtures).
+-- They hold nothing — 0 projects, 0 sessions, 0 invitations, 0 agent_runs,
+-- 3 test teams.
+--
+-- The fix is a second, independent sweep: an org with zero members, older than
+-- an hour, is dead by construction. It cannot arise from normal use — the
+-- `check_last_owner` trigger refuses to remove an org's final owner, so the only
+-- way to reach zero members is the cascade above — and the hour of grace keeps a
+-- signup that inserts the org before the membership safe.
+--
+-- Note this sweep is deliberately NOT restricted to test-shaped names. The
+-- defect is not "test orgs leak"; it is "an org outlives its last member". A
+-- real user who deletes their account leaves exactly the same corpse, and it
+-- should be reaped for exactly the same reason.
+--
+-- Two other changes, both forced by the first:
+--
+--   * The `v_user_ids is null` early return (00087) skipped everything below it.
+--     A quiet hour with no test users to reap would have skipped the memberless
+--     sweep too, which is the one that has 30k rows of backlog. The early return
+--     is gone; the user sweep is now guarded in place.
+--   * `superadmin_audit_log.target_org_id` is ON DELETE NO ACTION, so it is
+--     nulled for the memberless set before the delete, the same way the existing
+--     path does it for the membered set.
+--
+-- Return signature is unchanged (callers: `crons run cleanup_stale_test_users`).
+-- `deleted_orgs` now counts both sweeps.
+
+drop function if exists public.cleanup_stale_test_users();
+
+create function public.cleanup_stale_test_users()
+returns table(deleted_users int, deleted_orgs int, deleted_smoke_sessions int) as $$
+declare
+  v_user_ids uuid[];
+  v_org_ids uuid[];
+  v_orphan_org_ids uuid[];
+  v_deleted_users int := 0;
+  v_deleted_orgs int := 0;
+  v_deleted_orphan_orgs int := 0;
+  v_deleted_smoke int := 0;
+begin
+  -- Smoke-test session residue (any user). 1h threshold keeps an
+  -- in-flight smoke run safe.
+  delete from sessions
+  where created_at < now() - interval '1 hour'
+    and (
+      external_session_id like '00000000-0000-4000-8000-%'
+      or project_path = '/tmp/smoke-test-project'
+      or metadata->>'clientName' = 'smoke-test'
+    );
+  get diagnostics v_deleted_smoke = row_count;
+
+  -- Candidates: obvious test users older than 1h, excluding the pinned dev
+  -- login. See 00121 for why each arm can't hit a real signup.
+  select array_agg(id) into v_user_ids
+  from auth.users
+  where email != 'nick@rulemetric.test'
+    and created_at < now() - interval '1 hour'
+    and (
+      email ~* '\.(test|example|invalid|localhost|local)$'
+      or email ~* '@(example\.(com|net|org))$'
+      or email ~ '1[0-9]{12}@'
+    );
+
+  if v_user_ids is not null and array_length(v_user_ids, 1) > 0 then
+    -- Orgs whose every member is in the to-delete set. Cascade carve-out in
+    -- check_last_owner lets the user_organizations rows go when the parent
+    -- org is being deleted.
+    select array_agg(o.id) into v_org_ids
+    from organizations o
+    where not exists (
+      select 1 from user_organizations uo
+      where uo.org_id = o.id
+        and uo.user_id != all(v_user_ids)
+    )
+    and exists (
+      select 1 from user_organizations uo
+      where uo.org_id = o.id and uo.user_id = any(v_user_ids)
+    );
+
+    -- 1. NULL test-user references on NO ACTION FKs (preserve content)
+    update instructions set created_by = null where created_by = any(v_user_ids);
+    update instructions set archived_by = null where archived_by = any(v_user_ids);
+    update instruction_versions set created_by = null where created_by = any(v_user_ids);
+    update projects set created_by = null where created_by = any(v_user_ids);
+    update project_instructions set added_by = null where added_by = any(v_user_ids);
+
+    -- 2. Scrub superadmin_audit_log (actor_id is NOT NULL → delete; targets nullable)
+    delete from superadmin_audit_log where actor_id = any(v_user_ids);
+    update superadmin_audit_log set target_user_id = null where target_user_id = any(v_user_ids);
+    if v_org_ids is not null then
+      update superadmin_audit_log set target_org_id = null where target_org_id = any(v_org_ids);
+    end if;
+
+    -- 3. Delete orphan orgs
+    if v_org_ids is not null then
+      delete from organizations where id = any(v_org_ids);
+      get diagnostics v_deleted_orgs = row_count;
+    end if;
+
+    -- 4. Delete users (cascades to profiles, user_organizations, sessions, ...)
+    delete from auth.users where id = any(v_user_ids);
+    get diagnostics v_deleted_users = row_count;
+  end if;
+
+  -- 5. Memberless orgs, whoever emptied them and whenever. This runs on every
+  --    sweep, independent of whether step 4 deleted anyone, because the orgs it
+  --    reaps were emptied by an EARLIER run (or by a delete this function never
+  --    saw) and no later run can otherwise reach them.
+  select array_agg(o.id) into v_orphan_org_ids
+  from organizations o
+  where o.created_at < now() - interval '1 hour'
+    and not exists (select 1 from user_organizations uo where uo.org_id = o.id);
+
+  if v_orphan_org_ids is not null then
+    update superadmin_audit_log set target_org_id = null
+     where target_org_id = any(v_orphan_org_ids);
+    delete from organizations where id = any(v_orphan_org_ids);
+    get diagnostics v_deleted_orphan_orgs = row_count;
+    v_deleted_orgs := v_deleted_orgs + v_deleted_orphan_orgs;
+  end if;
+
+  return query select v_deleted_users, v_deleted_orgs, v_deleted_smoke;
+end;
+$$ language plpgsql security definer set search_path = public, auth;