@rulemetric/local 0.12.8 → 0.14.0

package/dist/supabase/migrations/00224_measurement_job_snapshots.sql

@@ -0,0 +1,246 @@
+-- Immutable selection for NEW jobs only. Existing payloads and all producer
+-- eligibility predicates from 00210 remain unchanged.
+create or replace function public.measurement_execution_for_user(owner_id uuid, workload text)
+returns jsonb
+language plpgsql stable
+set search_path = public
+as $$
+declare preference jsonb;
+begin
+  if workload not in ('eval', 'harbor') then
+    raise exception 'Unknown measurement workload: %', workload;
+  end if;
+  select measurement_preferences into preference from public.profiles where id = owner_id;
+  if preference is null or preference->>'engine' is null then return null; end if;
+  return jsonb_build_object(
+    'version', 1,
+    'engine', preference->>'engine',
+    'model', coalesce(nullif(btrim(preference->>'model'), ''),
+                      case when workload = 'harbor' then 'claude-haiku-4-5' else null end),
+    'source', 'account'
+  );
+end;
+$$;
+revoke all on function public.measurement_execution_for_user(uuid, text) from public, anon, authenticated;
+grant execute on function public.measurement_execution_for_user(uuid, text) to service_role;
+
+-- Keep a canonical model attached to the engine that supplied its evidence.
+-- Explicit historical model-only pins still mean Claude; inferred GPT judges
+-- must not acquire that legacy default on their second scheduled batch.
+create or replace function public.eval_target_judge_engine(p_target_id uuid)
+returns text
+language sql stable security definer
+set search_path = public
+as $$
+  with target as (
+    select nullif(btrim(metadata->>'graderModel'), '') as pin,
+      coalesce(metadata->>'graderEngine', 'claude') as pin_engine,
+      public.eval_target_judge_segment(id) as elected_model
+    from eval_targets where id = p_target_id
+  ), elector as (
+    select r.grading->>'judge_model' as model,
+      coalesce(r.grading->>'judge_engine', 'claude') as engine, r.created_at,
+      (r.grading->>'judge_model_requested' is null
+        or r.grading->>'judge_model_requested' = r.grading->>'judge_model') as clean,
+      case when jsonb_typeof(r.grading->'expectations') = 'array' then (
+        select count(*) from jsonb_array_elements(r.grading->'expectations') e
+        where coalesce(e->>'method', 'llm') = 'llm'
+          and coalesce(e->>'errored', 'false') <> 'true'
+      ) else 0 end as llm_total,
+      (select count(*) from eval_grade_annotations a
+        where a.eval_target_id = p_target_id and a.eval_run_id = r.id
+          and a.judge_method = 'llm') as llm_annotated
+    from eval_runs r
+    where r.eval_target_id = p_target_id and r.status = 'completed'
+      and r.configuration in ('with_target', 'without_target')
+      and r.grading->>'judge_model' is not null
+  ), eligible as (
+    select e.* from elector e
+    where (e.clean or not exists (select 1 from elector where clean))
+      and e.model = (select elected_model from target)
+  )
+  select case when t.pin is not null then t.pin_engine else (
+    select engine from eligible group by engine
+    order by count(*) filter (where llm_total > 0 and llm_annotated >= llm_total) desc,
+      max(created_at) desc, engine desc limit 1
+  ) end from target t;
+$$;
+revoke all on function public.eval_target_judge_engine(uuid) from public, anon, authenticated;
+grant execute on function public.eval_target_judge_engine(uuid) to service_role;
+
+-- ── enqueue_instruction_evolution, health gates now read from the view ───────
+create or replace function public.enqueue_instruction_evolution()
+returns integer
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+  queued integer;
+begin
+  with health_ok as (
+    select t.id, t.user_id
+    from eval_targets t
+    where coalesce(t.metadata->>'autoEvolvePaused', 'false') <> 'true'
+      -- Health 2a/2b/2c: ONE definition (00210), shared with eval autorun.
+      and exists (
+        select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
+      )
+      -- Live cases to measure against — at least MIN_LIVE_CASES (3).
+      -- `exists` (>= 1) was the bug: see 00183's header.
+      and (
+        select count(*) from evals ev
+        where ev.eval_target_id = t.id and ev.retired_at is null
+      ) >= 3
+      -- Change budget (cheap exclusion; the API is the authority).
+      and (
+        select count(*)
+        from instruction_promotions p
+        where p.eval_target_id = t.id
+          and p.applied
+          and p.created_at > now() - interval '7 days'
+      ) < 3
+      -- One in flight at a time.
+      and not exists (
+        select 1 from agent_jobs j
+        where j.task_kind = 'cron_instruction_evolution'
+          and j.status in ('pending', 'claimed', 'running')
+          and j.payload->>'evalTargetId' = t.id::text
+      )
+  ),
+  manual as (
+    -- The hand-picked path, unchanged and uncapped.
+    select h.id, h.user_id
+    from health_ok h
+    join eval_targets t on t.id = h.id
+    where coalesce(t.metadata->>'autoEvolveEnabled', 'false') = 'true'
+  ),
+  auto_ranked as (
+    -- The stated rule. Round-robin: never-decided first (nulls first), then
+    -- least-recently-decided, so the cap rotates through the candidate pool
+    -- instead of re-measuring the same winners nightly.
+    select h.id, h.user_id,
+           row_number() over (
+             partition by h.user_id
+             order by (
+               select max(p.created_at)
+               from instruction_promotions p
+               where p.eval_target_id = h.id
+             ) asc nulls first,
+             h.id
+           ) as rn
+    from health_ok h
+    join eval_targets t on t.id = h.id
+    where t.type = 'instruction'
+      and coalesce(t.metadata->>'autoEvolveOptOut', 'false') <> 'true'
+      -- Not already covered by the manual path.
+      and coalesce(t.metadata->>'autoEvolveEnabled', 'false') <> 'true'
+      -- ACTIVE (00145's load-bearing gate): a run in the last 30 days.
+      and exists (
+        select 1 from eval_runs r
+        where r.eval_target_id = t.id
+          and r.created_at > now() - interval '30 days'
+      )
+  ),
+  eligible as (
+    select id, user_id from manual
+    union
+    select id, user_id from auto_ranked where rn <= 3
+  ),
+  ins as (
+    insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
+    select e.user_id,
+           'cron_instruction_evolution',
+           'pending',
+           jsonb_build_object('evalTargetId', e.id::text)
+             || case
+                  when public.eval_target_judge_segment(e.id) is not null
+                    then jsonb_build_object(
+                           'graderModel', public.eval_target_judge_segment(e.id),
+                           'graderEngine', public.eval_target_judge_engine(e.id))
+                  else '{}'::jsonb
+                end
+             || case when public.measurement_execution_for_user(e.user_id, 'eval') is null
+                     then '{}'::jsonb
+                     else jsonb_build_object('measurement', public.measurement_execution_for_user(e.user_id, 'eval')) end,
+           'instruction-evolution:' || e.id::text
+    from eligible e
+    returning 1
+  )
+  select count(*)::int into queued from ins;
+
+  return queued;
+end;
+$$;
+
+comment on function public.enqueue_instruction_evolution() is
+  'Daily producer for cron_instruction_evolution. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); pause, live-cases floor (>=3, 00183), change budget, round-robin enrolment and in-flight dedupe stay here. Judge segment from eval_target_judge_segment() (00193).';
+
+-- ── enqueue_eval_autorun, health gates now read from the view ────────────────
+create or replace function public.enqueue_eval_autorun()
+returns integer
+language plpgsql
+security definer
+set search_path = public
+as $$
+declare
+  queued integer;
+begin
+  with eligible as (
+    select t.id, t.user_id,
+      -- Canonical segment (00193). Was an inline "most recent stamped run"
+      -- subquery, which is how a credit-exhaustion fallback became the pin.
+      public.eval_target_judge_segment(t.id) as current_judge,
+      public.eval_target_judge_engine(t.id) as current_judge_engine
+    from eval_targets t
+    where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
+      -- Gate 0 (00190): the target must be anchored to something a promotion
+      -- could land on. Without this, a batch is two LLM executions plus a
+      -- judged grade spent to rewrite a row nothing reads.
+      and (
+        t.instruction_id is not null
+        or t.project_path is not null
+        or t.name like '/%'
+      )
+      -- Gates 2a/2b/2c: ONE definition (00210), shared with evolution.
+      and exists (
+        select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
+      )
+      -- Gate 3: the target has live cases to actually run.
+      and exists (
+        select 1 from evals ev
+        where ev.eval_target_id = t.id and ev.retired_at is null
+      )
+      -- One in flight at a time, per the 00083/00086 pattern.
+      and not exists (
+        select 1 from agent_jobs j
+        where j.task_kind = 'cron_eval_autorun'
+          and j.status in ('pending', 'claimed', 'running')
+          and j.payload->>'evalTargetId' = t.id::text
+      )
+  ), ins as (
+    insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
+    select e.user_id,
+           'cron_eval_autorun',
+           'pending',
+           jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
+             || case
+                  when e.current_judge is not null
+                    then jsonb_build_object('graderModel', e.current_judge, 'graderEngine', e.current_judge_engine)
+                  else '{}'::jsonb
+                end
+             || case when public.measurement_execution_for_user(e.user_id, 'eval') is null
+                     then '{}'::jsonb
+                     else jsonb_build_object('measurement', public.measurement_execution_for_user(e.user_id, 'eval')) end,
+           'eval-autorun:' || e.id::text
+    from eligible e
+    returning 1
+  )
+  select count(*)::int into queued from ins;
+
+  return queued;
+end;
+$$;
+
+comment on function public.enqueue_eval_autorun() is
+  'Daily producer for cron_eval_autorun. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); the anchor gate (00190), live-cases exists, judge segment (00193) and in-flight dedupe stay here.';