@rulemetric/local 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/meta.json +2 -2
- package/dist/server.mjs +685 -416
- package/dist/supabase/migrations/00087_cleanup_test_data_fix.sql +1 -1
- package/dist/supabase/migrations/00185_trial_throughput_settings.sql +43 -0
- package/dist/supabase/migrations/00186_canonical_checkout_prefers_live.sql +256 -0
- package/dist/supabase/migrations/00187_recommendation_applications.sql +32 -0
- package/dist/supabase/migrations/00188_training_producer_canonical_path.sql +117 -0
- package/dist/supabase/migrations/00189_limit_obs_window_index.sql +37 -0
- package/dist/supabase/migrations/00190_eval_targets_artifact_link.sql +176 -0
- package/dist/supabase/migrations/00191_recommendation_applications_effect_ledger.sql +106 -0
- package/dist/supabase/migrations/00192_cleanup_reaps_memberless_orgs.sql +142 -0
- package/dist/supabase/migrations/00193_judge_segment_pin_and_anchors.sql +367 -0
- package/dist/supabase/migrations/00194_refine_cases_producer.sql +91 -0
- package/dist/supabase/migrations/00195_skill_usage_views.sql +109 -0
- package/dist/supabase/migrations/00196_retire_unused_skills_producer.sql +102 -0
- package/dist/supabase/migrations/00197_auto_accept_producer_admits_skills.sql +115 -0
- package/dist/supabase/migrations/00198_unstale_non_catalog_suggestions.sql +51 -0
- package/dist/supabase/migrations/00199_auto_accept_runs_every_two_hours.sql +54 -0
- package/dist/supabase/migrations/00200_auto_accept_runs_hourly.sql +42 -0
- package/dist/web/assets/{docs-B_21wyvj.js → docs-KIeJWTb-.js} +1 -1
- package/dist/web/assets/index-CFJatBZ3.css +1 -0
- package/dist/web/assets/{index-OhhQFJmS.js → index-MYof2fnF.js} +54 -54
- package/dist/web/index.html +3 -3
- package/package.json +2 -2
- package/dist/web/assets/index-_M9l6iMX.css +0 -1
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
-- Turn `recommendation_applications` into the effect ledger.
|
|
2
|
+
--
|
|
3
|
+
-- 00187 shipped this table with the apply → measure → revert lifecycle, the
|
|
4
|
+
-- before/after hashes, and a CAS transition guard — and then nothing ever wrote
|
|
5
|
+
-- a row (0 in prod as of 2026-08-16). The multi-artifact actuator plan
|
|
6
|
+
-- specified a second table, `applied_artifacts`, for the same job. Two empty
|
|
7
|
+
-- ledgers and two dead read surfaces is the outcome nobody wanted, so the
|
|
8
|
+
-- columns that plan needed are added here instead.
|
|
9
|
+
--
|
|
10
|
+
-- What was missing, and why each one is load-bearing:
|
|
11
|
+
--
|
|
12
|
+
-- boundary_class Whether the actuator owned its targets. `inside` means an
|
|
13
|
+
-- exact inverse was possible and is therefore required;
|
|
14
|
+
-- `outside` means the user/git/another tool also writes there,
|
|
15
|
+
-- so the row owes a compensation instead. Declared per
|
|
16
|
+
-- artifact rather than rediscovered per actuator.
|
|
17
|
+
--
|
|
18
|
+
-- targets EVERY path the apply path wrote. `target_path` (00187) holds
|
|
19
|
+
-- one, which is fine for a single-file recommendation and
|
|
20
|
+
-- wrong for a skill that lands in two locations. `target_path`
|
|
21
|
+
-- stays as the primary target so the existing unique index and
|
|
22
|
+
-- its callers keep working; `targets` is the complete set.
|
|
23
|
+
--
|
|
24
|
+
-- inverse How to undo this row, RECORDED AT WRITE TIME. The system's
|
|
25
|
+
-- other undos are reconstructed — `removeHookByCommand`
|
|
26
|
+
-- searches a settings file for a command substring, and skill
|
|
27
|
+
-- retraction searches six directories by name. A search is not
|
|
28
|
+
-- an inverse: it misses silently when the thing was renamed
|
|
29
|
+
-- and removes the wrong entry when it matches twice.
|
|
30
|
+
--
|
|
31
|
+
-- requires The coeffect declaration. Two rules make it load-bearing:
|
|
32
|
+
-- provides do not apply a dependent before its dependency is present,
|
|
33
|
+
-- and do not withdraw a provider while an applied dependent
|
|
34
|
+
-- names it. The second closes a live hole — archiving is a
|
|
35
|
+
-- soft delete, so archiving a skill today leaves every
|
|
36
|
+
-- instruction naming it pointing at nothing, silently.
|
|
37
|
+
--
|
|
38
|
+
-- apply_batch_id The ordering. Not decoration: inverses must run LIFO,
|
|
39
|
+
-- apply_seq because a batch can install a provider and its dependent
|
|
40
|
+
-- moments apart, and undoing in application order withdraws
|
|
41
|
+
-- the provider while the dependent is still live. It is also
|
|
42
|
+
-- the only thing that can answer "undo everything last
|
|
43
|
+
-- night's loop applied", a question that currently has no
|
|
44
|
+
-- answer for any artifact type.
|
|
45
|
+
--
|
|
46
|
+
-- Every column is nullable or defaulted: the 00187 rows that do not exist yet,
|
|
47
|
+
-- and any that appear between deploy and this migration, stay readable.
|
|
48
|
+
|
|
49
|
+
alter table public.recommendation_applications
|
|
50
|
+
add column if not exists boundary_class text,
|
|
51
|
+
add column if not exists targets text[] not null default '{}',
|
|
52
|
+
add column if not exists inverse jsonb,
|
|
53
|
+
add column if not exists requires text[] not null default '{}',
|
|
54
|
+
add column if not exists provides text[] not null default '{}',
|
|
55
|
+
add column if not exists apply_batch_id uuid,
|
|
56
|
+
add column if not exists apply_seq integer;
|
|
57
|
+
|
|
58
|
+
do $$
|
|
59
|
+
begin
|
|
60
|
+
if not exists (
|
|
61
|
+
select 1 from pg_constraint
|
|
62
|
+
where conname = 'recommendation_applications_boundary_class_check'
|
|
63
|
+
) then
|
|
64
|
+
alter table public.recommendation_applications
|
|
65
|
+
add constraint recommendation_applications_boundary_class_check
|
|
66
|
+
check (boundary_class is null or boundary_class in ('inside', 'outside'));
|
|
67
|
+
end if;
|
|
68
|
+
end $$;
|
|
69
|
+
|
|
70
|
+
-- A row that reached `applied` must carry the inverse it recorded at that
|
|
71
|
+
-- moment. Enforced in the database rather than only in the actuator because the
|
|
72
|
+
-- actuator is one of several writers-to-be (CLI accept, worker auto-apply, the
|
|
73
|
+
-- API's own transition endpoint), and "we forgot to record the undo" is not a
|
|
74
|
+
-- failure any of them would notice at the time — it surfaces months later when
|
|
75
|
+
-- someone asks to revert and the row cannot say how.
|
|
76
|
+
--
|
|
77
|
+
-- Statuses before `applied` (proposed, ready) and the terminal `failed` are
|
|
78
|
+
-- exempt: nothing was written, so there is nothing to undo.
|
|
79
|
+
do $$
|
|
80
|
+
begin
|
|
81
|
+
if not exists (
|
|
82
|
+
select 1 from pg_constraint
|
|
83
|
+
where conname = 'recommendation_applications_applied_has_inverse'
|
|
84
|
+
) then
|
|
85
|
+
alter table public.recommendation_applications
|
|
86
|
+
add constraint recommendation_applications_applied_has_inverse
|
|
87
|
+
check (
|
|
88
|
+
status in ('proposed', 'ready', 'failed')
|
|
89
|
+
or inverse is not null
|
|
90
|
+
) not valid;
|
|
91
|
+
end if;
|
|
92
|
+
end $$;
|
|
93
|
+
|
|
94
|
+
-- NOT VALID above, validated here: the constraint applies to every future write
|
|
95
|
+
-- immediately, and the validation pass confirms the (currently empty) backlog
|
|
96
|
+
-- without taking an ACCESS EXCLUSIVE lock for the scan.
|
|
97
|
+
alter table public.recommendation_applications
|
|
98
|
+
validate constraint recommendation_applications_applied_has_inverse;
|
|
99
|
+
|
|
100
|
+
-- The withdrawal guard's query: "which applied rows require any of these keys?"
|
|
101
|
+
create index if not exists idx_recommendation_applications_requires
|
|
102
|
+
on public.recommendation_applications using gin (requires);
|
|
103
|
+
|
|
104
|
+
-- The reversal query: one batch, newest first.
|
|
105
|
+
create index if not exists idx_recommendation_applications_batch
|
|
106
|
+
on public.recommendation_applications (user_id, apply_batch_id, apply_seq desc);
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
-- cleanup_stale_test_users cannot reach an org whose members are already gone.
|
|
2
|
+
--
|
|
3
|
+
-- The org sweep (00087, unchanged through 00121) elects orgs by joining
|
|
4
|
+
-- `user_organizations` to the set of users it is about to delete:
|
|
5
|
+
--
|
|
6
|
+
-- and exists (select 1 from user_organizations uo
|
|
7
|
+
-- where uo.org_id = o.id and uo.user_id = any(v_user_ids))
|
|
8
|
+
--
|
|
9
|
+
-- That arm only ever matches while the membership row still exists. Anything
|
|
10
|
+
-- that deletes the user first — the e2e suites' own `deleteTestUser`, an admin
|
|
11
|
+
-- delete, a user deleting their account — cascades `user_organizations` away,
|
|
12
|
+
-- and from that moment the org is invisible to every subsequent sweep. It has no
|
|
13
|
+
-- members, so nobody can see it, join it, or delete it through the product; it
|
|
14
|
+
-- is unreachable and permanent.
|
|
15
|
+
--
|
|
16
|
+
-- Measured in prod 2026-08-16: 30,276 memberless orgs of 30,307 total,
|
|
17
|
+
-- accumulating since 2026-07-12 (30,261 of them named "…'s Workspace", the
|
|
18
|
+
-- signup default; the other 15 are timestamp-suffixed org/team test fixtures).
|
|
19
|
+
-- They hold nothing — 0 projects, 0 sessions, 0 invitations, 0 agent_runs,
|
|
20
|
+
-- 3 test teams.
|
|
21
|
+
--
|
|
22
|
+
-- The fix is a second, independent sweep: an org with zero members, older than
|
|
23
|
+
-- an hour, is dead by construction. It cannot arise from normal use — the
|
|
24
|
+
-- `check_last_owner` trigger refuses to remove an org's final owner, so the only
|
|
25
|
+
-- way to reach zero members is the cascade above — and the hour of grace keeps a
|
|
26
|
+
-- signup that inserts the org before the membership safe.
|
|
27
|
+
--
|
|
28
|
+
-- Note this sweep is deliberately NOT restricted to test-shaped names. The
|
|
29
|
+
-- defect is not "test orgs leak"; it is "an org outlives its last member". A
|
|
30
|
+
-- real user who deletes their account leaves exactly the same corpse, and it
|
|
31
|
+
-- should be reaped for exactly the same reason.
|
|
32
|
+
--
|
|
33
|
+
-- Two other changes, both forced by the first:
|
|
34
|
+
--
|
|
35
|
+
-- * The `v_user_ids is null` early return (00087) skipped everything below it.
|
|
36
|
+
-- A quiet hour with no test users to reap would have skipped the memberless
|
|
37
|
+
-- sweep too, which is the one that has 30k rows of backlog. The early return
|
|
38
|
+
-- is gone; the user sweep is now guarded in place.
|
|
39
|
+
-- * `superadmin_audit_log.target_org_id` is ON DELETE NO ACTION, so it is
|
|
40
|
+
-- nulled for the memberless set before the delete, the same way the existing
|
|
41
|
+
-- path does it for the membered set.
|
|
42
|
+
--
|
|
43
|
+
-- Return signature is unchanged (callers: `crons run cleanup_stale_test_users`).
|
|
44
|
+
-- `deleted_orgs` now counts both sweeps.
|
|
45
|
+
|
|
46
|
+
drop function if exists public.cleanup_stale_test_users();
|
|
47
|
+
|
|
48
|
+
create function public.cleanup_stale_test_users()
|
|
49
|
+
returns table(deleted_users int, deleted_orgs int, deleted_smoke_sessions int) as $$
|
|
50
|
+
declare
|
|
51
|
+
v_user_ids uuid[];
|
|
52
|
+
v_org_ids uuid[];
|
|
53
|
+
v_orphan_org_ids uuid[];
|
|
54
|
+
v_deleted_users int := 0;
|
|
55
|
+
v_deleted_orgs int := 0;
|
|
56
|
+
v_deleted_orphan_orgs int := 0;
|
|
57
|
+
v_deleted_smoke int := 0;
|
|
58
|
+
begin
|
|
59
|
+
-- Smoke-test session residue (any user). 1h threshold keeps an
|
|
60
|
+
-- in-flight smoke run safe.
|
|
61
|
+
delete from sessions
|
|
62
|
+
where created_at < now() - interval '1 hour'
|
|
63
|
+
and (
|
|
64
|
+
external_session_id like '00000000-0000-4000-8000-%'
|
|
65
|
+
or project_path = '/tmp/smoke-test-project'
|
|
66
|
+
or metadata->>'clientName' = 'smoke-test'
|
|
67
|
+
);
|
|
68
|
+
get diagnostics v_deleted_smoke = row_count;
|
|
69
|
+
|
|
70
|
+
-- Candidates: obvious test users older than 1h, excluding the pinned dev
|
|
71
|
+
-- login. See 00121 for why each arm can't hit a real signup.
|
|
72
|
+
select array_agg(id) into v_user_ids
|
|
73
|
+
from auth.users
|
|
74
|
+
where email != 'nick@rulemetric.test'
|
|
75
|
+
and created_at < now() - interval '1 hour'
|
|
76
|
+
and (
|
|
77
|
+
email ~* '\.(test|example|invalid|localhost|local)$'
|
|
78
|
+
or email ~* '@(example\.(com|net|org))$'
|
|
79
|
+
or email ~ '1[0-9]{12}@'
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
if v_user_ids is not null and array_length(v_user_ids, 1) > 0 then
|
|
83
|
+
-- Orgs whose every member is in the to-delete set. Cascade carve-out in
|
|
84
|
+
-- check_last_owner lets the user_organizations rows go when the parent
|
|
85
|
+
-- org is being deleted.
|
|
86
|
+
select array_agg(o.id) into v_org_ids
|
|
87
|
+
from organizations o
|
|
88
|
+
where not exists (
|
|
89
|
+
select 1 from user_organizations uo
|
|
90
|
+
where uo.org_id = o.id
|
|
91
|
+
and uo.user_id != all(v_user_ids)
|
|
92
|
+
)
|
|
93
|
+
and exists (
|
|
94
|
+
select 1 from user_organizations uo
|
|
95
|
+
where uo.org_id = o.id and uo.user_id = any(v_user_ids)
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
-- 1. NULL test-user references on NO ACTION FKs (preserve content)
|
|
99
|
+
update instructions set created_by = null where created_by = any(v_user_ids);
|
|
100
|
+
update instructions set archived_by = null where archived_by = any(v_user_ids);
|
|
101
|
+
update instruction_versions set created_by = null where created_by = any(v_user_ids);
|
|
102
|
+
update projects set created_by = null where created_by = any(v_user_ids);
|
|
103
|
+
update project_instructions set added_by = null where added_by = any(v_user_ids);
|
|
104
|
+
|
|
105
|
+
-- 2. Scrub superadmin_audit_log (actor_id is NOT NULL → delete; targets nullable)
|
|
106
|
+
delete from superadmin_audit_log where actor_id = any(v_user_ids);
|
|
107
|
+
update superadmin_audit_log set target_user_id = null where target_user_id = any(v_user_ids);
|
|
108
|
+
if v_org_ids is not null then
|
|
109
|
+
update superadmin_audit_log set target_org_id = null where target_org_id = any(v_org_ids);
|
|
110
|
+
end if;
|
|
111
|
+
|
|
112
|
+
-- 3. Delete orphan orgs
|
|
113
|
+
if v_org_ids is not null then
|
|
114
|
+
delete from organizations where id = any(v_org_ids);
|
|
115
|
+
get diagnostics v_deleted_orgs = row_count;
|
|
116
|
+
end if;
|
|
117
|
+
|
|
118
|
+
-- 4. Delete users (cascades to profiles, user_organizations, sessions, ...)
|
|
119
|
+
delete from auth.users where id = any(v_user_ids);
|
|
120
|
+
get diagnostics v_deleted_users = row_count;
|
|
121
|
+
end if;
|
|
122
|
+
|
|
123
|
+
-- 5. Memberless orgs, whoever emptied them and whenever. This runs on every
|
|
124
|
+
-- sweep, independent of whether step 4 deleted anyone, because the orgs it
|
|
125
|
+
-- reaps were emptied by an EARLIER run (or by a delete this function never
|
|
126
|
+
-- saw) and no later run can otherwise reach them.
|
|
127
|
+
select array_agg(o.id) into v_orphan_org_ids
|
|
128
|
+
from organizations o
|
|
129
|
+
where o.created_at < now() - interval '1 hour'
|
|
130
|
+
and not exists (select 1 from user_organizations uo where uo.org_id = o.id);
|
|
131
|
+
|
|
132
|
+
if v_orphan_org_ids is not null then
|
|
133
|
+
update superadmin_audit_log set target_org_id = null
|
|
134
|
+
where target_org_id = any(v_orphan_org_ids);
|
|
135
|
+
delete from organizations where id = any(v_orphan_org_ids);
|
|
136
|
+
get diagnostics v_deleted_orphan_orgs = row_count;
|
|
137
|
+
v_deleted_orgs := v_deleted_orgs + v_deleted_orphan_orgs;
|
|
138
|
+
end if;
|
|
139
|
+
|
|
140
|
+
return query select v_deleted_users, v_deleted_orgs, v_deleted_smoke;
|
|
141
|
+
end;
|
|
142
|
+
$$ language plpgsql security definer set search_path = public, auth;
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
-- The judge segment gets ONE definition, and a pin outranks anything observed.
|
|
2
|
+
--
|
|
3
|
+
-- §6c scopes calibration to the current judge, and until now "current judge"
|
|
4
|
+
-- was hand-copied into four places (00148, 00169, 00190, judge-calibration.ts)
|
|
5
|
+
-- as "the most recently stamped run". Recency asks who graded LAST when the
|
|
6
|
+
-- question is whose bias we can CORRECT, and the two come apart the instant a
|
|
7
|
+
-- stray run lands. Measured 2026-08-18 on the one evolve-enrolled target:
|
|
8
|
+
--
|
|
9
|
+
-- claude-opus-5[1m] 355 runs Jul 30 - Aug 13
|
|
10
|
+
-- claude-opus-4-8[1m] 303 runs Jul 20 - Jul 29 <- all 37 labels
|
|
11
|
+
-- (unstamped) 159 runs Mar 25 - Aug 17
|
|
12
|
+
-- claude-fable-5 113 runs Aug 08 - Aug 14
|
|
13
|
+
-- claude-haiku-4-5 80 runs Aug 14 - Aug 16 <- elected, 0 labels
|
|
14
|
+
--
|
|
15
|
+
-- Recency elected the segment with zero anchors, so the adequacy gate reported
|
|
16
|
+
-- `0/50 human-labeled runs` and asked for fifty more while thirty-seven sat one
|
|
17
|
+
-- segment away. 00148's grader pin then pinned the NEXT batch to that same
|
|
18
|
+
-- stray, making the drift self-reinforcing; #357 quarantined fallback runs from
|
|
19
|
+
-- electing, but a run the producer never pinned has nothing to fall back FROM,
|
|
20
|
+
-- so unpinned evolution batches walked straight through the quarantine.
|
|
21
|
+
--
|
|
22
|
+
-- Three tiers, descending authority:
|
|
23
|
+
-- 1. PIN metadata.graderModel — a decision a person made. Nothing
|
|
24
|
+
-- observed in the run table may overrule it. A pinned segment
|
|
25
|
+
-- with zero runs is a correct answer: grading starts there.
|
|
26
|
+
-- 2. ANCHORS the segment holding the most fully human-labeled runs — the
|
|
27
|
+
-- only segment that can produce a corrected estimate at all.
|
|
28
|
+
-- 3. RECENCY the old rule, surviving as the tie-break and the
|
|
29
|
+
-- no-anchors-anywhere fallback.
|
|
30
|
+
--
|
|
31
|
+
-- Fallback runs (stamped != requested) are quarantined from ELECTING but still
|
|
32
|
+
-- count as data, exactly as #357 established. If every stamped run fell back,
|
|
33
|
+
-- the whole stamped set votes rather than electing nothing — an empty segment
|
|
34
|
+
-- would silently drop every LLM-graded run.
|
|
35
|
+
--
|
|
36
|
+
-- apps/api/src/lib/judge-calibration.ts `electJudgeSegment()` implements the
|
|
37
|
+
-- same three tiers for the TypeScript readers, and its unit tests pin the tier
|
|
38
|
+
-- order. Change one side without the other and the scheduled path will measure
|
|
39
|
+
-- a different segment than the reviewed path.
|
|
40
|
+
-- ---------------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
create or replace function public.eval_target_judge_segment(p_target_id uuid)
|
|
43
|
+
returns text
|
|
44
|
+
language sql
|
|
45
|
+
stable
|
|
46
|
+
security definer
|
|
47
|
+
set search_path = public
|
|
48
|
+
as $$
|
|
49
|
+
with elector as (
|
|
50
|
+
select
|
|
51
|
+
r.grading->>'judge_model' as model,
|
|
52
|
+
r.created_at,
|
|
53
|
+
(
|
|
54
|
+
r.grading->>'judge_model_requested' is null
|
|
55
|
+
or r.grading->>'judge_model_requested' = r.grading->>'judge_model'
|
|
56
|
+
) as clean,
|
|
57
|
+
-- Guarded: a grading blob whose expectations is not an array would make
|
|
58
|
+
-- jsonb_array_elements raise, and one malformed run must not take the
|
|
59
|
+
-- whole election down.
|
|
60
|
+
case
|
|
61
|
+
when jsonb_typeof(r.grading->'expectations') = 'array' then (
|
|
62
|
+
select count(*)
|
|
63
|
+
from jsonb_array_elements(r.grading->'expectations') e
|
|
64
|
+
where coalesce(e->>'method', 'llm') = 'llm'
|
|
65
|
+
and coalesce(e->>'errored', 'false') <> 'true'
|
|
66
|
+
)
|
|
67
|
+
else 0
|
|
68
|
+
end as llm_total,
|
|
69
|
+
coalesce(a.llm_annotated, 0) as llm_annotated
|
|
70
|
+
from eval_runs r
|
|
71
|
+
left join (
|
|
72
|
+
select eval_run_id, count(*) filter (where judge_method = 'llm') as llm_annotated
|
|
73
|
+
from eval_grade_annotations
|
|
74
|
+
where eval_target_id = p_target_id
|
|
75
|
+
group by eval_run_id
|
|
76
|
+
) a on a.eval_run_id = r.id
|
|
77
|
+
where r.eval_target_id = p_target_id
|
|
78
|
+
and r.status = 'completed'
|
|
79
|
+
and r.configuration in ('with_target', 'without_target')
|
|
80
|
+
and r.grading->>'judge_model' is not null
|
|
81
|
+
),
|
|
82
|
+
has_clean as (
|
|
83
|
+
select exists (select 1 from elector where clean) as v
|
|
84
|
+
),
|
|
85
|
+
electorate as (
|
|
86
|
+
select e.* from elector e, has_clean h where e.clean or not h.v
|
|
87
|
+
),
|
|
88
|
+
pin as (
|
|
89
|
+
select nullif(btrim(t.metadata->>'graderModel'), '') as model
|
|
90
|
+
from eval_targets t
|
|
91
|
+
where t.id = p_target_id
|
|
92
|
+
),
|
|
93
|
+
anchors as (
|
|
94
|
+
-- Ties broken by recency then model string: a tie-break that fell through
|
|
95
|
+
-- to nothing would leave the segment depending on row order, which is the
|
|
96
|
+
-- same instability recency had.
|
|
97
|
+
select model
|
|
98
|
+
from electorate
|
|
99
|
+
where llm_total > 0 and llm_annotated >= llm_total
|
|
100
|
+
group by model
|
|
101
|
+
order by count(*) desc, max(created_at) desc, model desc
|
|
102
|
+
limit 1
|
|
103
|
+
),
|
|
104
|
+
recency as (
|
|
105
|
+
select model from electorate order by created_at desc, model desc limit 1
|
|
106
|
+
)
|
|
107
|
+
select coalesce(
|
|
108
|
+
(select model from pin),
|
|
109
|
+
(select model from anchors),
|
|
110
|
+
(select model from recency)
|
|
111
|
+
);
|
|
112
|
+
$$;
|
|
113
|
+
|
|
114
|
+
comment on function public.eval_target_judge_segment(uuid) is
|
|
115
|
+
'Canonical §6c judge segment for an eval target: metadata.graderModel pin, else the segment holding the most fully human-labeled runs, else the most recently stamped run. Fallback-graded runs (stamped <> requested) cannot elect. Mirrors electJudgeSegment() in apps/api/src/lib/judge-calibration.ts — the two must not diverge. Never re-derive this inline: the hand-copied recency rule lived in four places and elected a zero-anchor segment for four days (2026-08-18).';
|
|
116
|
+
|
|
117
|
+
-- ---------------------------------------------------------------------------
|
|
118
|
+
-- Both producers now ask the function instead of carrying their own copy.
|
|
119
|
+
-- ---------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
create or replace function public.enqueue_eval_autorun()
|
|
122
|
+
returns integer
|
|
123
|
+
language plpgsql
|
|
124
|
+
security definer
|
|
125
|
+
set search_path = public
|
|
126
|
+
as $$
|
|
127
|
+
declare
|
|
128
|
+
queued integer;
|
|
129
|
+
begin
|
|
130
|
+
with eligible as (
|
|
131
|
+
select t.id, t.user_id,
|
|
132
|
+
-- Canonical segment (00193). Was an inline "most recent stamped run"
|
|
133
|
+
-- subquery, which is how a credit-exhaustion fallback became the pin.
|
|
134
|
+
public.eval_target_judge_segment(t.id) as current_judge
|
|
135
|
+
from eval_targets t
|
|
136
|
+
where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
|
|
137
|
+
-- Gate 0 (00190): the target must be anchored to something a promotion
|
|
138
|
+
-- could land on. Without this, a batch is two LLM executions plus a
|
|
139
|
+
-- judged grade spent to rewrite a row nothing reads.
|
|
140
|
+
and (
|
|
141
|
+
t.instruction_id is not null
|
|
142
|
+
or t.project_path is not null
|
|
143
|
+
or t.name like '/%'
|
|
144
|
+
)
|
|
145
|
+
-- Gate 2a: recent grading wasn't MOSTLY ungradeable (proportional, and
|
|
146
|
+
-- identical to the evolution producer's 2a since 00163).
|
|
147
|
+
and not exists (
|
|
148
|
+
select 1 from eval_runs r
|
|
149
|
+
where r.eval_target_id = t.id
|
|
150
|
+
and r.created_at > now() - interval '7 days'
|
|
151
|
+
group by r.eval_target_id
|
|
152
|
+
having count(*) >= 5
|
|
153
|
+
and count(*) filter (
|
|
154
|
+
where jsonb_typeof(r.grading->'expectations') = 'array'
|
|
155
|
+
and exists (
|
|
156
|
+
select 1 from jsonb_array_elements(r.grading->'expectations') e
|
|
157
|
+
where coalesce(e->>'errored', 'false') = 'true'
|
|
158
|
+
)
|
|
159
|
+
) * 4 >= count(*)
|
|
160
|
+
)
|
|
161
|
+
-- Gate 2b: no human anchors pointing at an ungradeable assertion.
|
|
162
|
+
and not exists (
|
|
163
|
+
select 1
|
|
164
|
+
from eval_grade_annotations a
|
|
165
|
+
join eval_runs r on r.id = a.eval_run_id
|
|
166
|
+
where a.eval_target_id = t.id
|
|
167
|
+
and jsonb_typeof(r.grading->'expectations') = 'array'
|
|
168
|
+
and coalesce(
|
|
169
|
+
r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
|
|
170
|
+
) = 'true'
|
|
171
|
+
)
|
|
172
|
+
-- Gate 2c: last week's batch wasn't mostly failures.
|
|
173
|
+
and not exists (
|
|
174
|
+
select 1 from eval_runs r
|
|
175
|
+
where r.eval_target_id = t.id
|
|
176
|
+
and r.created_at > now() - interval '7 days'
|
|
177
|
+
group by r.eval_target_id
|
|
178
|
+
having count(*) >= 5
|
|
179
|
+
and count(*) filter (where r.status = 'failed') * 4 >= count(*)
|
|
180
|
+
)
|
|
181
|
+
-- Gate 3: the target has live cases to actually run.
|
|
182
|
+
and exists (
|
|
183
|
+
select 1 from evals ev
|
|
184
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
185
|
+
)
|
|
186
|
+
-- One in flight at a time, per the 00083/00086 pattern.
|
|
187
|
+
and not exists (
|
|
188
|
+
select 1 from agent_jobs j
|
|
189
|
+
where j.task_kind = 'cron_eval_autorun'
|
|
190
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
191
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
192
|
+
)
|
|
193
|
+
), ins as (
|
|
194
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
195
|
+
select e.user_id,
|
|
196
|
+
'cron_eval_autorun',
|
|
197
|
+
'pending',
|
|
198
|
+
jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
|
|
199
|
+
|| case
|
|
200
|
+
when e.current_judge is not null
|
|
201
|
+
then jsonb_build_object('graderModel', e.current_judge)
|
|
202
|
+
else '{}'::jsonb
|
|
203
|
+
end,
|
|
204
|
+
'eval-autorun:' || e.id::text
|
|
205
|
+
from eligible e
|
|
206
|
+
returning 1
|
|
207
|
+
)
|
|
208
|
+
select count(*)::int into queued from ins;
|
|
209
|
+
|
|
210
|
+
return queued;
|
|
211
|
+
end;
|
|
212
|
+
$$;
|
|
213
|
+
|
|
214
|
+
comment on function public.enqueue_eval_autorun() is
|
|
215
|
+
'Nightly producer for cron_eval_autorun. Enqueues one bounded depth batch per OPTED-IN, ANCHORED, healthy, under-powered eval target, with the grader pinned to the target''s canonical judge segment (public.eval_target_judge_segment, 00193 — pin > anchors > recency; was an inline recency subquery from 00148/#196). Gate 0 (00190) requires instruction_id, project_path, or an absolute-path name. Health 2a is PROPORTIONAL as of 00169. Opt in with: update eval_targets set metadata = coalesce(metadata, ''{}''::jsonb) || ''{"autoRunEnabled":true}''::jsonb where id = ...';
|
|
216
|
+
|
|
217
|
+
-- ---------------------------------------------------------------------------
|
|
218
|
+
-- The evolution producer never pinned a grader AT ALL.
|
|
219
|
+
--
|
|
220
|
+
-- 00148 pinned the autorun batch precisely so a scheduled run could not fork
|
|
221
|
+
-- the segment. The evolution producer — which spends far more per night, and
|
|
222
|
+
-- whose paired replays are the ONLY route to a promotion (§13, Mode A) — was
|
|
223
|
+
-- never given the same treatment. Its runs therefore carried no
|
|
224
|
+
-- `judge_model_requested`, so #357's fallback quarantine had nothing to compare
|
|
225
|
+
-- against and they elected freely. Unpinned evolution batches are how
|
|
226
|
+
-- claude-haiku-4-5 took the segment on 2026-08-14.
|
|
227
|
+
--
|
|
228
|
+
-- Function is otherwise byte-identical to 00183.
|
|
229
|
+
-- ---------------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
create or replace function public.enqueue_instruction_evolution()
|
|
232
|
+
returns integer
|
|
233
|
+
language plpgsql
|
|
234
|
+
security definer
|
|
235
|
+
set search_path = public
|
|
236
|
+
as $$
|
|
237
|
+
declare
|
|
238
|
+
queued integer;
|
|
239
|
+
begin
|
|
240
|
+
with health_ok as (
|
|
241
|
+
-- The gates SHARED by both enrolment paths, evaluated once. All copied
|
|
242
|
+
-- byte-identical from the live definition (00163's revision): pause,
|
|
243
|
+
-- proportional 2a, absolute 2b, proportional 2c, live cases, change
|
|
244
|
+
-- budget, one-in-flight.
|
|
245
|
+
select t.id, t.user_id
|
|
246
|
+
from eval_targets t
|
|
247
|
+
where coalesce(t.metadata->>'autoEvolvePaused', 'false') <> 'true'
|
|
248
|
+
-- Health 2a: recent grading wasn't MOSTLY ungradeable (proportional, 00163).
|
|
249
|
+
and not exists (
|
|
250
|
+
select 1 from eval_runs r
|
|
251
|
+
where r.eval_target_id = t.id
|
|
252
|
+
and r.created_at > now() - interval '7 days'
|
|
253
|
+
group by r.eval_target_id
|
|
254
|
+
having count(*) >= 5
|
|
255
|
+
and count(*) filter (
|
|
256
|
+
where jsonb_typeof(r.grading->'expectations') = 'array'
|
|
257
|
+
and exists (
|
|
258
|
+
select 1 from jsonb_array_elements(r.grading->'expectations') e
|
|
259
|
+
where coalesce(e->>'errored', 'false') = 'true'
|
|
260
|
+
)
|
|
261
|
+
) * 4 >= count(*)
|
|
262
|
+
)
|
|
263
|
+
-- Health 2b: no human anchors pointing at an ungradeable assertion (absolute).
|
|
264
|
+
and not exists (
|
|
265
|
+
select 1
|
|
266
|
+
from eval_grade_annotations a
|
|
267
|
+
join eval_runs r on r.id = a.eval_run_id
|
|
268
|
+
where a.eval_target_id = t.id
|
|
269
|
+
and jsonb_typeof(r.grading->'expectations') = 'array'
|
|
270
|
+
and coalesce(
|
|
271
|
+
r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
|
|
272
|
+
) = 'true'
|
|
273
|
+
)
|
|
274
|
+
-- Health 2c: last week's batch wasn't mostly failures.
|
|
275
|
+
and not exists (
|
|
276
|
+
select 1 from eval_runs r
|
|
277
|
+
where r.eval_target_id = t.id
|
|
278
|
+
and r.created_at > now() - interval '7 days'
|
|
279
|
+
group by r.eval_target_id
|
|
280
|
+
having count(*) >= 5
|
|
281
|
+
and count(*) filter (where r.status = 'failed') * 4 >= count(*)
|
|
282
|
+
)
|
|
283
|
+
-- Live cases to measure against — at least MIN_LIVE_CASES (3).
|
|
284
|
+
-- `exists` (>= 1) was the bug: see 00183's header.
|
|
285
|
+
and (
|
|
286
|
+
select count(*) from evals ev
|
|
287
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
288
|
+
) >= 3
|
|
289
|
+
-- Change budget (cheap exclusion; the API is the authority).
|
|
290
|
+
and (
|
|
291
|
+
select count(*)
|
|
292
|
+
from instruction_promotions p
|
|
293
|
+
where p.eval_target_id = t.id
|
|
294
|
+
and p.applied
|
|
295
|
+
and p.created_at > now() - interval '7 days'
|
|
296
|
+
) < 3
|
|
297
|
+
-- One in flight at a time.
|
|
298
|
+
and not exists (
|
|
299
|
+
select 1 from agent_jobs j
|
|
300
|
+
where j.task_kind = 'cron_instruction_evolution'
|
|
301
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
302
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
303
|
+
)
|
|
304
|
+
),
|
|
305
|
+
manual as (
|
|
306
|
+
-- The hand-picked path, unchanged and uncapped.
|
|
307
|
+
select h.id, h.user_id
|
|
308
|
+
from health_ok h
|
|
309
|
+
join eval_targets t on t.id = h.id
|
|
310
|
+
where coalesce(t.metadata->>'autoEvolveEnabled', 'false') = 'true'
|
|
311
|
+
),
|
|
312
|
+
auto_ranked as (
|
|
313
|
+
-- The stated rule. Round-robin: never-decided first (nulls first), then
|
|
314
|
+
-- least-recently-decided, so the cap rotates through the candidate pool
|
|
315
|
+
-- instead of re-measuring the same winners nightly.
|
|
316
|
+
select h.id, h.user_id,
|
|
317
|
+
row_number() over (
|
|
318
|
+
partition by h.user_id
|
|
319
|
+
order by (
|
|
320
|
+
select max(p.created_at)
|
|
321
|
+
from instruction_promotions p
|
|
322
|
+
where p.eval_target_id = h.id
|
|
323
|
+
) asc nulls first,
|
|
324
|
+
h.id
|
|
325
|
+
) as rn
|
|
326
|
+
from health_ok h
|
|
327
|
+
join eval_targets t on t.id = h.id
|
|
328
|
+
where t.type = 'instruction'
|
|
329
|
+
and coalesce(t.metadata->>'autoEvolveOptOut', 'false') <> 'true'
|
|
330
|
+
-- Not already covered by the manual path.
|
|
331
|
+
and coalesce(t.metadata->>'autoEvolveEnabled', 'false') <> 'true'
|
|
332
|
+
-- ACTIVE (00145's load-bearing gate): a run in the last 30 days.
|
|
333
|
+
and exists (
|
|
334
|
+
select 1 from eval_runs r
|
|
335
|
+
where r.eval_target_id = t.id
|
|
336
|
+
and r.created_at > now() - interval '30 days'
|
|
337
|
+
)
|
|
338
|
+
),
|
|
339
|
+
eligible as (
|
|
340
|
+
select id, user_id from manual
|
|
341
|
+
union
|
|
342
|
+
select id, user_id from auto_ranked where rn <= 3
|
|
343
|
+
),
|
|
344
|
+
ins as (
|
|
345
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
346
|
+
select e.user_id,
|
|
347
|
+
'cron_instruction_evolution',
|
|
348
|
+
'pending',
|
|
349
|
+
jsonb_build_object('evalTargetId', e.id::text)
|
|
350
|
+
|| case
|
|
351
|
+
when public.eval_target_judge_segment(e.id) is not null
|
|
352
|
+
then jsonb_build_object(
|
|
353
|
+
'graderModel', public.eval_target_judge_segment(e.id))
|
|
354
|
+
else '{}'::jsonb
|
|
355
|
+
end,
|
|
356
|
+
'instruction-evolution:' || e.id::text
|
|
357
|
+
from eligible e
|
|
358
|
+
returning 1
|
|
359
|
+
)
|
|
360
|
+
select count(*)::int into queued from ins;
|
|
361
|
+
|
|
362
|
+
return queued;
|
|
363
|
+
end;
|
|
364
|
+
$$;
|
|
365
|
+
|
|
366
|
+
comment on function public.enqueue_instruction_evolution() is
|
|
367
|
+
'Nightly producer for cron_instruction_evolution. Two enrolment paths as of 00166: MANUAL (autoEvolveEnabled=true, uncapped) and the STATED RULE (type=instruction, not opted out via autoEvolveOptOut, an eval run in 30 days, capped at 3/user/night round-robin by least-recent decision). Both share the health gates (2a proportional per 00163, 2b absolute, 2c proportional), live-cases (>= 3 per 00183), change-budget and one-in-flight guards. As of 00193 the payload PINS graderModel to the canonical judge segment — without it the paired replays carried no judge_model_requested, so #357''s fallback quarantine could not see them and they forked the segment freely. Auto-enrolled cycles are measure-only: writing still requires autoEvolveApply AND an explicit immutableSections.';
|