@rulemetric/local 0.12.3 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/meta.json +2 -2
- package/dist/server.mjs +292 -255
- package/dist/supabase/migrations/00206_reconcile_attribution_producer.sql +156 -0
- package/dist/supabase/migrations/00207_repair_project_paths.sql +183 -0
- package/dist/supabase/migrations/00208_auto_accept_candidate_view.sql +131 -0
- package/dist/supabase/migrations/00209_suggestion_pipeline_conformance.sql +296 -0
- package/dist/supabase/migrations/00210_shared_eval_target_health_gates.sql +237 -0
- package/dist/supabase/migrations/00211_training_candidate_view.sql +119 -0
- package/dist/supabase/migrations/00212_reconcile_junk_session_paths.sql +393 -0
- package/package.json +2 -2
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
-- 00209 — the suggestion pipeline watches itself.
|
|
2
|
+
--
|
|
3
|
+
-- WHY. Nine gates between "proposal generated" and "proposal adopted" have now
|
|
4
|
+
-- been found one, two, three at a time, over six weeks, each by a human noticing
|
|
5
|
+
-- adoption had gone quiet and hand-writing a query. The mechanism that let every
|
|
6
|
+
-- one of them hide is identical and is recorded in four separate memories and
|
|
7
|
+
-- the 2026-08-27 census: each stage reports a single number — `nothing-eligible`,
|
|
8
|
+
-- `refused: 0`, `queued 0 rows` — that reads the same whether the pool is EMPTY
|
|
9
|
+
-- or FULL OF THINGS EVERY GATE REFUSES. A producer matching zero rows is
|
|
10
|
+
-- indistinguishable from a quiet night.
|
|
11
|
+
--
|
|
12
|
+
-- This function encodes each discovered failure SIGNATURE as a named check that
|
|
13
|
+
-- runs daily and lands as a notification (the 00147 pipeline_health_check
|
|
14
|
+
-- pattern: one consolidated, unread-deduped notification per user). The census
|
|
15
|
+
-- (scripts/dev/suggestion-gate-census.sql) is the manual deep-dive; this is the
|
|
16
|
+
-- alarm that says the deep-dive is needed.
|
|
17
|
+
--
|
|
18
|
+
-- DETECTION ONLY, on purpose. Nothing here mutates suggestions, opens gates or
|
|
19
|
+
-- writes files. Every incident in this pipeline's history that involved an
|
|
20
|
+
-- unattended writer (00205's near-miss of 14 skills into ~/.claude/skills/, the
|
|
21
|
+
-- armed-secrets workflows) argues that the automated half of "find and fix"
|
|
22
|
+
-- must be the FINDING. Fixes go through review.
|
|
23
|
+
--
|
|
24
|
+
-- Every check corresponds to a real, dated incident. A check nobody can trace
|
|
25
|
+
-- to a failure it would have caught is noise, and noise teaches people to
|
|
26
|
+
-- ignore the alarm.
|
|
27
|
+
|
|
28
|
+
-- Daily per-user counter snapshots, for the checks that detect CHRONIC
|
|
29
|
+
-- conditions (a flag that only ever grows) rather than point-in-time states.
|
|
30
|
+
create table if not exists public.pipeline_conformance_metrics (
|
|
31
|
+
id bigint generated always as identity primary key,
|
|
32
|
+
run_at timestamptz not null default now(),
|
|
33
|
+
user_id uuid not null references public.profiles(id) on delete cascade,
|
|
34
|
+
metric text not null,
|
|
35
|
+
value bigint not null
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
create index if not exists idx_conformance_metrics_lookup
|
|
39
|
+
on public.pipeline_conformance_metrics (user_id, metric, run_at desc);
|
|
40
|
+
|
|
41
|
+
alter table public.pipeline_conformance_metrics enable row level security;
|
|
42
|
+
|
|
43
|
+
-- Guarded so the file is safe to re-apply — the ledger-driven runner
|
|
44
|
+
-- (scripts/deploy/apply-migrations.sh) may re-run a file after a lost ledger,
|
|
45
|
+
-- and CREATE POLICY is the one statement here with no IF NOT EXISTS form.
|
|
46
|
+
drop policy if exists "conformance_metrics_read_own" on public.pipeline_conformance_metrics;
|
|
47
|
+
create policy "conformance_metrics_read_own" on public.pipeline_conformance_metrics
|
|
48
|
+
for select using (user_id = auth.uid());
|
|
49
|
+
|
|
50
|
+
comment on table public.pipeline_conformance_metrics is
|
|
51
|
+
'Daily per-user snapshots written by suggestion_pipeline_conformance() (00209). Exists so one-way-flag defects (gate 7: is_stale set and never cleared) are detectable as monotone growth instead of waiting for a human to notice.';
|
|
52
|
+
|
|
53
|
+
create or replace function public.suggestion_pipeline_conformance()
|
|
54
|
+
returns table (check_id text, user_id uuid, detail text)
|
|
55
|
+
language plpgsql
|
|
56
|
+
security definer
|
|
57
|
+
set search_path = public
|
|
58
|
+
as $$
|
|
59
|
+
begin
|
|
60
|
+
-- ── Snapshot metrics first, so the monotone checks below (and future ones)
|
|
61
|
+
-- have history even on days when every check is quiet. ────────────────────
|
|
62
|
+
insert into pipeline_conformance_metrics (user_id, metric, value)
|
|
63
|
+
select s.user_id, m.metric, m.value
|
|
64
|
+
from (select distinct is2.user_id from instruction_suggestions is2) s
|
|
65
|
+
cross join lateral (
|
|
66
|
+
select 'stale_open_rows' as metric,
|
|
67
|
+
(select count(*) from instruction_suggestions x
|
|
68
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
69
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
70
|
+
and x.is_stale) as value
|
|
71
|
+
union all
|
|
72
|
+
select 'undecided_add_rows',
|
|
73
|
+
(select count(*) from instruction_suggestions x
|
|
74
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
75
|
+
and x.accepted_at is null and x.dismissed_at is null)
|
|
76
|
+
union all
|
|
77
|
+
select 'auto_accept_candidates',
|
|
78
|
+
(select count(*) from auto_accept_candidate_suggestions v
|
|
79
|
+
where v.user_id = s.user_id)
|
|
80
|
+
union all
|
|
81
|
+
-- The 0.5–0.79 band: emitted by the scorer (recommend.ts floors at 0.5),
|
|
82
|
+
-- below the adoption default (0.8). Feed-only by construction — counted so
|
|
83
|
+
-- "open suggestions" stops being one number over two populations (census
|
|
84
|
+
-- cause 1, 2026-08-27: 393 of 477 open skills sat in this band).
|
|
85
|
+
select 'feed_only_band_rows',
|
|
86
|
+
(select count(*) from instruction_suggestions x
|
|
87
|
+
join instructions i on i.id = x.instruction_id
|
|
88
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
89
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
90
|
+
and i.type in ('instruction', 'skill')
|
|
91
|
+
and x.score < 0.8
|
|
92
|
+
and coalesce(i.frontmatter->>'source', '') <> 'insights')
|
|
93
|
+
union all
|
|
94
|
+
-- Undecided rows quiet for >90 days that the weekly prune can NEVER
|
|
95
|
+
-- collect, by design: they carry instruction_suggestion_events, the FK is
|
|
96
|
+
-- ON DELETE CASCADE, and those events are the only record the funnel's
|
|
97
|
+
-- numerator was ever zero (00180's comment names naive pruning as the
|
|
98
|
+
-- failure mode). Counted here instead of deleted: the census's 2026-08-27
|
|
99
|
+
-- "cause 4" proposed pruning them and was wrong — visibility is the fix
|
|
100
|
+
-- that does not destroy evidence.
|
|
101
|
+
select 'dormant_open_rows',
|
|
102
|
+
(select count(*) from instruction_suggestions x
|
|
103
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
104
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
105
|
+
and x.generated_at < now() - interval '90 days'
|
|
106
|
+
and coalesce(
|
|
107
|
+
(select max(ev.created_at) from instruction_suggestion_events ev
|
|
108
|
+
where ev.suggestion_id = x.id),
|
|
109
|
+
x.generated_at
|
|
110
|
+
) < now() - interval '90 days')
|
|
111
|
+
) m;
|
|
112
|
+
|
|
113
|
+
return query
|
|
114
|
+
|
|
115
|
+
-- ── CHECK 1: minted_on_forbidden_path ──────────────────────────────────────
|
|
116
|
+
-- Incident: census 2026-08-27 — 258 of 685 open proposals sat on paths the
|
|
117
|
+
-- writer is forbidden to touch (home dir, TMPDIR, worktrees, the dead
|
|
118
|
+
-- apps/cli). Fixed the same day by gating both mint sites on
|
|
119
|
+
-- assessMintablePath(). This check is that fix's regression alarm: any NEW
|
|
120
|
+
-- row on a non-canonical path means a mint site was added or the gate broke.
|
|
121
|
+
select 'minted_on_forbidden_path'::text,
|
|
122
|
+
s.user_id,
|
|
123
|
+
count(*) || ' suggestion(s) minted in the last 25h on non-canonical path(s), e.g. ' || min(s.project_path)
|
|
124
|
+
from instruction_suggestions s
|
|
125
|
+
where s.generated_at > now() - interval '25 hours'
|
|
126
|
+
and not exists (
|
|
127
|
+
select 1 from project_canonical_checkouts c where c.path = s.project_path
|
|
128
|
+
)
|
|
129
|
+
group by s.user_id
|
|
130
|
+
|
|
131
|
+
union all
|
|
132
|
+
|
|
133
|
+
-- ── CHECK 2: producer_silent_with_candidates ───────────────────────────────
|
|
134
|
+
-- Incident: gate 4 (00184→00197) — the DB producer's type clause matched zero
|
|
135
|
+
-- rows, pg_cron fired and SUCCEEDED at 05:00 for three days, enqueued
|
|
136
|
+
-- nothing, and no log line existed anywhere. Candidates older than 25h with
|
|
137
|
+
-- no job even CREATED in 25h cannot be a quiet night: the producer is hourly.
|
|
138
|
+
select 'producer_silent_with_candidates'::text,
|
|
139
|
+
v.user_id,
|
|
140
|
+
count(*) || ' candidate(s) (oldest ' ||
|
|
141
|
+
extract(day from now() - min(v.generated_at)) || 'd) but no auto-accept job created in 25h — the producer is not seeing them'
|
|
142
|
+
from auto_accept_candidate_suggestions v
|
|
143
|
+
where v.generated_at < now() - interval '25 hours'
|
|
144
|
+
and not exists (
|
|
145
|
+
select 1 from agent_jobs j
|
|
146
|
+
where j.user_id = v.user_id
|
|
147
|
+
and j.task_kind = 'cron_auto_accept_suggestions'
|
|
148
|
+
and j.created_at > now() - interval '25 hours'
|
|
149
|
+
)
|
|
150
|
+
group by v.user_id
|
|
151
|
+
|
|
152
|
+
union all
|
|
153
|
+
|
|
154
|
+
-- ── CHECK 3: endpoint_refuses_everything ───────────────────────────────────
|
|
155
|
+
-- Incident: gates 1–4 (#372, 2026-08-16..19) — the producer queued jobs, the
|
|
156
|
+
-- worker ran them, the ENDPOINT refused every candidate on a clause the
|
|
157
|
+
-- producer didn't have, and four days of `nothing-eligible` looked like four
|
|
158
|
+
-- quiet nights. Signature: jobs completing against a standing candidate set
|
|
159
|
+
-- while NOTHING moves — no acceptance, no refusal event. A healthy run
|
|
160
|
+
-- always moves one of those; a placeholder-refusal loop that repeats forever
|
|
161
|
+
-- without logging also lands here, and deserves to.
|
|
162
|
+
select 'endpoint_refuses_everything'::text,
|
|
163
|
+
v.user_id,
|
|
164
|
+
count(distinct v.suggestion_id) ||
|
|
165
|
+
' candidate(s) standing >48h while auto-accept jobs completed — zero adoptions, zero refusal events. The producer and the endpoint disagree about eligibility.'
|
|
166
|
+
from auto_accept_candidate_suggestions v
|
|
167
|
+
where v.generated_at < now() - interval '48 hours'
|
|
168
|
+
and exists (
|
|
169
|
+
select 1 from agent_jobs j
|
|
170
|
+
where j.user_id = v.user_id
|
|
171
|
+
and j.task_kind = 'cron_auto_accept_suggestions'
|
|
172
|
+
and j.status = 'completed'
|
|
173
|
+
and j.updated_at > now() - interval '48 hours'
|
|
174
|
+
)
|
|
175
|
+
and not exists (
|
|
176
|
+
select 1 from instruction_suggestions a
|
|
177
|
+
where a.user_id = v.user_id
|
|
178
|
+
and a.accepted_at > now() - interval '48 hours'
|
|
179
|
+
)
|
|
180
|
+
and not exists (
|
|
181
|
+
select 1 from instruction_suggestion_events ev
|
|
182
|
+
join instruction_suggestions es on es.id = ev.suggestion_id
|
|
183
|
+
where es.user_id = v.user_id
|
|
184
|
+
and ev.event_type in ('redundant', 'partial_overlap', 'unresolvable_reference')
|
|
185
|
+
and ev.created_at > now() - interval '48 hours'
|
|
186
|
+
)
|
|
187
|
+
group by v.user_id
|
|
188
|
+
|
|
189
|
+
union all
|
|
190
|
+
|
|
191
|
+
-- ── CHECK 4: stale_flag_only_grows ─────────────────────────────────────────
|
|
192
|
+
-- Incident: gate 7 (2026-08-24) — is_stale was set by the mark-stale cron and
|
|
193
|
+
-- NOTHING ever cleared it; 51 adoption-grade proposals went invisible, and
|
|
194
|
+
-- one user's whole skill lane was structurally dead for weeks. A one-way flag
|
|
195
|
+
-- is invisible at any instant; over days it is a staircase. Fires when the
|
|
196
|
+
-- last 8 daily samples are all positive, never decrease, and end higher than
|
|
197
|
+
-- they started.
|
|
198
|
+
select 'stale_flag_only_grows'::text,
|
|
199
|
+
g.user_id,
|
|
200
|
+
'stale-flagged open rows grew ' || g.first_value || ' → ' || g.last_value ||
|
|
201
|
+
' over ' || g.samples || ' daily samples with no decrease — nothing is un-flagging them'
|
|
202
|
+
from (
|
|
203
|
+
select m.user_id,
|
|
204
|
+
count(*) as samples,
|
|
205
|
+
(array_agg(m.value order by m.run_at))[1] as first_value,
|
|
206
|
+
(array_agg(m.value order by m.run_at desc))[1] as last_value,
|
|
207
|
+
bool_and(m.value > 0) as all_positive,
|
|
208
|
+
-- Non-decreasing across the window: max of pairwise drops is 0.
|
|
209
|
+
coalesce(max(m.drop), 0) = 0 as never_decreased
|
|
210
|
+
from (
|
|
211
|
+
select mm.user_id, mm.run_at, mm.value,
|
|
212
|
+
greatest(lag(mm.value) over (partition by mm.user_id order by mm.run_at) - mm.value, 0) as drop
|
|
213
|
+
from pipeline_conformance_metrics mm
|
|
214
|
+
where mm.metric = 'stale_open_rows'
|
|
215
|
+
and mm.run_at > now() - interval '8 days'
|
|
216
|
+
) m
|
|
217
|
+
group by m.user_id
|
|
218
|
+
) g
|
|
219
|
+
where g.samples >= 8 and g.all_positive and g.never_decreased and g.last_value > g.first_value
|
|
220
|
+
|
|
221
|
+
union all
|
|
222
|
+
|
|
223
|
+
-- ── CHECK 5: surface_budget_spent_on_unadoptable ───────────────────────────
|
|
224
|
+
-- Incident: gates 5/6 and census cause 2 — ~840 feature/hook/mcp pitches per
|
|
225
|
+
-- 30 days entered the SessionStart feed, burned their five-ask surface budget
|
|
226
|
+
-- on an audience that cannot act on them, and converted into permanent
|
|
227
|
+
-- silence-declines. Fixed 2026-08-27 by excluding non-adoptable types from
|
|
228
|
+
-- the session-start render set (feed.ts). Any surfaced event on such a row
|
|
229
|
+
-- after that is the filter regressing.
|
|
230
|
+
select 'surface_budget_spent_on_unadoptable'::text,
|
|
231
|
+
s.user_id,
|
|
232
|
+
count(distinct ev.suggestion_id) ||
|
|
233
|
+
' non-adoptable proposal(s) (feature/hook/mcp/…) surfaced into sessions in the last 7d — they burn budget for an audience that cannot act'
|
|
234
|
+
from instruction_suggestion_events ev
|
|
235
|
+
join instruction_suggestions s on s.id = ev.suggestion_id
|
|
236
|
+
join instructions i on i.id = s.instruction_id
|
|
237
|
+
where ev.event_type in ('surfaced', 'surfaced_with_content')
|
|
238
|
+
and ev.created_at > now() - interval '7 days'
|
|
239
|
+
and s.kind = 'add'
|
|
240
|
+
and i.type not in ('instruction', 'skill')
|
|
241
|
+
group by s.user_id;
|
|
242
|
+
end;
|
|
243
|
+
$$;
|
|
244
|
+
|
|
245
|
+
comment on function public.suggestion_pipeline_conformance() is
|
|
246
|
+
'Daily conformance sweep over the suggestion/promotion pipeline. Each check encodes the observable signature of a dated incident (gate 4 silent producer, #372 producer/endpoint drift, gate 7 one-way staleness, census causes 2 and 3). Detection only — never mutates pipeline state. Findings land as one consolidated notification per user (pattern: 00147).';
|
|
247
|
+
|
|
248
|
+
-- Deliver findings the way pipeline_health_check does: one consolidated
|
|
249
|
+
-- notification per user, deduped while a previous one is still unread.
|
|
250
|
+
create or replace function public.run_suggestion_pipeline_conformance()
|
|
251
|
+
returns integer
|
|
252
|
+
language plpgsql
|
|
253
|
+
security definer
|
|
254
|
+
set search_path = public
|
|
255
|
+
as $$
|
|
256
|
+
declare
|
|
257
|
+
created integer := 0;
|
|
258
|
+
begin
|
|
259
|
+
with findings as (
|
|
260
|
+
select * from public.suggestion_pipeline_conformance()
|
|
261
|
+
),
|
|
262
|
+
by_user as (
|
|
263
|
+
select f.user_id as uid, count(*)::int as n,
|
|
264
|
+
string_agg('• [' || f.check_id || '] ' || f.detail, E'\n' order by f.check_id) as body
|
|
265
|
+
from findings f
|
|
266
|
+
group by f.user_id
|
|
267
|
+
),
|
|
268
|
+
inserted as (
|
|
269
|
+
insert into notifications (user_id, type, title, body, url, metadata)
|
|
270
|
+
select b.uid,
|
|
271
|
+
'suggestion_conformance',
|
|
272
|
+
'Suggestion pipeline conformance findings',
|
|
273
|
+
b.body,
|
|
274
|
+
'/loop',
|
|
275
|
+
jsonb_build_object('findings', b.n)
|
|
276
|
+
from by_user b
|
|
277
|
+
where not exists (
|
|
278
|
+
select 1 from notifications n
|
|
279
|
+
where n.user_id = b.uid
|
|
280
|
+
and n.type = 'suggestion_conformance'
|
|
281
|
+
and n.read_at is null
|
|
282
|
+
)
|
|
283
|
+
returning 1
|
|
284
|
+
)
|
|
285
|
+
select count(*)::int into created from inserted;
|
|
286
|
+
return created;
|
|
287
|
+
end;
|
|
288
|
+
$$;
|
|
289
|
+
|
|
290
|
+
-- 05:35 UTC — after eval_health_check (05:20) and pipeline_health_check
|
|
291
|
+
-- (05:25), so the morning's findings arrive together.
|
|
292
|
+
select cron.schedule(
|
|
293
|
+
'suggestion_pipeline_conformance',
|
|
294
|
+
'35 5 * * *',
|
|
295
|
+
$$select public.run_suggestion_pipeline_conformance();$$
|
|
296
|
+
);
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
-- 00210 — the eval-target health gates, defined once.
|
|
2
|
+
--
|
|
3
|
+
-- `enqueue_instruction_evolution` and `enqueue_eval_autorun` carried the three
|
|
4
|
+
-- health gates (2a proportional-ungradeable, 2b anchored-on-errored, 2c
|
|
5
|
+
-- mostly-failures) as byte-identical hand-synced copies — the evolution
|
|
6
|
+
-- producer's own comment admits it: "All copied byte-identical from the live
|
|
7
|
+
-- definition (00163's revision)". Two copies kept in step by discipline is the
|
|
8
|
+
-- exact mechanism behind the canonical-checkout incident (00186: five copies,
|
|
9
|
+
-- wrong in all five at once, silent) and the judge-segment incident (00193:
|
|
10
|
+
-- five copies, one credit-exhaustion fallback re-pointed the evidence base).
|
|
11
|
+
-- This repo has now paid for that lesson three times; the fix is the same each
|
|
12
|
+
-- time — one named object, every reader reads it.
|
|
13
|
+
--
|
|
14
|
+
-- WHAT IS SHARED AND WHAT IS NOT. Only 2a/2b/2c are shared. The live-cases
|
|
15
|
+
-- floor is NOT: evolution requires >= 3 (00183 — `exists` was the bug) while
|
|
16
|
+
-- autorun requires >= 1, and collapsing that difference here would silently
|
|
17
|
+
-- change enrolment. Pause flags, enrolment paths, change budgets and
|
|
18
|
+
-- in-flight dedupe stay with their producers.
|
|
19
|
+
--
|
|
20
|
+
-- Gate bodies below are copied VERBATIM from the live prosrc (2026-08-27).
|
|
21
|
+
-- Behaviour-identical by construction; the existing suites
|
|
22
|
+
-- (evolution-enrolment-rule.test.ts, eval-autorun-health-gate.test.ts) are
|
|
23
|
+
-- the proof.
|
|
24
|
+
|
|
25
|
+
create or replace view public.healthy_eval_targets as
|
|
26
|
+
select t.id as eval_target_id, t.user_id
|
|
27
|
+
from eval_targets t
|
|
28
|
+
where
|
|
29
|
+
-- Health 2a: recent grading wasn't MOSTLY ungradeable (proportional, 00163).
|
|
30
|
+
not exists (
|
|
31
|
+
select 1 from eval_runs r
|
|
32
|
+
where r.eval_target_id = t.id
|
|
33
|
+
and r.created_at > now() - interval '7 days'
|
|
34
|
+
group by r.eval_target_id
|
|
35
|
+
having count(*) >= 5
|
|
36
|
+
and count(*) filter (
|
|
37
|
+
where jsonb_typeof(r.grading->'expectations') = 'array'
|
|
38
|
+
and exists (
|
|
39
|
+
select 1 from jsonb_array_elements(r.grading->'expectations') e
|
|
40
|
+
where coalesce(e->>'errored', 'false') = 'true'
|
|
41
|
+
)
|
|
42
|
+
) * 4 >= count(*)
|
|
43
|
+
)
|
|
44
|
+
-- Health 2b: no human anchors pointing at an ungradeable assertion (absolute).
|
|
45
|
+
and not exists (
|
|
46
|
+
select 1
|
|
47
|
+
from eval_grade_annotations a
|
|
48
|
+
join eval_runs r on r.id = a.eval_run_id
|
|
49
|
+
where a.eval_target_id = t.id
|
|
50
|
+
and jsonb_typeof(r.grading->'expectations') = 'array'
|
|
51
|
+
and coalesce(
|
|
52
|
+
r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
|
|
53
|
+
) = 'true'
|
|
54
|
+
)
|
|
55
|
+
-- Health 2c: last week's batch wasn't mostly failures.
|
|
56
|
+
and not exists (
|
|
57
|
+
select 1 from eval_runs r
|
|
58
|
+
where r.eval_target_id = t.id
|
|
59
|
+
and r.created_at > now() - interval '7 days'
|
|
60
|
+
group by r.eval_target_id
|
|
61
|
+
having count(*) >= 5
|
|
62
|
+
and count(*) filter (where r.status = 'failed') * 4 >= count(*)
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
grant select on public.healthy_eval_targets to authenticated, service_role;
|
|
66
|
+
|
|
67
|
+
comment on view public.healthy_eval_targets is
|
|
68
|
+
'Eval targets passing the three shared health gates (2a proportional-ungradeable, 2b human anchor on an errored assertion, 2c mostly-failed batch). ONE definition (00210) read by both enqueue_instruction_evolution and enqueue_eval_autorun, which previously carried byte-identical hand-synced copies. Live-cases floors are deliberately NOT here — the two producers differ (>=3 vs >=1).';
|
|
69
|
+
|
|
70
|
+
-- ── enqueue_instruction_evolution, health gates now read from the view ───────
|
|
71
|
+
create or replace function public.enqueue_instruction_evolution()
|
|
72
|
+
returns integer
|
|
73
|
+
language plpgsql
|
|
74
|
+
security definer
|
|
75
|
+
set search_path = public
|
|
76
|
+
as $$
|
|
77
|
+
declare
|
|
78
|
+
queued integer;
|
|
79
|
+
begin
|
|
80
|
+
with health_ok as (
|
|
81
|
+
select t.id, t.user_id
|
|
82
|
+
from eval_targets t
|
|
83
|
+
where coalesce(t.metadata->>'autoEvolvePaused', 'false') <> 'true'
|
|
84
|
+
-- Health 2a/2b/2c: ONE definition (00210), shared with eval autorun.
|
|
85
|
+
and exists (
|
|
86
|
+
select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
|
|
87
|
+
)
|
|
88
|
+
-- Live cases to measure against — at least MIN_LIVE_CASES (3).
|
|
89
|
+
-- `exists` (>= 1) was the bug: see 00183's header.
|
|
90
|
+
and (
|
|
91
|
+
select count(*) from evals ev
|
|
92
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
93
|
+
) >= 3
|
|
94
|
+
-- Change budget (cheap exclusion; the API is the authority).
|
|
95
|
+
and (
|
|
96
|
+
select count(*)
|
|
97
|
+
from instruction_promotions p
|
|
98
|
+
where p.eval_target_id = t.id
|
|
99
|
+
and p.applied
|
|
100
|
+
and p.created_at > now() - interval '7 days'
|
|
101
|
+
) < 3
|
|
102
|
+
-- One in flight at a time.
|
|
103
|
+
and not exists (
|
|
104
|
+
select 1 from agent_jobs j
|
|
105
|
+
where j.task_kind = 'cron_instruction_evolution'
|
|
106
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
107
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
108
|
+
)
|
|
109
|
+
),
|
|
110
|
+
manual as (
|
|
111
|
+
-- The hand-picked path, unchanged and uncapped.
|
|
112
|
+
select h.id, h.user_id
|
|
113
|
+
from health_ok h
|
|
114
|
+
join eval_targets t on t.id = h.id
|
|
115
|
+
where coalesce(t.metadata->>'autoEvolveEnabled', 'false') = 'true'
|
|
116
|
+
),
|
|
117
|
+
auto_ranked as (
|
|
118
|
+
-- The stated rule. Round-robin: never-decided first (nulls first), then
|
|
119
|
+
-- least-recently-decided, so the cap rotates through the candidate pool
|
|
120
|
+
-- instead of re-measuring the same winners nightly.
|
|
121
|
+
select h.id, h.user_id,
|
|
122
|
+
row_number() over (
|
|
123
|
+
partition by h.user_id
|
|
124
|
+
order by (
|
|
125
|
+
select max(p.created_at)
|
|
126
|
+
from instruction_promotions p
|
|
127
|
+
where p.eval_target_id = h.id
|
|
128
|
+
) asc nulls first,
|
|
129
|
+
h.id
|
|
130
|
+
) as rn
|
|
131
|
+
from health_ok h
|
|
132
|
+
join eval_targets t on t.id = h.id
|
|
133
|
+
where t.type = 'instruction'
|
|
134
|
+
and coalesce(t.metadata->>'autoEvolveOptOut', 'false') <> 'true'
|
|
135
|
+
-- Not already covered by the manual path.
|
|
136
|
+
and coalesce(t.metadata->>'autoEvolveEnabled', 'false') <> 'true'
|
|
137
|
+
-- ACTIVE (00145's load-bearing gate): a run in the last 30 days.
|
|
138
|
+
and exists (
|
|
139
|
+
select 1 from eval_runs r
|
|
140
|
+
where r.eval_target_id = t.id
|
|
141
|
+
and r.created_at > now() - interval '30 days'
|
|
142
|
+
)
|
|
143
|
+
),
|
|
144
|
+
eligible as (
|
|
145
|
+
select id, user_id from manual
|
|
146
|
+
union
|
|
147
|
+
select id, user_id from auto_ranked where rn <= 3
|
|
148
|
+
),
|
|
149
|
+
ins as (
|
|
150
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
151
|
+
select e.user_id,
|
|
152
|
+
'cron_instruction_evolution',
|
|
153
|
+
'pending',
|
|
154
|
+
jsonb_build_object('evalTargetId', e.id::text)
|
|
155
|
+
|| case
|
|
156
|
+
when public.eval_target_judge_segment(e.id) is not null
|
|
157
|
+
then jsonb_build_object(
|
|
158
|
+
'graderModel', public.eval_target_judge_segment(e.id))
|
|
159
|
+
else '{}'::jsonb
|
|
160
|
+
end,
|
|
161
|
+
'instruction-evolution:' || e.id::text
|
|
162
|
+
from eligible e
|
|
163
|
+
returning 1
|
|
164
|
+
)
|
|
165
|
+
select count(*)::int into queued from ins;
|
|
166
|
+
|
|
167
|
+
return queued;
|
|
168
|
+
end;
|
|
169
|
+
$$;
|
|
170
|
+
|
|
171
|
+
comment on function public.enqueue_instruction_evolution() is
|
|
172
|
+
'Daily producer for cron_instruction_evolution. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); pause, live-cases floor (>=3, 00183), change budget, round-robin enrolment and in-flight dedupe stay here. Judge segment from eval_target_judge_segment() (00193).';
|
|
173
|
+
|
|
174
|
+
-- ── enqueue_eval_autorun, health gates now read from the view ────────────────
|
|
175
|
+
create or replace function public.enqueue_eval_autorun()
|
|
176
|
+
returns integer
|
|
177
|
+
language plpgsql
|
|
178
|
+
security definer
|
|
179
|
+
set search_path = public
|
|
180
|
+
as $$
|
|
181
|
+
declare
|
|
182
|
+
queued integer;
|
|
183
|
+
begin
|
|
184
|
+
with eligible as (
|
|
185
|
+
select t.id, t.user_id,
|
|
186
|
+
-- Canonical segment (00193). Was an inline "most recent stamped run"
|
|
187
|
+
-- subquery, which is how a credit-exhaustion fallback became the pin.
|
|
188
|
+
public.eval_target_judge_segment(t.id) as current_judge
|
|
189
|
+
from eval_targets t
|
|
190
|
+
where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
|
|
191
|
+
-- Gate 0 (00190): the target must be anchored to something a promotion
|
|
192
|
+
-- could land on. Without this, a batch is two LLM executions plus a
|
|
193
|
+
-- judged grade spent to rewrite a row nothing reads.
|
|
194
|
+
and (
|
|
195
|
+
t.instruction_id is not null
|
|
196
|
+
or t.project_path is not null
|
|
197
|
+
or t.name like '/%'
|
|
198
|
+
)
|
|
199
|
+
-- Gates 2a/2b/2c: ONE definition (00210), shared with evolution.
|
|
200
|
+
and exists (
|
|
201
|
+
select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
|
|
202
|
+
)
|
|
203
|
+
-- Gate 3: the target has live cases to actually run.
|
|
204
|
+
and exists (
|
|
205
|
+
select 1 from evals ev
|
|
206
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
207
|
+
)
|
|
208
|
+
-- One in flight at a time, per the 00083/00086 pattern.
|
|
209
|
+
and not exists (
|
|
210
|
+
select 1 from agent_jobs j
|
|
211
|
+
where j.task_kind = 'cron_eval_autorun'
|
|
212
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
213
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
214
|
+
)
|
|
215
|
+
), ins as (
|
|
216
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
217
|
+
select e.user_id,
|
|
218
|
+
'cron_eval_autorun',
|
|
219
|
+
'pending',
|
|
220
|
+
jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
|
|
221
|
+
|| case
|
|
222
|
+
when e.current_judge is not null
|
|
223
|
+
then jsonb_build_object('graderModel', e.current_judge)
|
|
224
|
+
else '{}'::jsonb
|
|
225
|
+
end,
|
|
226
|
+
'eval-autorun:' || e.id::text
|
|
227
|
+
from eligible e
|
|
228
|
+
returning 1
|
|
229
|
+
)
|
|
230
|
+
select count(*)::int into queued from ins;
|
|
231
|
+
|
|
232
|
+
return queued;
|
|
233
|
+
end;
|
|
234
|
+
$$;
|
|
235
|
+
|
|
236
|
+
comment on function public.enqueue_eval_autorun() is
|
|
237
|
+
'Daily producer for cron_eval_autorun. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); the anchor gate (00190), live-cases exists, judge segment (00193) and in-flight dedupe stay here.';
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
-- 00211 — the trial-nomination candidate predicate, defined once.
|
|
2
|
+
--
|
|
3
|
+
-- Same treatment as 00208 gave the auto-accept producer: the eligibility
|
|
4
|
+
-- predicate becomes a named view so the producer, the conformance monitor and
|
|
5
|
+
-- diagnostics read ONE definition. `enqueue_instruction_training` is the
|
|
6
|
+
-- second-churniest producer in the corpus (six rewrites: 00171, 00174, 00175,
|
|
7
|
+
-- 00178, 00186, 00188 — including TWO separate canonical-path fixes), and
|
|
8
|
+
-- every rewrite was a predicate change hand-held through pl/pgsql.
|
|
9
|
+
--
|
|
10
|
+
-- TWO DELIBERATE NON-CHANGES, recorded so nobody "fixes" them casually:
|
|
11
|
+
--
|
|
12
|
+
-- * `declined_at IS NULL` stays STRICT. This is NOT the gate-8 drift: the
|
|
13
|
+
-- trialable endpoint (loop.ts) applies the same strict form, so producer
|
|
14
|
+
-- and endpoint AGREE — a silence-declined proposal is excluded from
|
|
15
|
+
-- trials at both sites today. Whether trials SHOULD admit silence
|
|
16
|
+
-- declines (measurement settles what silence cannot) is a policy question
|
|
17
|
+
-- for the owner; changing it belongs in its own migration with both sites
|
|
18
|
+
-- moving together, never smuggled into an extraction.
|
|
19
|
+
--
|
|
20
|
+
-- * The activity window stays a PARAMETER. The function takes
|
|
21
|
+
-- p_active_days (default 7); a view cannot. So the view exposes
|
|
22
|
+
-- `last_session_activity_at` and the FUNCTION applies the cutoff —
|
|
23
|
+
-- the predicate is centralised, the knob is preserved.
|
|
24
|
+
--
|
|
25
|
+
-- ONE EQUIVALENT SIMPLIFICATION: the old canonical check was
|
|
26
|
+
-- exists (project_checkouts pc join project_canonical_checkouts c
|
|
27
|
+
-- on c.project_id = pc.project_id
|
|
28
|
+
-- where pc.path = s.project_path and c.path = s.project_path)
|
|
29
|
+
-- Since every project_canonical_checkouts row IS a project_checkouts row
|
|
30
|
+
-- (the view selects from that table), `c.path = s.project_path` alone implies
|
|
31
|
+
-- the pc branch. Rewritten to the same single-EXISTS form 00208 uses.
|
|
32
|
+
|
|
33
|
+
create or replace view public.instruction_training_candidate_suggestions as
|
|
34
|
+
select s.id as suggestion_id,
|
|
35
|
+
s.user_id,
|
|
36
|
+
s.project_path,
|
|
37
|
+
s.instruction_id,
|
|
38
|
+
s.score,
|
|
39
|
+
(
|
|
40
|
+
select max(se.started_at)
|
|
41
|
+
from project_checkouts pc
|
|
42
|
+
join sessions se on se.project_id = pc.project_id
|
|
43
|
+
where pc.path = s.project_path
|
|
44
|
+
and se.user_id = s.user_id
|
|
45
|
+
) as last_session_activity_at
|
|
46
|
+
from instruction_suggestions s
|
|
47
|
+
where s.kind = 'add'
|
|
48
|
+
and s.accepted_at is null
|
|
49
|
+
and s.dismissed_at is null
|
|
50
|
+
-- STRICT on purpose — see header. The trialable endpoint agrees.
|
|
51
|
+
and s.declined_at is null
|
|
52
|
+
and s.is_stale = false
|
|
53
|
+
-- The canonical checkout, nowhere else (00186/00188 lineage; simplified
|
|
54
|
+
-- equivalent of the old double-join — see header).
|
|
55
|
+
and exists (
|
|
56
|
+
select 1 from public.project_canonical_checkouts c
|
|
57
|
+
where c.path = s.project_path
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
grant select on public.instruction_training_candidate_suggestions to authenticated, service_role;
|
|
61
|
+
|
|
62
|
+
comment on view public.instruction_training_candidate_suggestions is
|
|
63
|
+
'Trial-nomination candidate predicate, ONE definition (00211, the 00208 treatment for the training lane). The activity cutoff is applied by enqueue_instruction_training() against last_session_activity_at so p_active_days stays a parameter. declined_at IS NULL is strict BY AGREEMENT with the trialable endpoint — changing that policy must move both sites in one migration.';
|
|
64
|
+
|
|
65
|
+
create or replace function public.enqueue_instruction_training(p_active_days integer default 7)
|
|
66
|
+
returns integer
|
|
67
|
+
language plpgsql
|
|
68
|
+
security definer
|
|
69
|
+
set search_path = public
|
|
70
|
+
as $$
|
|
71
|
+
declare
|
|
72
|
+
queued integer;
|
|
73
|
+
begin
|
|
74
|
+
with eligible as (
|
|
75
|
+
select v.user_id, v.project_path
|
|
76
|
+
from public.instruction_training_candidate_suggestions v
|
|
77
|
+
where v.last_session_activity_at > now() - (p_active_days || ' days')::interval
|
|
78
|
+
and not exists (
|
|
79
|
+
select 1 from agent_jobs j
|
|
80
|
+
where j.task_kind = 'cron_instruction_training'
|
|
81
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
82
|
+
and j.payload->>'projectPath' = v.project_path
|
|
83
|
+
and j.user_id = v.user_id
|
|
84
|
+
)
|
|
85
|
+
group by v.user_id, v.project_path
|
|
86
|
+
|
|
87
|
+
union
|
|
88
|
+
|
|
89
|
+
-- Running experiments always get their refresh pass, candidates or not.
|
|
90
|
+
select e.user_id, e.project_path
|
|
91
|
+
from instruction_experiments e
|
|
92
|
+
where e.status = 'running'
|
|
93
|
+
and not exists (
|
|
94
|
+
select 1 from agent_jobs j
|
|
95
|
+
where j.task_kind = 'cron_instruction_training'
|
|
96
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
97
|
+
and j.payload->>'projectPath' = e.project_path
|
|
98
|
+
and j.user_id = e.user_id
|
|
99
|
+
)
|
|
100
|
+
group by e.user_id, e.project_path
|
|
101
|
+
),
|
|
102
|
+
ins as (
|
|
103
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
104
|
+
select e.user_id,
|
|
105
|
+
'cron_instruction_training',
|
|
106
|
+
'pending',
|
|
107
|
+
jsonb_build_object('projectPath', e.project_path),
|
|
108
|
+
'instruction-training:' || e.user_id::text || ':' || e.project_path
|
|
109
|
+
from eligible e
|
|
110
|
+
returning 1
|
|
111
|
+
)
|
|
112
|
+
select count(*)::int into queued from ins;
|
|
113
|
+
|
|
114
|
+
return queued;
|
|
115
|
+
end;
|
|
116
|
+
$$;
|
|
117
|
+
|
|
118
|
+
comment on function public.enqueue_instruction_training(integer) is
|
|
119
|
+
'Producer for cron_instruction_training. Candidate predicate lives in the instruction_training_candidate_suggestions view (00211); this function applies the p_active_days activity cutoff, the running-experiments refresh branch, and in-flight dedupe. History: 00171, 00174 (active gate), 00186/00188 (canonical path), 00210-era extraction.';
|