@rulemetric/local 0.12.4 → 0.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/meta.json +2 -2
- package/dist/server.mjs +277 -245
- package/dist/supabase/migrations/00208_auto_accept_candidate_view.sql +131 -0
- package/dist/supabase/migrations/00209_suggestion_pipeline_conformance.sql +296 -0
- package/dist/supabase/migrations/00210_shared_eval_target_health_gates.sql +237 -0
- package/dist/supabase/migrations/00211_training_candidate_view.sql +119 -0
- package/dist/supabase/migrations/00212_reconcile_junk_session_paths.sql +393 -0
- package/package.json +2 -2
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
-- 00208 — ONE candidate set for auto-accept, defined once, read everywhere.
|
|
2
|
+
--
|
|
3
|
+
-- THE PATTERN THIS ENDS. Nine separate gates have now been found between "a
|
|
4
|
+
-- proposal exists" and "the actuator adopts it", and every one of them was
|
|
5
|
+
-- found the same way: adoption looked stopped, somebody hand-wrote a query,
|
|
6
|
+
-- found ONE gate, opened it, and declared the chain clear. #372's own commit
|
|
7
|
+
-- message said "two gates had to open, not one" — there were four. Then five
|
|
8
|
+
-- and six (00198/00199), seven and eight (00203/00204), and the 2026-08-27
|
|
9
|
+
-- census. The mechanism is always the same: the eligibility predicate is
|
|
10
|
+
-- HAND-COPIED (TS endpoint, TS operator count, SQL producer), the copies
|
|
11
|
+
-- drift, and the stage that disagrees reports a number that reads identically
|
|
12
|
+
-- to "there was nothing to do".
|
|
13
|
+
--
|
|
14
|
+
-- This view is the producer's candidate predicate extracted to a single named
|
|
15
|
+
-- object. From now on:
|
|
16
|
+
--
|
|
17
|
+
-- * `enqueue_auto_accept_suggestions()` (below) SELECTS from it — no inline
|
|
18
|
+
-- copy of the predicate survives in the producer.
|
|
19
|
+
-- * `suggestion_pipeline_conformance()` (00209) counts it, so the monitor
|
|
20
|
+
-- and the producer can never disagree about what "candidates exist" means.
|
|
21
|
+
-- * Diagnostic queries (scripts/dev/suggestion-gate-census.sql) can read it
|
|
22
|
+
-- instead of restating the ladder.
|
|
23
|
+
--
|
|
24
|
+
-- WHAT IT DELIBERATELY IS NOT. It is not the endpoint's predicate. The
|
|
25
|
+
-- authority for "may this be written into a file" stays server-side in
|
|
26
|
+
-- `GET /instruction-suggestions/auto-acceptable` (loop.ts), which additionally
|
|
27
|
+
-- applies the SETTABLE score gate (`resolveAutoAcceptLimits`), per-run and
|
|
28
|
+
-- window budgets, placeholder detection, the 30-day actuator-refusal window
|
|
29
|
+
-- and the running-experiment guard. This view is the producer's CHEAP SUBSET —
|
|
30
|
+
-- its job is to avoid queueing work that would refuse, and 0.8 here is the
|
|
31
|
+
-- DEFAULT minScore, hardcoded exactly as 00205 had it. That asymmetry is
|
|
32
|
+
-- recorded, intended, and now monitored: 00209's `endpoint_refuses_everything`
|
|
33
|
+
-- check fires when jobs run against this candidate set and neither adopt nor
|
|
34
|
+
-- log a refusal, which is the observable signature of the two predicates
|
|
35
|
+
-- having drifted apart (the exact #372 failure, as an alarm instead of an
|
|
36
|
+
-- archaeology session).
|
|
37
|
+
--
|
|
38
|
+
-- Grain: one row per candidate SUGGESTION. The producer groups to
|
|
39
|
+
-- (user_id, project_path); the monitor counts rows and users. The in-flight
|
|
40
|
+
-- agent_jobs dedupe stays in the producer — that is queue state, not
|
|
41
|
+
-- eligibility, and a suggestion does not stop being a candidate because a job
|
|
42
|
+
-- is already on its way.
|
|
43
|
+
|
|
44
|
+
create or replace view public.auto_accept_candidate_suggestions as
|
|
45
|
+
select s.id as suggestion_id,
|
|
46
|
+
s.user_id,
|
|
47
|
+
s.project_path,
|
|
48
|
+
s.project_id,
|
|
49
|
+
s.instruction_id,
|
|
50
|
+
s.score,
|
|
51
|
+
i.type as instruction_type,
|
|
52
|
+
s.generated_at
|
|
53
|
+
from instruction_suggestions s
|
|
54
|
+
join profiles p on p.id = s.user_id
|
|
55
|
+
join instructions i on i.id = s.instruction_id
|
|
56
|
+
left join projects proj on proj.id = s.project_id
|
|
57
|
+
where s.kind = 'add'
|
|
58
|
+
-- Default-minScore or insights-authored (owner directive 2026-08-09; the
|
|
59
|
+
-- 0.5-vs-0.8 self-inflicted deadlock is documented in loop.ts:86-98).
|
|
60
|
+
and (s.score >= 0.8 or i.frontmatter->>'source' = 'insights')
|
|
61
|
+
-- The single auto-adoptable set. TS mirror: `isAutoAdoptable()` in
|
|
62
|
+
-- apps/api/src/lib/insights/artifacts/types.ts — the agreement between the
|
|
63
|
+
-- two is asserted by apps/api/test/auto-accept-drift.test.ts.
|
|
64
|
+
and i.type in ('instruction', 'skill')
|
|
65
|
+
and i.archived = false
|
|
66
|
+
and s.accepted_at is null
|
|
67
|
+
and s.dismissed_at is null
|
|
68
|
+
-- Nobody answering is not a refusal (00203); measurement declines still are.
|
|
69
|
+
and (s.declined_at is null or s.declined_reason = 'surface_budget')
|
|
70
|
+
and s.is_stale = false
|
|
71
|
+
-- Consent, project override first (00170 lineage).
|
|
72
|
+
and coalesce(
|
|
73
|
+
proj.metadata->>'autoAcceptSuggestions',
|
|
74
|
+
p.auto_accept_suggestions::text
|
|
75
|
+
) = 'true'
|
|
76
|
+
-- WHERE an unattended writer may write: the canonical checkout, nowhere
|
|
77
|
+
-- else (00205). Read from the view, never re-derived (00186 invariant).
|
|
78
|
+
and exists (
|
|
79
|
+
select 1 from public.project_canonical_checkouts c
|
|
80
|
+
where c.path = s.project_path
|
|
81
|
+
)
|
|
82
|
+
-- The worker said this directory is not there (00202, 24h-expiring).
|
|
83
|
+
and not public.project_path_recently_missing(s.user_id, s.project_path);
|
|
84
|
+
|
|
85
|
+
grant select on public.auto_accept_candidate_suggestions to authenticated, service_role;
|
|
86
|
+
|
|
87
|
+
comment on view public.auto_accept_candidate_suggestions is
|
|
88
|
+
'The auto-accept producer''s candidate predicate, extracted so the producer (00208), the conformance monitor (00209) and diagnostics read ONE definition. Cheap subset only — the endpoint (loop.ts /auto-acceptable) remains the authority and adds settable limits, budgets, placeholder and experiment gates.';
|
|
89
|
+
|
|
90
|
+
-- The producer, rewritten to read the view. Behaviour-identical to 00205 by
|
|
91
|
+
-- construction: the predicate moved, the queue-state dedupe stayed. The nine
|
|
92
|
+
-- tests in apps/api/test/auto-accept-producer.test.ts are the proof.
|
|
93
|
+
create or replace function public.enqueue_auto_accept_suggestions()
|
|
94
|
+
returns integer
|
|
95
|
+
language plpgsql
|
|
96
|
+
security definer
|
|
97
|
+
set search_path = public
|
|
98
|
+
as $$
|
|
99
|
+
declare
|
|
100
|
+
queued integer;
|
|
101
|
+
begin
|
|
102
|
+
with eligible as (
|
|
103
|
+
select v.user_id, v.project_path
|
|
104
|
+
from public.auto_accept_candidate_suggestions v
|
|
105
|
+
where not exists (
|
|
106
|
+
select 1 from agent_jobs j
|
|
107
|
+
where j.task_kind = 'cron_auto_accept_suggestions'
|
|
108
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
109
|
+
and j.payload->>'projectPath' = v.project_path
|
|
110
|
+
and j.user_id = v.user_id
|
|
111
|
+
)
|
|
112
|
+
group by v.user_id, v.project_path
|
|
113
|
+
),
|
|
114
|
+
ins as (
|
|
115
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
116
|
+
select e.user_id,
|
|
117
|
+
'cron_auto_accept_suggestions',
|
|
118
|
+
'pending',
|
|
119
|
+
jsonb_build_object('projectPath', e.project_path),
|
|
120
|
+
'auto-accept:' || e.user_id::text || ':' || e.project_path
|
|
121
|
+
from eligible e
|
|
122
|
+
returning 1
|
|
123
|
+
)
|
|
124
|
+
select count(*)::int into queued from ins;
|
|
125
|
+
|
|
126
|
+
return queued;
|
|
127
|
+
end;
|
|
128
|
+
$$;
|
|
129
|
+
|
|
130
|
+
comment on function public.enqueue_auto_accept_suggestions() is
|
|
131
|
+
'Hourly producer for cron_auto_accept_suggestions. Candidate predicate lives in the auto_accept_candidate_suggestions view (00208) — one definition shared with the conformance monitor. History: 00197 (type gate), 00202 (missing paths), 00203 (silence declines), 00205 (canonical checkouts only).';
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
-- 00209 — the suggestion pipeline watches itself.
|
|
2
|
+
--
|
|
3
|
+
-- WHY. Nine gates between "proposal generated" and "proposal adopted" have now
|
|
4
|
+
-- been found one, two, three at a time, over six weeks, each by a human noticing
|
|
5
|
+
-- adoption had gone quiet and hand-writing a query. The mechanism that let every
|
|
6
|
+
-- one of them hide is identical and is recorded in four separate memories and
|
|
7
|
+
-- the 2026-08-27 census: each stage reports a single number — `nothing-eligible`,
|
|
8
|
+
-- `refused: 0`, `queued 0 rows` — that reads the same whether the pool is EMPTY
|
|
9
|
+
-- or FULL OF THINGS EVERY GATE REFUSES. A producer matching zero rows is
|
|
10
|
+
-- indistinguishable from a quiet night.
|
|
11
|
+
--
|
|
12
|
+
-- This function encodes each discovered failure SIGNATURE as a named check that
|
|
13
|
+
-- runs daily and lands as a notification (the 00147 pipeline_health_check
|
|
14
|
+
-- pattern: one consolidated, unread-deduped notification per user). The census
|
|
15
|
+
-- (scripts/dev/suggestion-gate-census.sql) is the manual deep-dive; this is the
|
|
16
|
+
-- alarm that says the deep-dive is needed.
|
|
17
|
+
--
|
|
18
|
+
-- DETECTION ONLY, on purpose. Nothing here mutates suggestions, opens gates or
|
|
19
|
+
-- writes files. Every incident in this pipeline's history that involved an
|
|
20
|
+
-- unattended writer (00205's near-miss of 14 skills into ~/.claude/skills/, the
|
|
21
|
+
-- armed-secrets workflows) argues that the automated half of "find and fix"
|
|
22
|
+
-- must be the FINDING. Fixes go through review.
|
|
23
|
+
--
|
|
24
|
+
-- Every check corresponds to a real, dated incident. A check nobody can trace
|
|
25
|
+
-- to a failure it would have caught is noise, and noise teaches people to
|
|
26
|
+
-- ignore the alarm.
|
|
27
|
+
|
|
28
|
+
-- Daily per-user counter snapshots, for the checks that detect CHRONIC
|
|
29
|
+
-- conditions (a flag that only ever grows) rather than point-in-time states.
|
|
30
|
+
create table if not exists public.pipeline_conformance_metrics (
|
|
31
|
+
id bigint generated always as identity primary key,
|
|
32
|
+
run_at timestamptz not null default now(),
|
|
33
|
+
user_id uuid not null references public.profiles(id) on delete cascade,
|
|
34
|
+
metric text not null,
|
|
35
|
+
value bigint not null
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
create index if not exists idx_conformance_metrics_lookup
|
|
39
|
+
on public.pipeline_conformance_metrics (user_id, metric, run_at desc);
|
|
40
|
+
|
|
41
|
+
alter table public.pipeline_conformance_metrics enable row level security;
|
|
42
|
+
|
|
43
|
+
-- Guarded so the file is safe to re-apply — the ledger-driven runner
|
|
44
|
+
-- (scripts/deploy/apply-migrations.sh) may re-run a file after a lost ledger,
|
|
45
|
+
-- and CREATE POLICY is the one statement here with no IF NOT EXISTS form.
|
|
46
|
+
drop policy if exists "conformance_metrics_read_own" on public.pipeline_conformance_metrics;
|
|
47
|
+
create policy "conformance_metrics_read_own" on public.pipeline_conformance_metrics
|
|
48
|
+
for select using (user_id = auth.uid());
|
|
49
|
+
|
|
50
|
+
comment on table public.pipeline_conformance_metrics is
|
|
51
|
+
'Daily per-user snapshots written by suggestion_pipeline_conformance() (00209). Exists so one-way-flag defects (gate 7: is_stale set and never cleared) are detectable as monotone growth instead of waiting for a human to notice.';
|
|
52
|
+
|
|
53
|
+
create or replace function public.suggestion_pipeline_conformance()
|
|
54
|
+
returns table (check_id text, user_id uuid, detail text)
|
|
55
|
+
language plpgsql
|
|
56
|
+
security definer
|
|
57
|
+
set search_path = public
|
|
58
|
+
as $$
|
|
59
|
+
begin
|
|
60
|
+
-- ── Snapshot metrics first, so the monotone checks below (and future ones)
|
|
61
|
+
-- have history even on days when every check is quiet. ────────────────────
|
|
62
|
+
insert into pipeline_conformance_metrics (user_id, metric, value)
|
|
63
|
+
select s.user_id, m.metric, m.value
|
|
64
|
+
from (select distinct is2.user_id from instruction_suggestions is2) s
|
|
65
|
+
cross join lateral (
|
|
66
|
+
select 'stale_open_rows' as metric,
|
|
67
|
+
(select count(*) from instruction_suggestions x
|
|
68
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
69
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
70
|
+
and x.is_stale) as value
|
|
71
|
+
union all
|
|
72
|
+
select 'undecided_add_rows',
|
|
73
|
+
(select count(*) from instruction_suggestions x
|
|
74
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
75
|
+
and x.accepted_at is null and x.dismissed_at is null)
|
|
76
|
+
union all
|
|
77
|
+
select 'auto_accept_candidates',
|
|
78
|
+
(select count(*) from auto_accept_candidate_suggestions v
|
|
79
|
+
where v.user_id = s.user_id)
|
|
80
|
+
union all
|
|
81
|
+
-- The 0.5–0.79 band: emitted by the scorer (recommend.ts floors at 0.5),
|
|
82
|
+
-- below the adoption default (0.8). Feed-only by construction — counted so
|
|
83
|
+
-- "open suggestions" stops being one number over two populations (census
|
|
84
|
+
-- cause 1, 2026-08-27: 393 of 477 open skills sat in this band).
|
|
85
|
+
select 'feed_only_band_rows',
|
|
86
|
+
(select count(*) from instruction_suggestions x
|
|
87
|
+
join instructions i on i.id = x.instruction_id
|
|
88
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
89
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
90
|
+
and i.type in ('instruction', 'skill')
|
|
91
|
+
and x.score < 0.8
|
|
92
|
+
and coalesce(i.frontmatter->>'source', '') <> 'insights')
|
|
93
|
+
union all
|
|
94
|
+
-- Undecided rows quiet for >90 days that the weekly prune can NEVER
|
|
95
|
+
-- collect, by design: they carry instruction_suggestion_events, the FK is
|
|
96
|
+
-- ON DELETE CASCADE, and those events are the only record the funnel's
|
|
97
|
+
-- numerator was ever zero (00180's comment names naive pruning as the
|
|
98
|
+
-- failure mode). Counted here instead of deleted: the census's 2026-08-27
|
|
99
|
+
-- "cause 4" proposed pruning them and was wrong — visibility is the fix
|
|
100
|
+
-- that does not destroy evidence.
|
|
101
|
+
select 'dormant_open_rows',
|
|
102
|
+
(select count(*) from instruction_suggestions x
|
|
103
|
+
where x.user_id = s.user_id and x.kind = 'add'
|
|
104
|
+
and x.accepted_at is null and x.dismissed_at is null
|
|
105
|
+
and x.generated_at < now() - interval '90 days'
|
|
106
|
+
and coalesce(
|
|
107
|
+
(select max(ev.created_at) from instruction_suggestion_events ev
|
|
108
|
+
where ev.suggestion_id = x.id),
|
|
109
|
+
x.generated_at
|
|
110
|
+
) < now() - interval '90 days')
|
|
111
|
+
) m;
|
|
112
|
+
|
|
113
|
+
return query
|
|
114
|
+
|
|
115
|
+
-- ── CHECK 1: minted_on_forbidden_path ──────────────────────────────────────
|
|
116
|
+
-- Incident: census 2026-08-27 — 258 of 685 open proposals sat on paths the
|
|
117
|
+
-- writer is forbidden to touch (home dir, TMPDIR, worktrees, the dead
|
|
118
|
+
-- apps/cli). Fixed the same day by gating both mint sites on
|
|
119
|
+
-- assessMintablePath(). This check is that fix's regression alarm: any NEW
|
|
120
|
+
-- row on a non-canonical path means a mint site was added or the gate broke.
|
|
121
|
+
select 'minted_on_forbidden_path'::text,
|
|
122
|
+
s.user_id,
|
|
123
|
+
count(*) || ' suggestion(s) minted in the last 25h on non-canonical path(s), e.g. ' || min(s.project_path)
|
|
124
|
+
from instruction_suggestions s
|
|
125
|
+
where s.generated_at > now() - interval '25 hours'
|
|
126
|
+
and not exists (
|
|
127
|
+
select 1 from project_canonical_checkouts c where c.path = s.project_path
|
|
128
|
+
)
|
|
129
|
+
group by s.user_id
|
|
130
|
+
|
|
131
|
+
union all
|
|
132
|
+
|
|
133
|
+
-- ── CHECK 2: producer_silent_with_candidates ───────────────────────────────
|
|
134
|
+
-- Incident: gate 4 (00184→00197) — the DB producer's type clause matched zero
|
|
135
|
+
-- rows, pg_cron fired and SUCCEEDED at 05:00 for three days, enqueued
|
|
136
|
+
-- nothing, and no log line existed anywhere. Candidates older than 25h with
|
|
137
|
+
-- no job even CREATED in 25h cannot be a quiet night: the producer is hourly.
|
|
138
|
+
select 'producer_silent_with_candidates'::text,
|
|
139
|
+
v.user_id,
|
|
140
|
+
count(*) || ' candidate(s) (oldest ' ||
|
|
141
|
+
extract(day from now() - min(v.generated_at)) || 'd) but no auto-accept job created in 25h — the producer is not seeing them'
|
|
142
|
+
from auto_accept_candidate_suggestions v
|
|
143
|
+
where v.generated_at < now() - interval '25 hours'
|
|
144
|
+
and not exists (
|
|
145
|
+
select 1 from agent_jobs j
|
|
146
|
+
where j.user_id = v.user_id
|
|
147
|
+
and j.task_kind = 'cron_auto_accept_suggestions'
|
|
148
|
+
and j.created_at > now() - interval '25 hours'
|
|
149
|
+
)
|
|
150
|
+
group by v.user_id
|
|
151
|
+
|
|
152
|
+
union all
|
|
153
|
+
|
|
154
|
+
-- ── CHECK 3: endpoint_refuses_everything ───────────────────────────────────
|
|
155
|
+
-- Incident: gates 1–4 (#372, 2026-08-16..19) — the producer queued jobs, the
|
|
156
|
+
-- worker ran them, the ENDPOINT refused every candidate on a clause the
|
|
157
|
+
-- producer didn't have, and four days of `nothing-eligible` looked like four
|
|
158
|
+
-- quiet nights. Signature: jobs completing against a standing candidate set
|
|
159
|
+
-- while NOTHING moves — no acceptance, no refusal event. A healthy run
|
|
160
|
+
-- always moves one of those; a placeholder-refusal loop that repeats forever
|
|
161
|
+
-- without logging also lands here, and deserves to.
|
|
162
|
+
select 'endpoint_refuses_everything'::text,
|
|
163
|
+
v.user_id,
|
|
164
|
+
count(distinct v.suggestion_id) ||
|
|
165
|
+
' candidate(s) standing >48h while auto-accept jobs completed — zero adoptions, zero refusal events. The producer and the endpoint disagree about eligibility.'
|
|
166
|
+
from auto_accept_candidate_suggestions v
|
|
167
|
+
where v.generated_at < now() - interval '48 hours'
|
|
168
|
+
and exists (
|
|
169
|
+
select 1 from agent_jobs j
|
|
170
|
+
where j.user_id = v.user_id
|
|
171
|
+
and j.task_kind = 'cron_auto_accept_suggestions'
|
|
172
|
+
and j.status = 'completed'
|
|
173
|
+
and j.updated_at > now() - interval '48 hours'
|
|
174
|
+
)
|
|
175
|
+
and not exists (
|
|
176
|
+
select 1 from instruction_suggestions a
|
|
177
|
+
where a.user_id = v.user_id
|
|
178
|
+
and a.accepted_at > now() - interval '48 hours'
|
|
179
|
+
)
|
|
180
|
+
and not exists (
|
|
181
|
+
select 1 from instruction_suggestion_events ev
|
|
182
|
+
join instruction_suggestions es on es.id = ev.suggestion_id
|
|
183
|
+
where es.user_id = v.user_id
|
|
184
|
+
and ev.event_type in ('redundant', 'partial_overlap', 'unresolvable_reference')
|
|
185
|
+
and ev.created_at > now() - interval '48 hours'
|
|
186
|
+
)
|
|
187
|
+
group by v.user_id
|
|
188
|
+
|
|
189
|
+
union all
|
|
190
|
+
|
|
191
|
+
-- ── CHECK 4: stale_flag_only_grows ─────────────────────────────────────────
|
|
192
|
+
-- Incident: gate 7 (2026-08-24) — is_stale was set by the mark-stale cron and
|
|
193
|
+
-- NOTHING ever cleared it; 51 adoption-grade proposals went invisible, and
|
|
194
|
+
-- one user's whole skill lane was structurally dead for weeks. A one-way flag
|
|
195
|
+
-- is invisible at any instant; over days it is a staircase. Fires when the
|
|
196
|
+
-- last 8 daily samples are all positive, never decrease, and end higher than
|
|
197
|
+
-- they started.
|
|
198
|
+
select 'stale_flag_only_grows'::text,
|
|
199
|
+
g.user_id,
|
|
200
|
+
'stale-flagged open rows grew ' || g.first_value || ' → ' || g.last_value ||
|
|
201
|
+
' over ' || g.samples || ' daily samples with no decrease — nothing is un-flagging them'
|
|
202
|
+
from (
|
|
203
|
+
select m.user_id,
|
|
204
|
+
count(*) as samples,
|
|
205
|
+
(array_agg(m.value order by m.run_at))[1] as first_value,
|
|
206
|
+
(array_agg(m.value order by m.run_at desc))[1] as last_value,
|
|
207
|
+
bool_and(m.value > 0) as all_positive,
|
|
208
|
+
-- Non-decreasing across the window: max of pairwise drops is 0.
|
|
209
|
+
coalesce(max(m.drop), 0) = 0 as never_decreased
|
|
210
|
+
from (
|
|
211
|
+
select mm.user_id, mm.run_at, mm.value,
|
|
212
|
+
greatest(lag(mm.value) over (partition by mm.user_id order by mm.run_at) - mm.value, 0) as drop
|
|
213
|
+
from pipeline_conformance_metrics mm
|
|
214
|
+
where mm.metric = 'stale_open_rows'
|
|
215
|
+
and mm.run_at > now() - interval '8 days'
|
|
216
|
+
) m
|
|
217
|
+
group by m.user_id
|
|
218
|
+
) g
|
|
219
|
+
where g.samples >= 8 and g.all_positive and g.never_decreased and g.last_value > g.first_value
|
|
220
|
+
|
|
221
|
+
union all
|
|
222
|
+
|
|
223
|
+
-- ── CHECK 5: surface_budget_spent_on_unadoptable ───────────────────────────
|
|
224
|
+
-- Incident: gates 5/6 and census cause 2 — ~840 feature/hook/mcp pitches per
|
|
225
|
+
-- 30 days entered the SessionStart feed, burned their five-ask surface budget
|
|
226
|
+
-- on an audience that cannot act on them, and converted into permanent
|
|
227
|
+
-- silence-declines. Fixed 2026-08-27 by excluding non-adoptable types from
|
|
228
|
+
-- the session-start render set (feed.ts). Any surfaced event on such a row
|
|
229
|
+
-- after that is the filter regressing.
|
|
230
|
+
select 'surface_budget_spent_on_unadoptable'::text,
|
|
231
|
+
s.user_id,
|
|
232
|
+
count(distinct ev.suggestion_id) ||
|
|
233
|
+
' non-adoptable proposal(s) (feature/hook/mcp/…) surfaced into sessions in the last 7d — they burn budget for an audience that cannot act'
|
|
234
|
+
from instruction_suggestion_events ev
|
|
235
|
+
join instruction_suggestions s on s.id = ev.suggestion_id
|
|
236
|
+
join instructions i on i.id = s.instruction_id
|
|
237
|
+
where ev.event_type in ('surfaced', 'surfaced_with_content')
|
|
238
|
+
and ev.created_at > now() - interval '7 days'
|
|
239
|
+
and s.kind = 'add'
|
|
240
|
+
and i.type not in ('instruction', 'skill')
|
|
241
|
+
group by s.user_id;
|
|
242
|
+
end;
|
|
243
|
+
$$;
|
|
244
|
+
|
|
245
|
+
comment on function public.suggestion_pipeline_conformance() is
|
|
246
|
+
'Daily conformance sweep over the suggestion/promotion pipeline. Each check encodes the observable signature of a dated incident (gate 4 silent producer, #372 producer/endpoint drift, gate 7 one-way staleness, census causes 2 and 3). Detection only — never mutates pipeline state. Findings land as one consolidated notification per user (pattern: 00147).';
|
|
247
|
+
|
|
248
|
+
-- Deliver findings the way pipeline_health_check does: one consolidated
|
|
249
|
+
-- notification per user, deduped while a previous one is still unread.
|
|
250
|
+
create or replace function public.run_suggestion_pipeline_conformance()
|
|
251
|
+
returns integer
|
|
252
|
+
language plpgsql
|
|
253
|
+
security definer
|
|
254
|
+
set search_path = public
|
|
255
|
+
as $$
|
|
256
|
+
declare
|
|
257
|
+
created integer := 0;
|
|
258
|
+
begin
|
|
259
|
+
with findings as (
|
|
260
|
+
select * from public.suggestion_pipeline_conformance()
|
|
261
|
+
),
|
|
262
|
+
by_user as (
|
|
263
|
+
select f.user_id as uid, count(*)::int as n,
|
|
264
|
+
string_agg('• [' || f.check_id || '] ' || f.detail, E'\n' order by f.check_id) as body
|
|
265
|
+
from findings f
|
|
266
|
+
group by f.user_id
|
|
267
|
+
),
|
|
268
|
+
inserted as (
|
|
269
|
+
insert into notifications (user_id, type, title, body, url, metadata)
|
|
270
|
+
select b.uid,
|
|
271
|
+
'suggestion_conformance',
|
|
272
|
+
'Suggestion pipeline conformance findings',
|
|
273
|
+
b.body,
|
|
274
|
+
'/loop',
|
|
275
|
+
jsonb_build_object('findings', b.n)
|
|
276
|
+
from by_user b
|
|
277
|
+
where not exists (
|
|
278
|
+
select 1 from notifications n
|
|
279
|
+
where n.user_id = b.uid
|
|
280
|
+
and n.type = 'suggestion_conformance'
|
|
281
|
+
and n.read_at is null
|
|
282
|
+
)
|
|
283
|
+
returning 1
|
|
284
|
+
)
|
|
285
|
+
select count(*)::int into created from inserted;
|
|
286
|
+
return created;
|
|
287
|
+
end;
|
|
288
|
+
$$;
|
|
289
|
+
|
|
290
|
+
-- 05:35 UTC — after eval_health_check (05:20) and pipeline_health_check
|
|
291
|
+
-- (05:25), so the morning's findings arrive together.
|
|
292
|
+
select cron.schedule(
|
|
293
|
+
'suggestion_pipeline_conformance',
|
|
294
|
+
'35 5 * * *',
|
|
295
|
+
$$select public.run_suggestion_pipeline_conformance();$$
|
|
296
|
+
);
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
-- 00210 — the eval-target health gates, defined once.
|
|
2
|
+
--
|
|
3
|
+
-- `enqueue_instruction_evolution` and `enqueue_eval_autorun` carried the three
|
|
4
|
+
-- health gates (2a proportional-ungradeable, 2b anchored-on-errored, 2c
|
|
5
|
+
-- mostly-failures) as byte-identical hand-synced copies — the evolution
|
|
6
|
+
-- producer's own comment admits it: "All copied byte-identical from the live
|
|
7
|
+
-- definition (00163's revision)". Two copies kept in step by discipline is the
|
|
8
|
+
-- exact mechanism behind the canonical-checkout incident (00186: five copies,
|
|
9
|
+
-- wrong in all five at once, silent) and the judge-segment incident (00193:
|
|
10
|
+
-- five copies, one credit-exhaustion fallback re-pointed the evidence base).
|
|
11
|
+
-- This repo has now paid for that lesson three times; the fix is the same each
|
|
12
|
+
-- time — one named object, every reader reads it.
|
|
13
|
+
--
|
|
14
|
+
-- WHAT IS SHARED AND WHAT IS NOT. Only 2a/2b/2c are shared. The live-cases
|
|
15
|
+
-- floor is NOT: evolution requires >= 3 (00183 — `exists` was the bug) while
|
|
16
|
+
-- autorun requires >= 1, and collapsing that difference here would silently
|
|
17
|
+
-- change enrolment. Pause flags, enrolment paths, change budgets and
|
|
18
|
+
-- in-flight dedupe stay with their producers.
|
|
19
|
+
--
|
|
20
|
+
-- Gate bodies below are copied VERBATIM from the live prosrc (2026-08-27).
|
|
21
|
+
-- Behaviour-identical by construction; the existing suites
|
|
22
|
+
-- (evolution-enrolment-rule.test.ts, eval-autorun-health-gate.test.ts) are
|
|
23
|
+
-- the proof.
|
|
24
|
+
|
|
25
|
+
create or replace view public.healthy_eval_targets as
|
|
26
|
+
select t.id as eval_target_id, t.user_id
|
|
27
|
+
from eval_targets t
|
|
28
|
+
where
|
|
29
|
+
-- Health 2a: recent grading wasn't MOSTLY ungradeable (proportional, 00163).
|
|
30
|
+
not exists (
|
|
31
|
+
select 1 from eval_runs r
|
|
32
|
+
where r.eval_target_id = t.id
|
|
33
|
+
and r.created_at > now() - interval '7 days'
|
|
34
|
+
group by r.eval_target_id
|
|
35
|
+
having count(*) >= 5
|
|
36
|
+
and count(*) filter (
|
|
37
|
+
where jsonb_typeof(r.grading->'expectations') = 'array'
|
|
38
|
+
and exists (
|
|
39
|
+
select 1 from jsonb_array_elements(r.grading->'expectations') e
|
|
40
|
+
where coalesce(e->>'errored', 'false') = 'true'
|
|
41
|
+
)
|
|
42
|
+
) * 4 >= count(*)
|
|
43
|
+
)
|
|
44
|
+
-- Health 2b: no human anchors pointing at an ungradeable assertion (absolute).
|
|
45
|
+
and not exists (
|
|
46
|
+
select 1
|
|
47
|
+
from eval_grade_annotations a
|
|
48
|
+
join eval_runs r on r.id = a.eval_run_id
|
|
49
|
+
where a.eval_target_id = t.id
|
|
50
|
+
and jsonb_typeof(r.grading->'expectations') = 'array'
|
|
51
|
+
and coalesce(
|
|
52
|
+
r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
|
|
53
|
+
) = 'true'
|
|
54
|
+
)
|
|
55
|
+
-- Health 2c: last week's batch wasn't mostly failures.
|
|
56
|
+
and not exists (
|
|
57
|
+
select 1 from eval_runs r
|
|
58
|
+
where r.eval_target_id = t.id
|
|
59
|
+
and r.created_at > now() - interval '7 days'
|
|
60
|
+
group by r.eval_target_id
|
|
61
|
+
having count(*) >= 5
|
|
62
|
+
and count(*) filter (where r.status = 'failed') * 4 >= count(*)
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
grant select on public.healthy_eval_targets to authenticated, service_role;
|
|
66
|
+
|
|
67
|
+
comment on view public.healthy_eval_targets is
|
|
68
|
+
'Eval targets passing the three shared health gates (2a proportional-ungradeable, 2b human anchor on an errored assertion, 2c mostly-failed batch). ONE definition (00210) read by both enqueue_instruction_evolution and enqueue_eval_autorun, which previously carried byte-identical hand-synced copies. Live-cases floors are deliberately NOT here — the two producers differ (>=3 vs >=1).';
|
|
69
|
+
|
|
70
|
+
-- ── enqueue_instruction_evolution, health gates now read from the view ───────
|
|
71
|
+
create or replace function public.enqueue_instruction_evolution()
|
|
72
|
+
returns integer
|
|
73
|
+
language plpgsql
|
|
74
|
+
security definer
|
|
75
|
+
set search_path = public
|
|
76
|
+
as $$
|
|
77
|
+
declare
|
|
78
|
+
queued integer;
|
|
79
|
+
begin
|
|
80
|
+
with health_ok as (
|
|
81
|
+
select t.id, t.user_id
|
|
82
|
+
from eval_targets t
|
|
83
|
+
where coalesce(t.metadata->>'autoEvolvePaused', 'false') <> 'true'
|
|
84
|
+
-- Health 2a/2b/2c: ONE definition (00210), shared with eval autorun.
|
|
85
|
+
and exists (
|
|
86
|
+
select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
|
|
87
|
+
)
|
|
88
|
+
-- Live cases to measure against — at least MIN_LIVE_CASES (3).
|
|
89
|
+
-- `exists` (>= 1) was the bug: see 00183's header.
|
|
90
|
+
and (
|
|
91
|
+
select count(*) from evals ev
|
|
92
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
93
|
+
) >= 3
|
|
94
|
+
-- Change budget (cheap exclusion; the API is the authority).
|
|
95
|
+
and (
|
|
96
|
+
select count(*)
|
|
97
|
+
from instruction_promotions p
|
|
98
|
+
where p.eval_target_id = t.id
|
|
99
|
+
and p.applied
|
|
100
|
+
and p.created_at > now() - interval '7 days'
|
|
101
|
+
) < 3
|
|
102
|
+
-- One in flight at a time.
|
|
103
|
+
and not exists (
|
|
104
|
+
select 1 from agent_jobs j
|
|
105
|
+
where j.task_kind = 'cron_instruction_evolution'
|
|
106
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
107
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
108
|
+
)
|
|
109
|
+
),
|
|
110
|
+
manual as (
|
|
111
|
+
-- The hand-picked path, unchanged and uncapped.
|
|
112
|
+
select h.id, h.user_id
|
|
113
|
+
from health_ok h
|
|
114
|
+
join eval_targets t on t.id = h.id
|
|
115
|
+
where coalesce(t.metadata->>'autoEvolveEnabled', 'false') = 'true'
|
|
116
|
+
),
|
|
117
|
+
auto_ranked as (
|
|
118
|
+
-- The stated rule. Round-robin: never-decided first (nulls first), then
|
|
119
|
+
-- least-recently-decided, so the cap rotates through the candidate pool
|
|
120
|
+
-- instead of re-measuring the same winners nightly.
|
|
121
|
+
select h.id, h.user_id,
|
|
122
|
+
row_number() over (
|
|
123
|
+
partition by h.user_id
|
|
124
|
+
order by (
|
|
125
|
+
select max(p.created_at)
|
|
126
|
+
from instruction_promotions p
|
|
127
|
+
where p.eval_target_id = h.id
|
|
128
|
+
) asc nulls first,
|
|
129
|
+
h.id
|
|
130
|
+
) as rn
|
|
131
|
+
from health_ok h
|
|
132
|
+
join eval_targets t on t.id = h.id
|
|
133
|
+
where t.type = 'instruction'
|
|
134
|
+
and coalesce(t.metadata->>'autoEvolveOptOut', 'false') <> 'true'
|
|
135
|
+
-- Not already covered by the manual path.
|
|
136
|
+
and coalesce(t.metadata->>'autoEvolveEnabled', 'false') <> 'true'
|
|
137
|
+
-- ACTIVE (00145's load-bearing gate): a run in the last 30 days.
|
|
138
|
+
and exists (
|
|
139
|
+
select 1 from eval_runs r
|
|
140
|
+
where r.eval_target_id = t.id
|
|
141
|
+
and r.created_at > now() - interval '30 days'
|
|
142
|
+
)
|
|
143
|
+
),
|
|
144
|
+
eligible as (
|
|
145
|
+
select id, user_id from manual
|
|
146
|
+
union
|
|
147
|
+
select id, user_id from auto_ranked where rn <= 3
|
|
148
|
+
),
|
|
149
|
+
ins as (
|
|
150
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
151
|
+
select e.user_id,
|
|
152
|
+
'cron_instruction_evolution',
|
|
153
|
+
'pending',
|
|
154
|
+
jsonb_build_object('evalTargetId', e.id::text)
|
|
155
|
+
|| case
|
|
156
|
+
when public.eval_target_judge_segment(e.id) is not null
|
|
157
|
+
then jsonb_build_object(
|
|
158
|
+
'graderModel', public.eval_target_judge_segment(e.id))
|
|
159
|
+
else '{}'::jsonb
|
|
160
|
+
end,
|
|
161
|
+
'instruction-evolution:' || e.id::text
|
|
162
|
+
from eligible e
|
|
163
|
+
returning 1
|
|
164
|
+
)
|
|
165
|
+
select count(*)::int into queued from ins;
|
|
166
|
+
|
|
167
|
+
return queued;
|
|
168
|
+
end;
|
|
169
|
+
$$;
|
|
170
|
+
|
|
171
|
+
comment on function public.enqueue_instruction_evolution() is
|
|
172
|
+
'Daily producer for cron_instruction_evolution. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); pause, live-cases floor (>=3, 00183), change budget, round-robin enrolment and in-flight dedupe stay here. Judge segment from eval_target_judge_segment() (00193).';
|
|
173
|
+
|
|
174
|
+
-- ── enqueue_eval_autorun, health gates now read from the view ────────────────
|
|
175
|
+
create or replace function public.enqueue_eval_autorun()
|
|
176
|
+
returns integer
|
|
177
|
+
language plpgsql
|
|
178
|
+
security definer
|
|
179
|
+
set search_path = public
|
|
180
|
+
as $$
|
|
181
|
+
declare
|
|
182
|
+
queued integer;
|
|
183
|
+
begin
|
|
184
|
+
with eligible as (
|
|
185
|
+
select t.id, t.user_id,
|
|
186
|
+
-- Canonical segment (00193). Was an inline "most recent stamped run"
|
|
187
|
+
-- subquery, which is how a credit-exhaustion fallback became the pin.
|
|
188
|
+
public.eval_target_judge_segment(t.id) as current_judge
|
|
189
|
+
from eval_targets t
|
|
190
|
+
where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
|
|
191
|
+
-- Gate 0 (00190): the target must be anchored to something a promotion
|
|
192
|
+
-- could land on. Without this, a batch is two LLM executions plus a
|
|
193
|
+
-- judged grade spent to rewrite a row nothing reads.
|
|
194
|
+
and (
|
|
195
|
+
t.instruction_id is not null
|
|
196
|
+
or t.project_path is not null
|
|
197
|
+
or t.name like '/%'
|
|
198
|
+
)
|
|
199
|
+
-- Gates 2a/2b/2c: ONE definition (00210), shared with evolution.
|
|
200
|
+
and exists (
|
|
201
|
+
select 1 from public.healthy_eval_targets h where h.eval_target_id = t.id
|
|
202
|
+
)
|
|
203
|
+
-- Gate 3: the target has live cases to actually run.
|
|
204
|
+
and exists (
|
|
205
|
+
select 1 from evals ev
|
|
206
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
207
|
+
)
|
|
208
|
+
-- One in flight at a time, per the 00083/00086 pattern.
|
|
209
|
+
and not exists (
|
|
210
|
+
select 1 from agent_jobs j
|
|
211
|
+
where j.task_kind = 'cron_eval_autorun'
|
|
212
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
213
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
214
|
+
)
|
|
215
|
+
), ins as (
|
|
216
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
217
|
+
select e.user_id,
|
|
218
|
+
'cron_eval_autorun',
|
|
219
|
+
'pending',
|
|
220
|
+
jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
|
|
221
|
+
|| case
|
|
222
|
+
when e.current_judge is not null
|
|
223
|
+
then jsonb_build_object('graderModel', e.current_judge)
|
|
224
|
+
else '{}'::jsonb
|
|
225
|
+
end,
|
|
226
|
+
'eval-autorun:' || e.id::text
|
|
227
|
+
from eligible e
|
|
228
|
+
returning 1
|
|
229
|
+
)
|
|
230
|
+
select count(*)::int into queued from ins;
|
|
231
|
+
|
|
232
|
+
return queued;
|
|
233
|
+
end;
|
|
234
|
+
$$;
|
|
235
|
+
|
|
236
|
+
comment on function public.enqueue_eval_autorun() is
|
|
237
|
+
'Daily producer for cron_eval_autorun. Health gates 2a/2b/2c come from the healthy_eval_targets view (00210); the anchor gate (00190), live-cases exists, judge segment (00193) and in-flight dedupe stay here.';
|