@rulemetric/local 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/meta.json +2 -2
- package/dist/server.mjs +685 -416
- package/dist/supabase/migrations/00087_cleanup_test_data_fix.sql +1 -1
- package/dist/supabase/migrations/00185_trial_throughput_settings.sql +43 -0
- package/dist/supabase/migrations/00186_canonical_checkout_prefers_live.sql +256 -0
- package/dist/supabase/migrations/00187_recommendation_applications.sql +32 -0
- package/dist/supabase/migrations/00188_training_producer_canonical_path.sql +117 -0
- package/dist/supabase/migrations/00189_limit_obs_window_index.sql +37 -0
- package/dist/supabase/migrations/00190_eval_targets_artifact_link.sql +176 -0
- package/dist/supabase/migrations/00191_recommendation_applications_effect_ledger.sql +106 -0
- package/dist/supabase/migrations/00192_cleanup_reaps_memberless_orgs.sql +142 -0
- package/dist/supabase/migrations/00193_judge_segment_pin_and_anchors.sql +367 -0
- package/dist/supabase/migrations/00194_refine_cases_producer.sql +91 -0
- package/dist/supabase/migrations/00195_skill_usage_views.sql +109 -0
- package/dist/supabase/migrations/00196_retire_unused_skills_producer.sql +102 -0
- package/dist/supabase/migrations/00197_auto_accept_producer_admits_skills.sql +115 -0
- package/dist/supabase/migrations/00198_unstale_non_catalog_suggestions.sql +51 -0
- package/dist/supabase/migrations/00199_auto_accept_runs_every_two_hours.sql +54 -0
- package/dist/supabase/migrations/00200_auto_accept_runs_hourly.sql +42 -0
- package/dist/web/assets/{docs-B_21wyvj.js → docs-KIeJWTb-.js} +1 -1
- package/dist/web/assets/index-CFJatBZ3.css +1 -0
- package/dist/web/assets/{index-OhhQFJmS.js → index-MYof2fnF.js} +54 -54
- package/dist/web/index.html +3 -3
- package/package.json +2 -2
- package/dist/web/assets/index-_M9l6iMX.css +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
-- Fix cleanup_stale_test_users: it failed nightly 2026-06-07..09 on
|
|
2
2
|
-- projects_created_by_fkey because projects / project_instructions gained
|
|
3
3
|
-- NO ACTION FKs to auth.users after 00062 was written. Also extend the
|
|
4
|
-
-- sweep to delete leaked smoke-test sessions — scripts/smoke-test.sh's
|
|
4
|
+
-- sweep to delete leaked smoke-test sessions — scripts/verify/smoke-test.sh's
|
|
5
5
|
-- hook-execution section creates sessions under the dev user
|
|
6
6
|
-- (external_session_id '00000000-0000-4000-8000-…') that the teardown
|
|
7
7
|
-- only closed, never deleted.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
-- Randomised measurement gets a throughput dial.
|
|
2
|
+
--
|
|
3
|
+
-- ## The complaint, and what actually caused it
|
|
4
|
+
--
|
|
5
|
+
-- "One a day per project takes forever." Correct, and it was not the
|
|
6
|
+
-- concurrency ceiling that caused it. Two constants in
|
|
7
|
+
-- `routes/instruction-suggestions/loop.ts` compose badly:
|
|
8
|
+
--
|
|
9
|
+
-- TRIAL_MAX_CONCURRENT = 3 how many rules may be under measurement at once
|
|
10
|
+
-- TRIAL_MAX_PER_RUN = 1 how many NEW trials one cycle may start
|
|
11
|
+
--
|
|
12
|
+
-- The producer runs nightly, so `per run` is really `per night`. Starting from
|
|
13
|
+
-- zero, a project needs THREE NIGHTS to reach a ceiling of three — the first
|
|
14
|
+
-- two nights leave measurement capacity deliberately idle. That is pure
|
|
15
|
+
-- latency: it buys no statistical protection, because the steady state it
|
|
16
|
+
-- ramps toward is the same either way.
|
|
17
|
+
--
|
|
18
|
+
-- The ceiling is a real trade and stays put by default: every running
|
|
19
|
+
-- experiment withholds its rule from ~half that project's sessions, so
|
|
20
|
+
-- concurrency is context the agent does not get. The RAMP is not a trade at
|
|
21
|
+
-- all. So the default for per-run rises to fill the ceiling in one cycle
|
|
22
|
+
-- (3 nights -> 1), and both numbers become settings for anyone who wants to
|
|
23
|
+
-- spend more context to answer more questions at once.
|
|
24
|
+
--
|
|
25
|
+
-- What this does NOT touch: GRADUATION_MIN_N (25 measured sessions per arm).
|
|
26
|
+
-- Sessions are the scarce resource and that gate is what keeps a verdict from
|
|
27
|
+
-- enshrining noise. Testing faster must mean asking more questions in
|
|
28
|
+
-- parallel, never accepting thinner evidence for each.
|
|
29
|
+
--
|
|
30
|
+
-- NULL = use the shipped default, so an untouched account tracks changes to
|
|
31
|
+
-- the default instead of freezing today's value at signup. Per-project
|
|
32
|
+
-- overrides live in `projects.metadata.trialLimits` and win over these.
|
|
33
|
+
-- Mirrors 00184's auto-accept budget settings exactly.
|
|
34
|
+
|
|
35
|
+
ALTER TABLE public.profiles
|
|
36
|
+
ADD COLUMN IF NOT EXISTS trial_max_concurrent integer,
|
|
37
|
+
ADD COLUMN IF NOT EXISTS trial_max_per_run integer;
|
|
38
|
+
|
|
39
|
+
COMMENT ON COLUMN public.profiles.trial_max_concurrent IS
|
|
40
|
+
'Account override for how many instruction experiments may run at once on one project. NULL = shipped default (3). Each running experiment withholds its rule from ~half that project''s sessions, so this is a context-vs-questions trade. Per-project override: projects.metadata.trialLimits.maxConcurrent.';
|
|
41
|
+
|
|
42
|
+
COMMENT ON COLUMN public.profiles.trial_max_per_run IS
|
|
43
|
+
'Account override for how many NEW experiments one training cycle may start. NULL = shipped default (3, i.e. fill the concurrency ceiling in a single run). Raising this shortens ramp latency only; it cannot exceed the concurrency ceiling, which is what actually bounds context withholding. Per-project override: projects.metadata.trialLimits.maxPerRun.';
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
-- One authority for "which path is this project", and it may never be a dead one.
|
|
2
|
+
--
|
|
3
|
+
-- THE DEFECT (measured against production 2026-08-13).
|
|
4
|
+
--
|
|
5
|
+
-- Five call sites resolved a project's canonical checkout with the same
|
|
6
|
+
-- hand-copied rule: `distinct on (project_id) ... order by is_worktree asc,
|
|
7
|
+
-- last_seen desc`. Non-worktree first, most-recent second. `project_checkouts`
|
|
8
|
+
-- for momento-mori holds three rows:
|
|
9
|
+
--
|
|
10
|
+
-- /Users/nickyeager/Code/agents/momento-mori is_worktree=TRUE 2026-08-10
|
|
11
|
+
-- /Users/nickyeager/Code/agents/momento-mori/.worktrees/… is_worktree=true 2026-05-04
|
|
12
|
+
-- /Users/nickyeager/Code/agents/momento-mori/apps/cli is_worktree=false 2026-05-01
|
|
13
|
+
--
|
|
14
|
+
-- Session ingest upserts `is_worktree` from each session's own metadata on
|
|
15
|
+
-- every create (routes/sessions/ingest/create.ts), so ONE session that reported
|
|
16
|
+
-- `isWorktree: true` while its projectPath was the repo root flipped the root's
|
|
17
|
+
-- flag. `is_worktree ASC` then outranks recency absolutely, and the crown went
|
|
18
|
+
-- to `apps/cli` — a path no session has rendered from since 2026-05-01.
|
|
19
|
+
--
|
|
20
|
+
-- Consequences, all silent, all of them looking like "nothing eligible":
|
|
21
|
+
--
|
|
22
|
+
-- * `regenerate_insights_jobs` analysed apps/cli every 6h from 2026-08-10
|
|
23
|
+
-- onward — 136 completed jobs re-deriving the same May-era conclusions
|
|
24
|
+
-- ("Across 13 CLI sessions…") while the repo they describe moved on.
|
|
25
|
+
-- * The insights-authored suggestions those runs produce are filed under
|
|
26
|
+
-- project_path=…/apps/cli, so `cron_auto_accept_suggestions` writes
|
|
27
|
+
-- apps/cli/CLAUDE.md and reports `nothing-eligible` for the repo root
|
|
28
|
+
-- every single night.
|
|
29
|
+
-- * 00175 pinned trial nomination to this same rule. Its own comment
|
|
30
|
+
-- diagnoses the apps/cli experiment as undead ("no session has rendered
|
|
31
|
+
-- from apps/cli in months") and then adopts the rule that elects it.
|
|
32
|
+
--
|
|
33
|
+
-- tinyworlds is mis-routed identically, to `…/tinyworlds/src` (2026-05-04).
|
|
34
|
+
--
|
|
35
|
+
-- THE RULE. Recency outranks the worktree flag, but only when the gap is
|
|
36
|
+
-- material: a path may not win if it is more than 30 days staler than the
|
|
37
|
+
-- freshest path that project has. Stated against the project's OWN freshest
|
|
38
|
+
-- path rather than against `now()`, so a project dormant for a year still
|
|
39
|
+
-- resolves to the checkout it was last worked in instead of falling back to
|
|
40
|
+
-- whichever dead sibling happens to carry is_worktree=false.
|
|
41
|
+
--
|
|
42
|
+
-- Within a staleness tier the original intent survives untouched: a real
|
|
43
|
+
-- checkout beats a worktree, and the most recently seen breaks the tie.
|
|
44
|
+
--
|
|
45
|
+
-- This is a VIEW rather than a sixth copy of the ORDER BY on purpose. The
|
|
46
|
+
-- rule has now been wrong in five places at once; the repo's own recorded
|
|
47
|
+
-- lesson for two-copies drift (00177's comment) is that the copies disagree
|
|
48
|
+
-- in the silent direction. TypeScript callers select from this view too.
|
|
49
|
+
|
|
50
|
+
create or replace view public.project_canonical_checkouts as
|
|
51
|
+
select distinct on (project_id)
|
|
52
|
+
project_id,
|
|
53
|
+
path,
|
|
54
|
+
is_worktree,
|
|
55
|
+
last_seen,
|
|
56
|
+
-- Exposed so callers can SAY the path is stale rather than silently
|
|
57
|
+
-- acting on it. /api/loop/status renders this; a canonical path that
|
|
58
|
+
-- has gone quiet while the project has not is how this defect hid.
|
|
59
|
+
(last_seen < freshest - interval '30 days') as is_stale,
|
|
60
|
+
freshest as project_last_seen
|
|
61
|
+
from (
|
|
62
|
+
select pc.project_id,
|
|
63
|
+
pc.path,
|
|
64
|
+
pc.is_worktree,
|
|
65
|
+
pc.last_seen,
|
|
66
|
+
max(pc.last_seen) over (partition by pc.project_id) as freshest
|
|
67
|
+
from public.project_checkouts pc
|
|
68
|
+
where pc.path is not null
|
|
69
|
+
and pc.path <> ''
|
|
70
|
+
) q
|
|
71
|
+
order by project_id,
|
|
72
|
+
(last_seen < freshest - interval '30 days') asc, -- live before stale
|
|
73
|
+
is_worktree asc, -- checkout before worktree
|
|
74
|
+
last_seen desc; -- then most recent
|
|
75
|
+
|
|
76
|
+
comment on view public.project_canonical_checkouts is
|
|
77
|
+
'The single authority for a project''s canonical checkout path. Ranks live paths above ones more than 30 days staler than the project''s freshest, THEN non-worktree, THEN most-recently-seen. Replaces the hand-copied `order by is_worktree asc, last_seen desc` in regenerate_insights_jobs, enqueue_instruction_training, routes/insights/jobs.ts and routes/sessions/query.ts (x3), which elected a path dead since 2026-05-01 for momento-mori and tinyworlds because one session flipped the live root''s is_worktree flag. See 00186.';
|
|
78
|
+
|
|
79
|
+
grant select on public.project_canonical_checkouts to authenticated, service_role;
|
|
80
|
+
|
|
81
|
+
-- ── The insights producer ───────────────────────────────────────────────────
|
|
82
|
+
-- Body unchanged from 00176 except that `canonical` now selects from the view.
|
|
83
|
+
|
|
84
|
+
create or replace function public.regenerate_insights_jobs(
|
|
85
|
+
p_min_sessions integer default 3,
|
|
86
|
+
p_freshness_hours integer default 6
|
|
87
|
+
)
|
|
88
|
+
returns text
|
|
89
|
+
language plpgsql
|
|
90
|
+
security definer
|
|
91
|
+
set search_path = public
|
|
92
|
+
as $$
|
|
93
|
+
declare
|
|
94
|
+
v_inserted int := 0;
|
|
95
|
+
v_skipped_fresh int := 0;
|
|
96
|
+
v_skipped_queued int := 0;
|
|
97
|
+
v_reclaimed int := 0;
|
|
98
|
+
v_id uuid;
|
|
99
|
+
r record;
|
|
100
|
+
begin
|
|
101
|
+
for r in
|
|
102
|
+
with eligible as (
|
|
103
|
+
select s.user_id, s.project_id
|
|
104
|
+
from sessions s
|
|
105
|
+
where s.project_id is not null
|
|
106
|
+
group by s.user_id, s.project_id
|
|
107
|
+
having count(*) >= p_min_sessions
|
|
108
|
+
)
|
|
109
|
+
select e.user_id, e.project_id, c.path as project_path
|
|
110
|
+
from eligible e
|
|
111
|
+
join public.project_canonical_checkouts c on c.project_id = e.project_id
|
|
112
|
+
-- ACTIVITY since this project's last analysis — start, progress, or end.
|
|
113
|
+
where exists (
|
|
114
|
+
select 1 from sessions s2
|
|
115
|
+
where s2.user_id = e.user_id
|
|
116
|
+
and s2.project_id = e.project_id
|
|
117
|
+
and greatest(
|
|
118
|
+
s2.started_at,
|
|
119
|
+
coalesce(s2.ended_at, s2.started_at)
|
|
120
|
+
) > coalesce(
|
|
121
|
+
(select max(j.completed_at)
|
|
122
|
+
from insights_jobs j
|
|
123
|
+
where j.user_id = e.user_id
|
|
124
|
+
and j.project_path = c.path
|
|
125
|
+
and j.status = 'completed'),
|
|
126
|
+
now() - interval '30 days'
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
loop
|
|
130
|
+
if exists (
|
|
131
|
+
select 1 from insights_jobs j
|
|
132
|
+
where j.user_id = r.user_id
|
|
133
|
+
and j.project_path = r.project_path
|
|
134
|
+
and j.status = 'completed'
|
|
135
|
+
and j.completed_at > now() - (p_freshness_hours || ' hours')::interval
|
|
136
|
+
) then
|
|
137
|
+
v_skipped_fresh := v_skipped_fresh + 1;
|
|
138
|
+
continue;
|
|
139
|
+
end if;
|
|
140
|
+
|
|
141
|
+
update insights_jobs j
|
|
142
|
+
set status = 'failed',
|
|
143
|
+
completed_at = now(),
|
|
144
|
+
error = 'reclaimed by regenerate_insights_jobs: queued/in-flight >24h with no progress — worker likely died before any state write'
|
|
145
|
+
where j.user_id = r.user_id
|
|
146
|
+
and j.project_path = r.project_path
|
|
147
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
148
|
+
and j.created_at < now() - interval '24 hours';
|
|
149
|
+
if found then
|
|
150
|
+
v_reclaimed := v_reclaimed + 1;
|
|
151
|
+
end if;
|
|
152
|
+
|
|
153
|
+
if exists (
|
|
154
|
+
select 1 from insights_jobs j
|
|
155
|
+
where j.user_id = r.user_id
|
|
156
|
+
and j.project_path = r.project_path
|
|
157
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
158
|
+
) then
|
|
159
|
+
v_skipped_queued := v_skipped_queued + 1;
|
|
160
|
+
continue;
|
|
161
|
+
end if;
|
|
162
|
+
|
|
163
|
+
v_id := gen_random_uuid();
|
|
164
|
+
insert into insights_jobs (id, user_id, project_id, status, project_path, task_kind, payload)
|
|
165
|
+
values (
|
|
166
|
+
v_id,
|
|
167
|
+
r.user_id,
|
|
168
|
+
r.project_id,
|
|
169
|
+
'pending',
|
|
170
|
+
r.project_path,
|
|
171
|
+
'process_insights',
|
|
172
|
+
jsonb_build_object('mirrorId', v_id, 'projectPath', r.project_path)
|
|
173
|
+
);
|
|
174
|
+
v_inserted := v_inserted + 1;
|
|
175
|
+
end loop;
|
|
176
|
+
|
|
177
|
+
return format(
|
|
178
|
+
'regenerated: %s inserted, %s skipped (fresh), %s skipped (queued), %s reclaimed (stale)',
|
|
179
|
+
v_inserted, v_skipped_fresh, v_skipped_queued, v_reclaimed
|
|
180
|
+
);
|
|
181
|
+
end;
|
|
182
|
+
$$;
|
|
183
|
+
|
|
184
|
+
comment on function public.regenerate_insights_jobs(integer, integer) is
|
|
185
|
+
'Every-6h producer for process_insights. Enqueues one job per (user, project) with >= p_min_sessions sessions and activity since that project''s last completed analysis. As of 00186 the canonical path comes from public.project_canonical_checkouts, so the analysis can no longer be aimed at a checkout dead for months.';
|
|
186
|
+
|
|
187
|
+
-- ── The training producer ───────────────────────────────────────────────────
|
|
188
|
+
-- Body unchanged from 00175 except that `canonical` now selects from the view.
|
|
189
|
+
-- The settle half stays ungated and per-path so off-canonical experiments
|
|
190
|
+
-- retire rather than orphan.
|
|
191
|
+
|
|
192
|
+
create or replace function public.enqueue_instruction_training(p_active_days integer default 7)
|
|
193
|
+
returns integer
|
|
194
|
+
language plpgsql
|
|
195
|
+
security definer
|
|
196
|
+
set search_path = public
|
|
197
|
+
as $$
|
|
198
|
+
declare
|
|
199
|
+
queued integer;
|
|
200
|
+
begin
|
|
201
|
+
with eligible as (
|
|
202
|
+
select s.user_id, s.project_path
|
|
203
|
+
from instruction_suggestions s
|
|
204
|
+
join public.project_canonical_checkouts c on c.path = s.project_path
|
|
205
|
+
where s.kind = 'add'
|
|
206
|
+
and s.accepted_at is null
|
|
207
|
+
and s.dismissed_at is null
|
|
208
|
+
and s.declined_at is null
|
|
209
|
+
and s.is_stale = false
|
|
210
|
+
and exists (
|
|
211
|
+
select 1 from sessions se
|
|
212
|
+
where se.project_id = c.project_id
|
|
213
|
+
and se.user_id = s.user_id
|
|
214
|
+
and se.started_at > now() - (p_active_days || ' days')::interval
|
|
215
|
+
)
|
|
216
|
+
and not exists (
|
|
217
|
+
select 1 from agent_jobs j
|
|
218
|
+
where j.task_kind = 'cron_instruction_training'
|
|
219
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
220
|
+
and j.payload->>'projectPath' = s.project_path
|
|
221
|
+
and j.user_id = s.user_id
|
|
222
|
+
)
|
|
223
|
+
group by s.user_id, s.project_path
|
|
224
|
+
|
|
225
|
+
union
|
|
226
|
+
|
|
227
|
+
select e.user_id, e.project_path
|
|
228
|
+
from instruction_experiments e
|
|
229
|
+
where e.status = 'running'
|
|
230
|
+
and not exists (
|
|
231
|
+
select 1 from agent_jobs j
|
|
232
|
+
where j.task_kind = 'cron_instruction_training'
|
|
233
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
234
|
+
and j.payload->>'projectPath' = e.project_path
|
|
235
|
+
and j.user_id = e.user_id
|
|
236
|
+
)
|
|
237
|
+
group by e.user_id, e.project_path
|
|
238
|
+
),
|
|
239
|
+
ins as (
|
|
240
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
241
|
+
select e.user_id,
|
|
242
|
+
'cron_instruction_training',
|
|
243
|
+
'pending',
|
|
244
|
+
jsonb_build_object('projectPath', e.project_path),
|
|
245
|
+
'instruction-training:' || e.user_id::text || ':' || e.project_path
|
|
246
|
+
from eligible e
|
|
247
|
+
returning 1
|
|
248
|
+
)
|
|
249
|
+
select count(*)::int into queued from ins;
|
|
250
|
+
|
|
251
|
+
return queued;
|
|
252
|
+
end;
|
|
253
|
+
$$;
|
|
254
|
+
|
|
255
|
+
comment on function public.enqueue_instruction_training(integer) is
|
|
256
|
+
'Nightly producer for cron_instruction_training. The TRIAL half nominates only on a project''s canonical checkout path with a session inside p_active_days (default 7); SETTLE stays ungated and per-path so existing off-path experiments retire rather than orphan. As of 00186 canonical comes from public.project_canonical_checkouts — 00175 pinned this to a rule that elected the very apps/cli path its own comment describes as undead.';
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
create table public.recommendation_applications (
|
|
2
|
+
id uuid primary key default gen_random_uuid(),
|
|
3
|
+
user_id uuid not null references public.profiles(id) on delete cascade,
|
|
4
|
+
instruction_id uuid not null references public.instructions(id) on delete cascade,
|
|
5
|
+
project_id uuid references public.projects(id) on delete set null,
|
|
6
|
+
project_path text not null,
|
|
7
|
+
harness text not null check (harness in ('claude_code', 'codex', 'shared')),
|
|
8
|
+
target_path text not null,
|
|
9
|
+
status text not null default 'proposed' check (status in ('proposed', 'ready', 'applied', 'measuring', 'improved', 'inconclusive', 'harmful', 'rollback_ready', 'reverted', 'failed')),
|
|
10
|
+
apply_mode text not null check (apply_mode in ('auto', 'human')),
|
|
11
|
+
before_hash text,
|
|
12
|
+
applied_hash text,
|
|
13
|
+
applied_at timestamptz,
|
|
14
|
+
measurement_started_at timestamptz,
|
|
15
|
+
measurement_ended_at timestamptz,
|
|
16
|
+
verdict text check (verdict is null or verdict in ('improved', 'inconclusive', 'harmful')),
|
|
17
|
+
verdict_detail text,
|
|
18
|
+
reverted_at timestamptz,
|
|
19
|
+
created_at timestamptz not null default now(),
|
|
20
|
+
updated_at timestamptz not null default now(),
|
|
21
|
+
unique (user_id, instruction_id, project_path, harness, target_path)
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
create index idx_recommendation_applications_user_status on public.recommendation_applications(user_id, status, updated_at desc);
|
|
25
|
+
create index idx_recommendation_applications_user_harness on public.recommendation_applications(user_id, harness, updated_at desc);
|
|
26
|
+
create index idx_recommendation_applications_project on public.recommendation_applications(project_id, updated_at desc);
|
|
27
|
+
|
|
28
|
+
alter table public.recommendation_applications enable row level security;
|
|
29
|
+
create policy "Users manage own recommendation applications"
|
|
30
|
+
on public.recommendation_applications for all
|
|
31
|
+
using (auth.uid() = user_id)
|
|
32
|
+
with check (auth.uid() = user_id);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
-- The training producer's ACTIVE gate passes for paths nobody works in.
|
|
2
|
+
--
|
|
3
|
+
-- ## Measured 2026-08-14
|
|
4
|
+
--
|
|
5
|
+
-- 00174 added the gate that stops undead trials, and it resolves
|
|
6
|
+
-- `suggestion.project_path → project_checkouts → project_id → sessions`. That
|
|
7
|
+
-- last hop is the defect: it asks "has this PROJECT been active?", not "has
|
|
8
|
+
-- this PATH been active?". Every checkout of an active repo therefore passes,
|
|
9
|
+
-- including ones with no session in months.
|
|
10
|
+
--
|
|
11
|
+
-- Concretely, `/Users/nickyeager/Code/agents/momento-mori/apps/cli` maps to the
|
|
12
|
+
-- momento-mori project, whose ROOT is worked in daily, so the subpath cleared
|
|
13
|
+
-- the gate and the producer enqueued a training job for it every night. Those
|
|
14
|
+
-- jobs nominated real trials against a checkout with no session since
|
|
15
|
+
-- 2026-05-01: 6 running experiments were sitting on it tonight, holding slots,
|
|
16
|
+
-- unable to ever enroll an arm — the exact "undead" failure 00174 was written
|
|
17
|
+
-- to end, arriving through the one hop it did not close. (They were stopped by
|
|
18
|
+
-- hand under `inactive_project`; this stops them being created again.)
|
|
19
|
+
--
|
|
20
|
+
-- ## The rule
|
|
21
|
+
--
|
|
22
|
+
-- A suggestion's path may start a trial only if it IS the project's canonical
|
|
23
|
+
-- checkout. `public.project_canonical_checkouts` (00186) is the single
|
|
24
|
+
-- authority — CLAUDE.md's hard invariant is that this is never re-derived, and
|
|
25
|
+
-- 00174's hand-rolled join is precisely a re-derivation. The view already ranks
|
|
26
|
+
-- live-before-stale, so "canonical" cannot silently mean a dead path.
|
|
27
|
+
--
|
|
28
|
+
-- The per-path activity check is KEPT as well: canonical and recently active
|
|
29
|
+
-- are different claims, and a project that has gone quiet entirely should still
|
|
30
|
+
-- start no new trials.
|
|
31
|
+
--
|
|
32
|
+
-- The SETTLE half stays ungated, unchanged, for 00174's reason: gating it would
|
|
33
|
+
-- orphan an experiment whose project went quiet mid-trial.
|
|
34
|
+
|
|
35
|
+
create or replace function public.enqueue_instruction_training(p_active_days integer default 7)
|
|
36
|
+
returns integer
|
|
37
|
+
language plpgsql
|
|
38
|
+
security definer
|
|
39
|
+
set search_path = public
|
|
40
|
+
as $$
|
|
41
|
+
declare
|
|
42
|
+
queued integer;
|
|
43
|
+
begin
|
|
44
|
+
with eligible as (
|
|
45
|
+
select s.user_id, s.project_path
|
|
46
|
+
from instruction_suggestions s
|
|
47
|
+
where s.kind = 'add'
|
|
48
|
+
and s.accepted_at is null
|
|
49
|
+
and s.dismissed_at is null
|
|
50
|
+
and s.declined_at is null
|
|
51
|
+
and s.is_stale = false
|
|
52
|
+
-- CANONICAL (new in 00188): this path is the project's canonical
|
|
53
|
+
-- checkout, per the view that owns that answer. Without it, every
|
|
54
|
+
-- subdirectory of an active repo is treated as its own live project.
|
|
55
|
+
and exists (
|
|
56
|
+
select 1
|
|
57
|
+
from project_checkouts pc
|
|
58
|
+
join project_canonical_checkouts c on c.project_id = pc.project_id
|
|
59
|
+
where pc.path = s.project_path
|
|
60
|
+
and c.path = s.project_path
|
|
61
|
+
)
|
|
62
|
+
-- ACTIVE (00174, unchanged in intent): the project produced a session
|
|
63
|
+
-- inside the window. An experiment enrolls sessions at render time, so a
|
|
64
|
+
-- project with none coming can only ever produce an undead trial.
|
|
65
|
+
and exists (
|
|
66
|
+
select 1
|
|
67
|
+
from project_checkouts pc
|
|
68
|
+
join sessions se on se.project_id = pc.project_id
|
|
69
|
+
where pc.path = s.project_path
|
|
70
|
+
and se.user_id = s.user_id
|
|
71
|
+
and se.started_at > now() - (p_active_days || ' days')::interval
|
|
72
|
+
)
|
|
73
|
+
and not exists (
|
|
74
|
+
select 1 from agent_jobs j
|
|
75
|
+
where j.task_kind = 'cron_instruction_training'
|
|
76
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
77
|
+
and j.payload->>'projectPath' = s.project_path
|
|
78
|
+
and j.user_id = s.user_id
|
|
79
|
+
)
|
|
80
|
+
group by s.user_id, s.project_path
|
|
81
|
+
|
|
82
|
+
union
|
|
83
|
+
|
|
84
|
+
-- Settle half: deliberately NOT gated (neither on activity nor on
|
|
85
|
+
-- canonicality). One cheap API call per project per night, and gating it
|
|
86
|
+
-- would orphan a running experiment the moment its project went quiet — or,
|
|
87
|
+
-- with the new gate, the moment its checkout stopped being canonical.
|
|
88
|
+
select e.user_id, e.project_path
|
|
89
|
+
from instruction_experiments e
|
|
90
|
+
where e.status = 'running'
|
|
91
|
+
and not exists (
|
|
92
|
+
select 1 from agent_jobs j
|
|
93
|
+
where j.task_kind = 'cron_instruction_training'
|
|
94
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
95
|
+
and j.payload->>'projectPath' = e.project_path
|
|
96
|
+
and j.user_id = e.user_id
|
|
97
|
+
)
|
|
98
|
+
group by e.user_id, e.project_path
|
|
99
|
+
),
|
|
100
|
+
ins as (
|
|
101
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
102
|
+
select e.user_id,
|
|
103
|
+
'cron_instruction_training',
|
|
104
|
+
'pending',
|
|
105
|
+
jsonb_build_object('projectPath', e.project_path),
|
|
106
|
+
'instruction-training:' || e.user_id::text || ':' || e.project_path
|
|
107
|
+
from eligible e
|
|
108
|
+
returning 1
|
|
109
|
+
)
|
|
110
|
+
select count(*)::int into queued from ins;
|
|
111
|
+
|
|
112
|
+
return queued;
|
|
113
|
+
end;
|
|
114
|
+
$$;
|
|
115
|
+
|
|
116
|
+
comment on function public.enqueue_instruction_training(integer) is
|
|
117
|
+
'Nightly producer for cron_instruction_training. TRIAL half requires (a) the path IS the project canonical checkout per project_canonical_checkouts (00188 — the project-level activity join in 00174 let every subpath of an active repo through, and apps/cli collected 6 undead trials that way), and (b) a session on the project within p_active_days (default 7). SETTLE half stays ungated so a quiet or non-canonical project cannot orphan a running experiment. Widen deliberately: select enqueue_instruction_training(30).';
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
-- 00189: make GET /api/usage/current-limits index-bounded instead of
|
|
2
|
+
-- partition-scanning.
|
|
3
|
+
--
|
|
4
|
+
-- Measured on prod 2026-08-15 with EXPLAIN (ANALYZE, BUFFERS): 133 ms and
|
|
5
|
+
-- 155,494 shared buffer hits PER CALL, x 122,479 calls = 19.0B buffer hits.
|
|
6
|
+
-- That matched the 19.3B in pg_stat_statements exactly and made this the #1
|
|
7
|
+
-- query by total DB time (25.3%) — ahead of realtime WAL decode and far ahead
|
|
8
|
+
-- of the context_snapshots inserts everyone was looking at.
|
|
9
|
+
--
|
|
10
|
+
-- The cost was never the row count of the ANSWER (3 rows). It was that both
|
|
11
|
+
-- LATERAL probes read the ENTIRE (user, provider, source_type, limit_type)
|
|
12
|
+
-- partition — 37,946 index tuples x 3 limit types x 2 probes = 227,676 tuples
|
|
13
|
+
-- to return 3 rows:
|
|
14
|
+
-- * probe 1 is `max(resets_at)`, an aggregate with no LIMIT, so it must visit
|
|
15
|
+
-- every row in the prefix;
|
|
16
|
+
-- * probe 2 filters `resets_at >= <window> - interval '5 minutes'`, which is
|
|
17
|
+
-- NOT sargable against idx_limit_obs_current — that index orders its
|
|
18
|
+
-- trailing column by observed_at, so resets_at is only ever a filter.
|
|
19
|
+
--
|
|
20
|
+
-- Ordering the trailing column by resets_at makes both probes index-bounded:
|
|
21
|
+
-- probe 1 becomes ORDER BY ... LIMIT 1 (one tuple), probe 2 becomes a range
|
|
22
|
+
-- scan over just the current window. NULLS LAST is load-bearing — Postgres
|
|
23
|
+
-- defaults DESC to NULLS FIRST, which would put legacy null-window rows at the
|
|
24
|
+
-- head and make `ORDER BY resets_at DESC LIMIT 1` return NULL instead of the
|
|
25
|
+
-- live window.
|
|
26
|
+
--
|
|
27
|
+
-- INCLUDE (value, observed_at) keeps it index-only: those are the only other
|
|
28
|
+
-- columns the endpoint reads.
|
|
29
|
+
--
|
|
30
|
+
-- idx_limit_obs_current is NOT dropped — the legacy null-window fallback path
|
|
31
|
+
-- still orders by observed_at DESC, and the ingest dedup path uses it too.
|
|
32
|
+
--
|
|
33
|
+
-- CONCURRENTLY; run OUTSIDE a transaction (same convention as 00094/00182).
|
|
34
|
+
|
|
35
|
+
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_limit_obs_window
|
|
36
|
+
ON limit_observations (user_id, provider, source_type, limit_type, resets_at DESC NULLS LAST)
|
|
37
|
+
INCLUDE (value, observed_at);
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
-- Give an eval target a real link to the artifact it measures.
|
|
2
|
+
--
|
|
3
|
+
-- THE STRUCTURAL DEFECT
|
|
4
|
+
--
|
|
5
|
+
-- `eval_targets` stored no reference to the thing under test: no
|
|
6
|
+
-- `instruction_id`, no project. Just a COPY of the content and a `name` string.
|
|
7
|
+
-- "Which artifact does this target measure?" was answered by parsing the name —
|
|
8
|
+
-- if it starts with '/', treat it as a file path — a convention enforced
|
|
9
|
+
-- nowhere that the actuator nonetheless depends on
|
|
10
|
+
-- (`cron-instruction-evolution.ts`: `target.name.startsWith('/') && existsSync`).
|
|
11
|
+
--
|
|
12
|
+
-- When the name is not a path, the loop still measures the target and still
|
|
13
|
+
-- "promotes" it; the promotion writes back to `eval_targets.content`, the
|
|
14
|
+
-- target's own copy, which nothing downstream reads. Measured 2026-08-15:
|
|
15
|
+
-- `TypeScript Best Practices` — a 129-character demo row seeded 2026-03-10 —
|
|
16
|
+
-- was one of only 2 autorun-enabled targets out of 251, write-enabled, and had
|
|
17
|
+
-- burned 102 eval runs producing edits no session could ever load.
|
|
18
|
+
--
|
|
19
|
+
-- The link was available and discarded. `sessions/linking.ts` auto-creates
|
|
20
|
+
-- targets from captured instructions (169 of 197 instruction targets came from
|
|
21
|
+
-- there); it SELECTs `instructions` by `inst.id` to read the content, then
|
|
22
|
+
-- stores only `inst.name`. The session — and therefore the project — is in
|
|
23
|
+
-- scope in the same function.
|
|
24
|
+
--
|
|
25
|
+
-- WHY project_path RATHER THAN project_id
|
|
26
|
+
--
|
|
27
|
+
-- The trial half of the system already models this correctly:
|
|
28
|
+
-- `instruction_experiments` carries `instruction_id` + `project_path`, and its
|
|
29
|
+
-- verdicts (`instruction_verdicts`) carry the same pair. Skills are globally
|
|
30
|
+
-- defined (scope 'public') but their EFFECTIVENESS is per-project — the same
|
|
31
|
+
-- skill helps in one repo and not another. Matching that table's shape keeps
|
|
32
|
+
-- the two measurement modes joinable on the same keys. (Modes A and B are
|
|
33
|
+
-- complementary, not redundant: a repo file cannot be withheld from real work,
|
|
34
|
+
-- so CLAUDE.md is Mode A only, while a skill can be, so it is trialled.)
|
|
35
|
+
--
|
|
36
|
+
-- Both columns are NULLABLE and nothing is backfilled. 249 of 251 targets are
|
|
37
|
+
-- not enrolled in anything; guessing their provenance from a name would be
|
|
38
|
+
-- inventing data. The gate below applies only where promotions can happen.
|
|
39
|
+
|
|
40
|
+
alter table public.eval_targets
|
|
41
|
+
add column if not exists instruction_id uuid references public.instructions(id) on delete set null,
|
|
42
|
+
add column if not exists project_path text;
|
|
43
|
+
|
|
44
|
+
comment on column public.eval_targets.instruction_id is
|
|
45
|
+
'The instruction this target measures. Null for targets whose provenance was never recorded (everything created before 00190) and for prompt-type targets captured from a session. Populated at auto-creation, where the id is already in hand.';
|
|
46
|
+
|
|
47
|
+
comment on column public.eval_targets.project_path is
|
|
48
|
+
'Project the target is measured FOR. Mirrors instruction_experiments.project_path so the eval (Mode A) and trial (Mode B) halves join on the same keys. A public skill is global; its effectiveness is not.';
|
|
49
|
+
|
|
50
|
+
create index if not exists idx_eval_targets_instruction
|
|
51
|
+
on public.eval_targets (instruction_id) where instruction_id is not null;
|
|
52
|
+
|
|
53
|
+
create index if not exists idx_eval_targets_project
|
|
54
|
+
on public.eval_targets (project_path) where project_path is not null;
|
|
55
|
+
|
|
56
|
+
-- ---------------------------------------------------------------------------
|
|
57
|
+
-- The gate, at the producer — the same place 00188 gated the training producer
|
|
58
|
+
-- on the canonical checkout, and for the same reason: enrolment is a manual
|
|
59
|
+
-- metadata toggle, so the producer is the only chokepoint every scheduled batch
|
|
60
|
+
-- must pass through.
|
|
61
|
+
--
|
|
62
|
+
-- A target may only consume the autorun budget if the loop can name what a
|
|
63
|
+
-- promotion would land on. Two ways to qualify:
|
|
64
|
+
--
|
|
65
|
+
-- (a) it carries a link (instruction_id or project_path), or
|
|
66
|
+
-- (b) it is named by an absolute path — the legacy convention, kept so the
|
|
67
|
+
-- 163 file-anchored targets that predate this migration keep working
|
|
68
|
+
-- without a backfill.
|
|
69
|
+
--
|
|
70
|
+
-- A target with neither is refused. That is exactly the demo row: no
|
|
71
|
+
-- instruction, no project, and a name that is a label rather than a path.
|
|
72
|
+
-- ---------------------------------------------------------------------------
|
|
73
|
+
|
|
74
|
+
create or replace function public.enqueue_eval_autorun()
|
|
75
|
+
returns integer
|
|
76
|
+
language plpgsql
|
|
77
|
+
security definer
|
|
78
|
+
set search_path = public
|
|
79
|
+
as $$
|
|
80
|
+
declare
|
|
81
|
+
queued integer;
|
|
82
|
+
begin
|
|
83
|
+
with eligible as (
|
|
84
|
+
select t.id, t.user_id,
|
|
85
|
+
-- Current judge for this target = most recent stamped run, mirroring
|
|
86
|
+
-- judge-calibration's own resolution (created_at desc, model desc for a
|
|
87
|
+
-- stable tie-break).
|
|
88
|
+
(
|
|
89
|
+
select r.grading->>'judge_model'
|
|
90
|
+
from eval_runs r
|
|
91
|
+
where r.eval_target_id = t.id
|
|
92
|
+
and r.grading->>'judge_model' is not null
|
|
93
|
+
order by r.created_at desc, r.grading->>'judge_model' desc
|
|
94
|
+
limit 1
|
|
95
|
+
) as current_judge
|
|
96
|
+
from eval_targets t
|
|
97
|
+
where coalesce(t.metadata->>'autoRunEnabled', 'false') = 'true'
|
|
98
|
+
-- Gate 0 (00190): the target must be anchored to something a promotion
|
|
99
|
+
-- could land on. Without this, a batch is two LLM executions plus a
|
|
100
|
+
-- judged grade spent to rewrite a row nothing reads.
|
|
101
|
+
and (
|
|
102
|
+
t.instruction_id is not null
|
|
103
|
+
or t.project_path is not null
|
|
104
|
+
or t.name like '/%'
|
|
105
|
+
)
|
|
106
|
+
-- Gate 2a: recent grading wasn't MOSTLY ungradeable (proportional, and
|
|
107
|
+
-- identical to the evolution producer's 2a since 00163).
|
|
108
|
+
and not exists (
|
|
109
|
+
select 1 from eval_runs r
|
|
110
|
+
where r.eval_target_id = t.id
|
|
111
|
+
and r.created_at > now() - interval '7 days'
|
|
112
|
+
group by r.eval_target_id
|
|
113
|
+
having count(*) >= 5
|
|
114
|
+
and count(*) filter (
|
|
115
|
+
where jsonb_typeof(r.grading->'expectations') = 'array'
|
|
116
|
+
and exists (
|
|
117
|
+
select 1 from jsonb_array_elements(r.grading->'expectations') e
|
|
118
|
+
where coalesce(e->>'errored', 'false') = 'true'
|
|
119
|
+
)
|
|
120
|
+
) * 4 >= count(*)
|
|
121
|
+
)
|
|
122
|
+
-- Gate 2b: no human anchors pointing at an ungradeable assertion.
|
|
123
|
+
and not exists (
|
|
124
|
+
select 1
|
|
125
|
+
from eval_grade_annotations a
|
|
126
|
+
join eval_runs r on r.id = a.eval_run_id
|
|
127
|
+
where a.eval_target_id = t.id
|
|
128
|
+
and jsonb_typeof(r.grading->'expectations') = 'array'
|
|
129
|
+
and coalesce(
|
|
130
|
+
r.grading->'expectations'->(a.expectation_index)->>'errored', 'false'
|
|
131
|
+
) = 'true'
|
|
132
|
+
)
|
|
133
|
+
-- Gate 2c: last week's batch wasn't mostly failures.
|
|
134
|
+
and not exists (
|
|
135
|
+
select 1 from eval_runs r
|
|
136
|
+
where r.eval_target_id = t.id
|
|
137
|
+
and r.created_at > now() - interval '7 days'
|
|
138
|
+
group by r.eval_target_id
|
|
139
|
+
having count(*) >= 5
|
|
140
|
+
and count(*) filter (where r.status = 'failed') * 4 >= count(*)
|
|
141
|
+
)
|
|
142
|
+
-- Gate 3: the target has live cases to actually run.
|
|
143
|
+
and exists (
|
|
144
|
+
select 1 from evals ev
|
|
145
|
+
where ev.eval_target_id = t.id and ev.retired_at is null
|
|
146
|
+
)
|
|
147
|
+
-- One in flight at a time, per the 00083/00086 pattern.
|
|
148
|
+
and not exists (
|
|
149
|
+
select 1 from agent_jobs j
|
|
150
|
+
where j.task_kind = 'cron_eval_autorun'
|
|
151
|
+
and j.status in ('pending', 'claimed', 'running')
|
|
152
|
+
and j.payload->>'evalTargetId' = t.id::text
|
|
153
|
+
)
|
|
154
|
+
), ins as (
|
|
155
|
+
insert into agent_jobs (user_id, task_kind, status, payload, dedupe_key)
|
|
156
|
+
select e.user_id,
|
|
157
|
+
'cron_eval_autorun',
|
|
158
|
+
'pending',
|
|
159
|
+
jsonb_build_object('evalTargetId', e.id::text, 'maxRuns', 12)
|
|
160
|
+
|| case
|
|
161
|
+
when e.current_judge is not null
|
|
162
|
+
then jsonb_build_object('graderModel', e.current_judge)
|
|
163
|
+
else '{}'::jsonb
|
|
164
|
+
end,
|
|
165
|
+
'eval-autorun:' || e.id::text
|
|
166
|
+
from eligible e
|
|
167
|
+
returning 1
|
|
168
|
+
)
|
|
169
|
+
select count(*)::int into queued from ins;
|
|
170
|
+
|
|
171
|
+
return queued;
|
|
172
|
+
end;
|
|
173
|
+
$$;
|
|
174
|
+
|
|
175
|
+
comment on function public.enqueue_eval_autorun() is
|
|
176
|
+
'Nightly producer for cron_eval_autorun. Enqueues one bounded depth batch per OPTED-IN, ANCHORED, healthy, under-powered eval target, with the grader pinned to the target''s current judge (00148, #196). Gate 0 (00190) requires instruction_id, project_path, or an absolute-path name: a target anchored to nothing can be measured and "promoted" forever while rewriting only its own content column — measured 2026-08-15, a 129-char demo row had burned 102 runs that way. Health 2a is PROPORTIONAL as of 00169. Opt in with: update eval_targets set metadata = coalesce(metadata, ''{}''::jsonb) || ''{"autoRunEnabled":true}''::jsonb where id = ...';
|