belay 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +155 -0
- data/app/assets/belay/application.css +1 -0
- data/app/controllers/belay/application_controller.rb +48 -0
- data/app/controllers/belay/runs_controller.rb +166 -0
- data/app/controllers/belay/stats_controller.rb +14 -0
- data/app/helpers/belay/application_helper.rb +76 -0
- data/app/views/belay/runs/_header.html.erb +20 -0
- data/app/views/belay/runs/_pending_gate.html.erb +25 -0
- data/app/views/belay/runs/_pending_needs_review.html.erb +30 -0
- data/app/views/belay/runs/_replayed_from.html.erb +7 -0
- data/app/views/belay/runs/_replays.html.erb +14 -0
- data/app/views/belay/runs/_step.html.erb +52 -0
- data/app/views/belay/runs/diff.html.erb +47 -0
- data/app/views/belay/runs/index.html.erb +60 -0
- data/app/views/belay/runs/show.html.erb +34 -0
- data/app/views/belay/shared/_flash.html.erb +11 -0
- data/app/views/belay/shared/_sidebar.html.erb +22 -0
- data/app/views/belay/stats/_chart.html.erb +56 -0
- data/app/views/belay/stats/_table.html.erb +22 -0
- data/app/views/belay/stats/index.html.erb +16 -0
- data/app/views/layouts/belay/application.html.erb +19 -0
- data/config/routes.rb +13 -0
- data/lib/belay/anthropic_provider.rb +166 -0
- data/lib/belay/application_record.rb +7 -0
- data/lib/belay/approval.rb +35 -0
- data/lib/belay/cache_entry.rb +65 -0
- data/lib/belay/configuration.rb +60 -0
- data/lib/belay/context.rb +241 -0
- data/lib/belay/daily_stats.rb +60 -0
- data/lib/belay/engine.rb +40 -0
- data/lib/belay/executor.rb +411 -0
- data/lib/belay/fake_provider.rb +41 -0
- data/lib/belay/janitor_job.rb +48 -0
- data/lib/belay/provider.rb +15 -0
- data/lib/belay/provider_response.rb +19 -0
- data/lib/belay/resume_run_job.rb +42 -0
- data/lib/belay/rule.rb +113 -0
- data/lib/belay/run.rb +629 -0
- data/lib/belay/step.rb +57 -0
- data/lib/belay/version.rb +5 -0
- data/lib/belay/workflow.rb +199 -0
- data/lib/belay.rb +49 -0
- data/lib/generators/belay/install/install_generator.rb +48 -0
- data/lib/generators/belay/install/templates/create_belay_approvals.rb.tt +43 -0
- data/lib/generators/belay/install/templates/create_belay_cache_entries.rb.tt +27 -0
- data/lib/generators/belay/install/templates/create_belay_rules.rb.tt +46 -0
- data/lib/generators/belay/install/templates/create_belay_runs.rb.tt +37 -0
- data/lib/generators/belay/install/templates/create_belay_steps.rb.tt +58 -0
- metadata +226 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
<% error_message = step.error&.dig("message") || step.error&.dig("reason") %>
|
|
2
|
+
|
|
3
|
+
<div class="flex gap-4">
|
|
4
|
+
<div class="flex flex-col items-center">
|
|
5
|
+
<div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full border border-slate-700 bg-slate-800
|
|
6
|
+
font-mono text-xs font-semibold text-slate-400">
|
|
7
|
+
<%= step.position %>
|
|
8
|
+
</div>
|
|
9
|
+
<div class="mt-1 w-px flex-1 bg-slate-800"></div>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div class="card mb-4 flex-1 p-4">
|
|
13
|
+
<div class="mb-2 flex flex-wrap items-center justify-between gap-2">
|
|
14
|
+
<div class="flex items-center gap-2">
|
|
15
|
+
<span class="step-name font-medium text-slate-100"><%= step.name %></span>
|
|
16
|
+
<span class="rounded bg-slate-800 px-1.5 py-0.5 font-mono text-xs text-slate-400"><%= step.kind %></span>
|
|
17
|
+
<% if step.attempt > 1 %>
|
|
18
|
+
<span class="rounded bg-slate-800 px-1.5 py-0.5 font-mono text-xs text-slate-400">attempt <%= step.attempt %></span>
|
|
19
|
+
<% end %>
|
|
20
|
+
</div>
|
|
21
|
+
<span class="step-status"><%= status_badge(step.status) %></span>
|
|
22
|
+
</div>
|
|
23
|
+
|
|
24
|
+
<% if step.input.present? %>
|
|
25
|
+
<div class="mb-2">
|
|
26
|
+
<p class="field-label">Prompt</p>
|
|
27
|
+
<pre class="step-input overflow-x-auto whitespace-pre-wrap rounded bg-slate-950/60 p-2 font-mono text-xs text-slate-400"><%= step.input %></pre>
|
|
28
|
+
</div>
|
|
29
|
+
<% end %>
|
|
30
|
+
|
|
31
|
+
<% if step.output.present? %>
|
|
32
|
+
<div class="mb-2">
|
|
33
|
+
<p class="field-label">Response</p>
|
|
34
|
+
<pre class="step-output overflow-x-auto whitespace-pre-wrap rounded bg-slate-950/60 p-2 font-mono text-xs text-slate-400"><%= step.output %></pre>
|
|
35
|
+
</div>
|
|
36
|
+
<% end %>
|
|
37
|
+
|
|
38
|
+
<% if step.model.present? %>
|
|
39
|
+
<div class="flex flex-wrap gap-3 font-mono text-xs text-slate-500">
|
|
40
|
+
<span><%= step.model %></span>
|
|
41
|
+
<span><%= step.input_tokens %> in / <%= step.output_tokens %> out tokens</span>
|
|
42
|
+
<% if step.cost_cents %>
|
|
43
|
+
<span><%= number_to_currency(step.cost_cents / 100.0) %></span>
|
|
44
|
+
<% end %>
|
|
45
|
+
</div>
|
|
46
|
+
<% end %>
|
|
47
|
+
|
|
48
|
+
<% if error_message %>
|
|
49
|
+
<p class="step-error mt-2 rounded bg-rose-500/10 px-2 py-1 text-xs text-rose-300"><%= error_message %></p>
|
|
50
|
+
<% end %>
|
|
51
|
+
</div>
|
|
52
|
+
</div>
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
<h1 class="mb-1 text-lg font-semibold text-slate-100">Replay diff</h1>
|
|
2
|
+
<p class="mb-6 text-sm text-slate-500">
|
|
3
|
+
<%= link_to @run.id, run_path(@run), class: "font-mono text-violet-300 hover:text-violet-200" %>
|
|
4
|
+
replayed from
|
|
5
|
+
<%= link_to @run.replay_of_id, run_path(@run.replay_of), class: "font-mono text-violet-300 hover:text-violet-200" %>
|
|
6
|
+
<% if @run.replay_model %>
|
|
7
|
+
using <code class="rounded bg-slate-800 px-1 py-0.5 font-mono text-xs text-slate-300"><%= @run.replay_model %></code>
|
|
8
|
+
<% end %>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
<div class="card overflow-hidden">
|
|
12
|
+
<table class="min-w-full divide-y divide-slate-800 text-sm">
|
|
13
|
+
<thead class="bg-slate-900/80">
|
|
14
|
+
<tr>
|
|
15
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Step</th>
|
|
16
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Original output</th>
|
|
17
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Replay output</th>
|
|
18
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500"></th>
|
|
19
|
+
</tr>
|
|
20
|
+
</thead>
|
|
21
|
+
<tbody class="divide-y divide-slate-800">
|
|
22
|
+
<% @diff.each do |entry| %>
|
|
23
|
+
<tr class="diff-row <%= "bg-amber-500/5" if entry[:changed] %>">
|
|
24
|
+
<td class="px-4 py-2">
|
|
25
|
+
<span class="font-medium text-slate-100"><%= entry[:name] %></span>
|
|
26
|
+
<span class="rounded bg-slate-800 px-1.5 py-0.5 font-mono text-xs text-slate-400"><%= entry[:kind] %></span>
|
|
27
|
+
</td>
|
|
28
|
+
<td class="px-4 py-2">
|
|
29
|
+
<pre class="whitespace-pre-wrap font-mono text-xs text-slate-400"><%= entry[:original_output] %></pre>
|
|
30
|
+
</td>
|
|
31
|
+
<td class="px-4 py-2">
|
|
32
|
+
<pre class="whitespace-pre-wrap font-mono text-xs text-slate-400"><%= entry[:replay_output] %></pre>
|
|
33
|
+
</td>
|
|
34
|
+
<td class="diff-status px-4 py-2 text-xs">
|
|
35
|
+
<% if !entry[:replayed] %>
|
|
36
|
+
<span class="rounded-full bg-slate-800 px-2 py-0.5 text-slate-400">not reached</span>
|
|
37
|
+
<% elsif entry[:changed] %>
|
|
38
|
+
<span class="rounded-full bg-amber-500/10 px-2 py-0.5 text-amber-400">changed</span>
|
|
39
|
+
<% else %>
|
|
40
|
+
<span class="rounded-full bg-emerald-500/10 px-2 py-0.5 text-emerald-400">unchanged</span>
|
|
41
|
+
<% end %>
|
|
42
|
+
</td>
|
|
43
|
+
</tr>
|
|
44
|
+
<% end %>
|
|
45
|
+
</tbody>
|
|
46
|
+
</table>
|
|
47
|
+
</div>
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
<div class="mb-6 flex items-center justify-between">
|
|
2
|
+
<h1 class="text-xl font-semibold text-slate-100">Runs</h1>
|
|
3
|
+
|
|
4
|
+
<%= form_with url: runs_path, method: :get, local: true, class: "flex items-center gap-2" do |f| %>
|
|
5
|
+
<%= f.select :status, options_for_select(@statuses, params[:status]),
|
|
6
|
+
{ include_blank: "All statuses" }, class: "select-field" %>
|
|
7
|
+
<%= f.select :workflow, options_for_select(@workflows, params[:workflow]),
|
|
8
|
+
{ include_blank: "All workflows" }, class: "select-field" %>
|
|
9
|
+
<%= f.submit "Filter", class: "btn-secondary" %>
|
|
10
|
+
<% end %>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
<% if @runs.any? %>
|
|
14
|
+
<div class="card overflow-hidden">
|
|
15
|
+
<table class="min-w-full divide-y divide-slate-800 text-sm">
|
|
16
|
+
<thead class="bg-slate-900/80">
|
|
17
|
+
<tr>
|
|
18
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Workflow</th>
|
|
19
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Status</th>
|
|
20
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Started</th>
|
|
21
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Finished</th>
|
|
22
|
+
</tr>
|
|
23
|
+
</thead>
|
|
24
|
+
<tbody class="divide-y divide-slate-800">
|
|
25
|
+
<% @runs.each do |run| %>
|
|
26
|
+
<tr class="hover:bg-slate-800/60">
|
|
27
|
+
<td class="px-4 py-2">
|
|
28
|
+
<%= link_to run.workflow, run_path(run, return_to: request.fullpath),
|
|
29
|
+
class: "run-workflow font-medium text-violet-300 hover:text-violet-200" %>
|
|
30
|
+
</td>
|
|
31
|
+
<td class="run-status px-4 py-2"><%= status_badge(run.status) %></td>
|
|
32
|
+
<td class="px-4 py-2 font-mono text-xs text-slate-500"><%= run.started_at %></td>
|
|
33
|
+
<td class="px-4 py-2 font-mono text-xs text-slate-500"><%= run.finished_at %></td>
|
|
34
|
+
</tr>
|
|
35
|
+
<% end %>
|
|
36
|
+
</tbody>
|
|
37
|
+
</table>
|
|
38
|
+
</div>
|
|
39
|
+
<% if @total_pages > 1 %>
|
|
40
|
+
<div class="mt-4 flex items-center justify-between text-sm">
|
|
41
|
+
<% if @page > 1 %>
|
|
42
|
+
<%= link_to "← Previous", runs_path(pagination_params(request, @page - 1)), class: "pagination-prev btn-secondary" %>
|
|
43
|
+
<% else %>
|
|
44
|
+
<span class="btn-secondary pointer-events-none opacity-40">← Previous</span>
|
|
45
|
+
<% end %>
|
|
46
|
+
|
|
47
|
+
<span class="pagination-status text-slate-400">Page <%= @page %> of <%= @total_pages %></span>
|
|
48
|
+
|
|
49
|
+
<% if @page < @total_pages %>
|
|
50
|
+
<%= link_to "Next →", runs_path(pagination_params(request, @page + 1)), class: "pagination-next btn-secondary" %>
|
|
51
|
+
<% else %>
|
|
52
|
+
<span class="btn-secondary pointer-events-none opacity-40">Next →</span>
|
|
53
|
+
<% end %>
|
|
54
|
+
</div>
|
|
55
|
+
<% end %>
|
|
56
|
+
<% else %>
|
|
57
|
+
<div class="card border-dashed px-6 py-12 text-center">
|
|
58
|
+
<p class="text-sm text-slate-500">No runs yet.</p>
|
|
59
|
+
</div>
|
|
60
|
+
<% end %>
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
<%= link_to "← Back to runs", back_to_runs_path(request),
|
|
2
|
+
class: "back-to-runs mb-4 inline-flex items-center gap-1 text-sm text-slate-400 hover:text-slate-100" %>
|
|
3
|
+
|
|
4
|
+
<%= render "belay/runs/header" %>
|
|
5
|
+
|
|
6
|
+
<% if @run.replay_of %>
|
|
7
|
+
<%= render "belay/runs/replayed_from" %>
|
|
8
|
+
<% end %>
|
|
9
|
+
|
|
10
|
+
<% if @run.replays.any? %>
|
|
11
|
+
<%= render "belay/runs/replays" %>
|
|
12
|
+
<% end %>
|
|
13
|
+
|
|
14
|
+
<% if @run.pending_gate %>
|
|
15
|
+
<%= render "belay/runs/pending_gate" %>
|
|
16
|
+
<% end %>
|
|
17
|
+
|
|
18
|
+
<%# Not rendered for a replay shadow (@run.replay_of present) even if it has
|
|
19
|
+
one: resolve_needs_review! expects a run flag_needs_review! actually
|
|
20
|
+
paused, which seeding a shadow's rows never does - calling it here would
|
|
21
|
+
either raise a confusing InvalidTransitionError or, for outcome: "failed",
|
|
22
|
+
silently "resolve" a run that was never really awaiting a human in the
|
|
23
|
+
first place. A shadow's needs_review row means "the original didn't
|
|
24
|
+
cleanly pass this point, reproduced for visibility" - look at #replay_of
|
|
25
|
+
to resolve the real thing, not this mirror of it. %>
|
|
26
|
+
<% if @run.pending_needs_review && !@run.replay_of %>
|
|
27
|
+
<%= render "belay/runs/pending_needs_review" %>
|
|
28
|
+
<% end %>
|
|
29
|
+
|
|
30
|
+
<h2 class="mb-4 text-sm font-semibold uppercase tracking-wide text-slate-500">Steps</h2>
|
|
31
|
+
|
|
32
|
+
<div>
|
|
33
|
+
<%= render partial: "belay/runs/step", collection: @steps, as: :step %>
|
|
34
|
+
</div>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<% if notice %>
|
|
2
|
+
<div class="mb-6 rounded-md border border-emerald-500/20 bg-emerald-500/10 px-4 py-3 text-sm text-emerald-300" role="status">
|
|
3
|
+
<%= notice %>
|
|
4
|
+
</div>
|
|
5
|
+
<% end %>
|
|
6
|
+
|
|
7
|
+
<% if alert %>
|
|
8
|
+
<div class="mb-6 rounded-md border border-rose-500/20 bg-rose-500/10 px-4 py-3 text-sm text-rose-300" role="alert">
|
|
9
|
+
<%= alert %>
|
|
10
|
+
</div>
|
|
11
|
+
<% end %>
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
<aside class="flex w-56 shrink-0 flex-col gap-6 border-r border-slate-800 bg-slate-900/60 px-4 py-6">
|
|
2
|
+
<div class="px-2">
|
|
3
|
+
<%= link_to runs_path, class: "flex items-center gap-2 text-base font-semibold text-slate-100" do %>
|
|
4
|
+
<span class="h-2 w-2 rounded-sm bg-violet-400"></span>
|
|
5
|
+
Belay
|
|
6
|
+
<% end %>
|
|
7
|
+
<p class="mt-1 text-xs text-slate-500">durable AI runs</p>
|
|
8
|
+
</div>
|
|
9
|
+
|
|
10
|
+
<nav class="flex flex-col gap-1 text-sm">
|
|
11
|
+
<%= link_to runs_path,
|
|
12
|
+
class: "rounded-md px-3 py-2 font-medium #{current_page?(runs_path) ? 'bg-violet-500/10 text-slate-100' : 'text-slate-400 hover:bg-slate-800 hover:text-slate-100'}" do %>
|
|
13
|
+
Runs
|
|
14
|
+
<% end %>
|
|
15
|
+
<%= link_to stats_path,
|
|
16
|
+
class: "rounded-md px-3 py-2 font-medium #{current_page?(stats_path) ? 'bg-violet-500/10 text-slate-100' : 'text-slate-400 hover:bg-slate-800 hover:text-slate-100'}" do %>
|
|
17
|
+
Stats
|
|
18
|
+
<% end %>
|
|
19
|
+
</nav>
|
|
20
|
+
|
|
21
|
+
<p class="mt-auto px-2 text-xs text-slate-600">mounted at /belay</p>
|
|
22
|
+
</aside>
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
<%
|
|
2
|
+
# Bar height is a computed percentage, not a Tailwind utility class - an
|
|
3
|
+
# interpolated class string like "h-[#{pct}%]" wouldn't be seen by
|
|
4
|
+
# Tailwind's build-time content scanner (it reads the template's literal
|
|
5
|
+
# source text, not its rendered output), so it would compile away to
|
|
6
|
+
# nothing. A plain inline style is the correct tool here, not a workaround.
|
|
7
|
+
#
|
|
8
|
+
# Gridlines round UP to the next clean multiple of 20 above the real max,
|
|
9
|
+
# not the raw max itself - an axis that tops out at an arbitrary value like
|
|
10
|
+
# "83.4" reads as noise; "100" is a value a glance can actually anchor to.
|
|
11
|
+
raw_max = daily_stats.map(&:calls_per_100_runs).max.to_f
|
|
12
|
+
axis_max = raw_max.zero? ? 20.0 : (raw_max / 20.0).ceil * 20.0
|
|
13
|
+
gridlines = [axis_max, axis_max * 0.75, axis_max * 0.5, axis_max * 0.25, 0]
|
|
14
|
+
%>
|
|
15
|
+
<div class="card mb-6 p-6">
|
|
16
|
+
<h2 class="text-xs font-semibold uppercase tracking-wide text-slate-500">Calls per 100 runs</h2>
|
|
17
|
+
|
|
18
|
+
<div class="mt-6 flex h-48">
|
|
19
|
+
<div class="flex w-7 flex-col justify-between pb-6 pr-2 text-right">
|
|
20
|
+
<% gridlines.each do |value| %>
|
|
21
|
+
<span class="font-mono text-[10px] leading-none text-slate-600"><%= value.to_i %></span>
|
|
22
|
+
<% end %>
|
|
23
|
+
</div>
|
|
24
|
+
|
|
25
|
+
<div class="relative flex-1">
|
|
26
|
+
<div class="absolute inset-0 flex flex-col justify-between pb-6">
|
|
27
|
+
<% gridlines.each do %>
|
|
28
|
+
<div class="border-t border-slate-800"></div>
|
|
29
|
+
<% end %>
|
|
30
|
+
</div>
|
|
31
|
+
|
|
32
|
+
<%# No items-end here - each bar-col needs its OWN definite height (via the
|
|
33
|
+
default stretch) for its bar's percentage height to resolve against;
|
|
34
|
+
justify-end on each bar-col below is what actually bottom-anchors it. %>
|
|
35
|
+
<div class="absolute inset-0 z-10 flex gap-2 pb-6">
|
|
36
|
+
<% daily_stats.each_with_index do |day, index| %>
|
|
37
|
+
<% is_endpoint = index == daily_stats.size - 1 %>
|
|
38
|
+
<div class="group flex flex-1 flex-col items-center justify-end gap-1.5"
|
|
39
|
+
title="<%= day.date %>: <%= day.calls_per_100_runs %> per 100 runs (<%= day.real_calls %> calls, <%= day.runs %> runs)">
|
|
40
|
+
<% if is_endpoint %>
|
|
41
|
+
<span class="stat-rate-label font-mono text-[11px] font-medium text-slate-200"><%= day.calls_per_100_runs %></span>
|
|
42
|
+
<% end %>
|
|
43
|
+
<div class="w-full max-w-[22px] rounded-t <%= is_endpoint ? "bg-violet-400" : "bg-violet-500/40 group-hover:bg-violet-400/70" %>"
|
|
44
|
+
style="height: <%= [(day.calls_per_100_runs / axis_max * 100), 1.5].max.round(1) %>%"></div>
|
|
45
|
+
</div>
|
|
46
|
+
<% end %>
|
|
47
|
+
</div>
|
|
48
|
+
|
|
49
|
+
<div class="pointer-events-none absolute inset-x-0 bottom-0 flex gap-2">
|
|
50
|
+
<% daily_stats.each do |day| %>
|
|
51
|
+
<span class="flex-1 text-center font-mono text-[10px] text-slate-600"><%= day.date.strftime("%-m/%-d") %></span>
|
|
52
|
+
<% end %>
|
|
53
|
+
</div>
|
|
54
|
+
</div>
|
|
55
|
+
</div>
|
|
56
|
+
</div>
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
<div class="card overflow-hidden">
|
|
2
|
+
<table class="min-w-full divide-y divide-slate-800 text-sm">
|
|
3
|
+
<thead class="bg-slate-900/80">
|
|
4
|
+
<tr>
|
|
5
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Date</th>
|
|
6
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Runs</th>
|
|
7
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Real API calls</th>
|
|
8
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Per 100 runs</th>
|
|
9
|
+
</tr>
|
|
10
|
+
</thead>
|
|
11
|
+
<tbody class="divide-y divide-slate-800">
|
|
12
|
+
<% daily_stats.each do |day| %>
|
|
13
|
+
<tr class="hover:bg-slate-800/60">
|
|
14
|
+
<td class="stat-date px-4 py-2 font-mono text-xs text-slate-300"><%= day.date %></td>
|
|
15
|
+
<td class="stat-runs px-4 py-2 font-mono text-xs text-slate-500"><%= day.runs %></td>
|
|
16
|
+
<td class="stat-calls px-4 py-2 font-mono text-xs text-slate-500"><%= day.real_calls %></td>
|
|
17
|
+
<td class="stat-rate px-4 py-2 font-mono text-xs font-medium text-slate-100"><%= day.calls_per_100_runs %></td>
|
|
18
|
+
</tr>
|
|
19
|
+
<% end %>
|
|
20
|
+
</tbody>
|
|
21
|
+
</table>
|
|
22
|
+
</div>
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
<div class="mb-6">
|
|
2
|
+
<h1 class="text-xl font-semibold text-slate-100">Cost</h1>
|
|
3
|
+
<p class="mt-1 text-sm text-slate-500">
|
|
4
|
+
Real provider calls per 100 runs, by day. A cache hit or a rule-auto-settled gate
|
|
5
|
+
makes no call at all, so a falling trend here is the cost-reduction features working.
|
|
6
|
+
</p>
|
|
7
|
+
</div>
|
|
8
|
+
|
|
9
|
+
<% if @daily_stats.any? %>
|
|
10
|
+
<%= render "chart", daily_stats: @daily_stats %>
|
|
11
|
+
<%= render "table", daily_stats: @daily_stats %>
|
|
12
|
+
<% else %>
|
|
13
|
+
<div class="card border-dashed px-6 py-12 text-center">
|
|
14
|
+
<p class="text-sm text-slate-500">No runs yet.</p>
|
|
15
|
+
</div>
|
|
16
|
+
<% end %>
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Belay</title>
|
|
5
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
6
|
+
<%= csrf_meta_tags %>
|
|
7
|
+
<style><%= raw Belay::Engine.stylesheet %><%= raw Belay.config.custom_css %></style>
|
|
8
|
+
</head>
|
|
9
|
+
<body class="flex min-h-screen bg-slate-950 font-sans text-slate-100 antialiased">
|
|
10
|
+
<%= render "belay/shared/sidebar" %>
|
|
11
|
+
|
|
12
|
+
<main class="min-w-0 flex-1 px-6 py-8 sm:px-10">
|
|
13
|
+
<div class="mx-auto max-w-4xl">
|
|
14
|
+
<%= render "belay/shared/flash" %>
|
|
15
|
+
<%= yield %>
|
|
16
|
+
</div>
|
|
17
|
+
</main>
|
|
18
|
+
</body>
|
|
19
|
+
</html>
|
data/config/routes.rb
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Belay
|
|
8
|
+
# Calls Anthropic's Messages API directly over Net::HTTP (stdlib) rather than via
|
|
9
|
+
# the official anthropic gem - see belay-planning/DECISIONS.md for the reasoning:
|
|
10
|
+
# this gem already has one cross-version dependency incident (the json/
|
|
11
|
+
# activesupport quirks_mode break on Rails 7.1, see gemfiles/rails_7_1.gemfile),
|
|
12
|
+
# and a provider this central shouldn't add a second gem's own version matrix to
|
|
13
|
+
# track. The wire format needed here - POST a JSON body, read a JSON body back -
|
|
14
|
+
# is small enough that stdlib Net::HTTP plus JSON covers it without one.
|
|
15
|
+
#
|
|
16
|
+
# Endpoint and headers verified against Anthropic's Messages API docs as of
|
|
17
|
+
# 2026-10-19 (anthropic-version: 2023-06-01 is still the current, required
|
|
18
|
+
# version string).
|
|
19
|
+
#
|
|
20
|
+
# schema: uses Anthropic's structured outputs (top-level output_config.format,
|
|
21
|
+
# type: "json_schema"), not forced tool_choice. An earlier draft of this file used
|
|
22
|
+
# tool_choice: {type: "tool", name: ...}, which 400s outright on Claude Opus 5.5,
|
|
23
|
+
# Sonnet 5.5, Fable 5.1 and Mythos 5.1 - i.e. it would have broken on exactly the
|
|
24
|
+
# model this file's own README example calls. output_config.format works across
|
|
25
|
+
# the current model lineup, so that's what's implemented. Any object in schema:
|
|
26
|
+
# needs additionalProperties: false, per Anthropic's own requirement for this
|
|
27
|
+
# field - not validated here; an object schema missing it surfaces as a 400
|
|
28
|
+
# (RequestError), same as any other malformed request.
|
|
29
|
+
#
|
|
30
|
+
# stop_reason is checked before parsing: "refusal" means Claude declined to
|
|
31
|
+
# produce schema-matching output at all, "max_tokens" means what came back may be
|
|
32
|
+
# truncated mid-JSON. Both raise StructuredOutputError with that reason rather
|
|
33
|
+
# than attempt JSON.parse on text that may not actually be valid JSON.
|
|
34
|
+
#
|
|
35
|
+
# Deliberately does NOT ship a baked-in per-model price table. Anthropic's prices
|
|
36
|
+
# change, this gem has no way to keep a hardcoded table current, and a silently
|
|
37
|
+
# stale number is worse than an honest nil - the same reasoning ProviderResponse's
|
|
38
|
+
# own comment gives for why cost_cents defaults to nil at all. Pass pricing: with
|
|
39
|
+
# rates for whichever models you actually use; cost_cents is nil for any model not
|
|
40
|
+
# in it, same as it would be if usage data were missing entirely.
|
|
41
|
+
class AnthropicProvider < Provider
|
|
42
|
+
API_URL = URI("https://api.anthropic.com/v1/messages")
|
|
43
|
+
API_VERSION = "2023-06-01"
|
|
44
|
+
|
|
45
|
+
# Raised on any non-2xx response. Anthropic's own error body (JSON, usually
|
|
46
|
+
# {"error": {"type":, "message":}}) is preserved verbatim in the message rather
|
|
47
|
+
# than parsed and re-interpreted, so whatever Anthropic actually said is visible
|
|
48
|
+
# in the step's recorded error, not lossily summarized.
|
|
49
|
+
class RequestError < Error
|
|
50
|
+
attr_reader :status
|
|
51
|
+
|
|
52
|
+
def initialize(status:, body:)
|
|
53
|
+
@status = status
|
|
54
|
+
super("Anthropic API request failed with status #{status}: #{body}")
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Raised when schema: was given but Claude's response can't be trusted to
|
|
59
|
+
# contain valid, complete schema-matching JSON. reason: is Anthropic's own
|
|
60
|
+
# stop_reason ("refusal" or "max_tokens"), exposed so a caller can tell "ask a
|
|
61
|
+
# human" (refusal) apart from "retry with a higher max_tokens" (max_tokens),
|
|
62
|
+
# rather than both surfacing as the same undifferentiated Belay::Error.
|
|
63
|
+
class StructuredOutputError < Error
|
|
64
|
+
attr_reader :reason, :response
|
|
65
|
+
|
|
66
|
+
def initialize(reason:, response:)
|
|
67
|
+
@reason = reason
|
|
68
|
+
@response = response
|
|
69
|
+
super("Anthropic did not return schema-matching output (stop_reason: #{reason.inspect}): #{response.inspect}")
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# pricing: {"claude-..." => { input_per_million: <USD Float>, output_per_million: <USD Float> }, ...}
|
|
74
|
+
#
|
|
75
|
+
# max_tokens: counts thinking tokens too on models with thinking enabled by
|
|
76
|
+
# default (e.g. Claude Opus 5) - a schema: call can hit max_tokens, and raise
|
|
77
|
+
# StructuredOutputError, more easily than the number alone suggests. Raise this
|
|
78
|
+
# if that happens in practice; there's no way to size it correctly up front
|
|
79
|
+
# without knowing the model and its thinking behavior.
|
|
80
|
+
def initialize(api_key: ENV.fetch("ANTHROPIC_API_KEY", nil), max_tokens: 4096, pricing: {})
|
|
81
|
+
super()
|
|
82
|
+
raise ArgumentError, "#{self.class} requires api_key: (or ANTHROPIC_API_KEY)" if api_key.nil?
|
|
83
|
+
|
|
84
|
+
@api_key = api_key
|
|
85
|
+
@max_tokens = max_tokens
|
|
86
|
+
@pricing = pricing
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def call(model:, prompt:, schema: nil)
|
|
90
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
91
|
+
status, body = perform_request(request_body(model: model, prompt: prompt, schema: schema))
|
|
92
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
93
|
+
|
|
94
|
+
raise RequestError.new(status: status, body: body) unless status.between?(200, 299)
|
|
95
|
+
|
|
96
|
+
build_response(JSON.parse(body), model: model, schema: schema, latency_ms: latency_ms)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def request_body(model:, prompt:, schema:)
|
|
102
|
+
body = { model: model, max_tokens: @max_tokens, messages: [{ role: "user", content: prompt }] }
|
|
103
|
+
return body if schema.nil?
|
|
104
|
+
|
|
105
|
+
body.merge(output_config: { format: { type: "json_schema", schema: schema } })
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def perform_request(body)
|
|
109
|
+
response = Net::HTTP.post(API_URL, JSON.generate(body), request_headers)
|
|
110
|
+
[response.code.to_i, response.body]
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def request_headers
|
|
114
|
+
{
|
|
115
|
+
"x-api-key" => @api_key,
|
|
116
|
+
"anthropic-version" => API_VERSION,
|
|
117
|
+
"content-type" => "application/json"
|
|
118
|
+
}
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def build_response(response, model:, schema:, latency_ms:)
|
|
122
|
+
usage = response["usage"] || {}
|
|
123
|
+
input_tokens, output_tokens = usage.values_at("input_tokens", "output_tokens")
|
|
124
|
+
|
|
125
|
+
ProviderResponse.new(
|
|
126
|
+
content: extract_content(response, schema: schema),
|
|
127
|
+
model: response["model"] || model,
|
|
128
|
+
input_tokens: input_tokens,
|
|
129
|
+
output_tokens: output_tokens,
|
|
130
|
+
cost_cents: cost_cents(model: model, input_tokens: input_tokens, output_tokens: output_tokens),
|
|
131
|
+
latency_ms: latency_ms
|
|
132
|
+
)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def extract_content(response, schema:)
|
|
136
|
+
schema ? extract_structured_output(response) : extract_text(response["content"] || [])
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def extract_structured_output(response)
|
|
140
|
+
stop_reason = response["stop_reason"]
|
|
141
|
+
if %w[refusal max_tokens].include?(stop_reason)
|
|
142
|
+
raise StructuredOutputError.new(reason: stop_reason, response: response)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
JSON.parse(extract_text(response["content"] || []))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def extract_text(blocks)
|
|
149
|
+
block = blocks.find { |b| b["type"] == "text" }
|
|
150
|
+
raise Error, "Anthropic response had no text content block" if block.nil?
|
|
151
|
+
|
|
152
|
+
block["text"]
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def cost_cents(model:, input_tokens:, output_tokens:)
|
|
156
|
+
return nil if input_tokens.nil? || output_tokens.nil?
|
|
157
|
+
|
|
158
|
+
rates = @pricing[model]
|
|
159
|
+
return nil if rates.nil?
|
|
160
|
+
|
|
161
|
+
input_cost = input_tokens * rates.fetch(:input_per_million)
|
|
162
|
+
output_cost = output_tokens * rates.fetch(:output_per_million)
|
|
163
|
+
((input_cost + output_cost) / 1_000_000.0 * 100).round
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Belay
|
|
4
|
+
class Approval < ApplicationRecord
|
|
5
|
+
self.table_name = "belay_approvals"
|
|
6
|
+
|
|
7
|
+
# Explicit class_name and optional: false for the same reason Step declares
|
|
8
|
+
# them this way: no Zeitwerk/app-autoloading here to infer the namespace from,
|
|
9
|
+
# and optional: false stated explicitly doesn't depend on the host app's
|
|
10
|
+
# load_defaults to get a friendly validation error instead of a raw DB
|
|
11
|
+
# NotNullViolation.
|
|
12
|
+
belongs_to :run, class_name: "Belay::Run", optional: false
|
|
13
|
+
belongs_to :step, class_name: "Belay::Step", optional: false
|
|
14
|
+
|
|
15
|
+
enum :status, {
|
|
16
|
+
pending: "pending",
|
|
17
|
+
approved: "approved",
|
|
18
|
+
rejected: "rejected"
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
# status has a DB default ("pending"), but the enum writer accepts nil as a
|
|
22
|
+
# valid "blank" assignment (no ArgumentError) - without this, an explicit
|
|
23
|
+
# status: nil would skip the default and hit a raw NotNullViolation instead of
|
|
24
|
+
# RecordInvalid. Matches Step's own comment on the identical situation.
|
|
25
|
+
validates :status, presence: true
|
|
26
|
+
|
|
27
|
+
# No approve!/reject! here. Those are Run methods (feat(runs): add approve! and
|
|
28
|
+
# reject! that enqueue ResumeRunJob, not this commit) - a decision resolves the
|
|
29
|
+
# RUN's pending gate, not just this row in isolation, the same reason
|
|
30
|
+
# transitions live on Run rather than Step: Belay::Run#start!/pause!/etc. own
|
|
31
|
+
# the run-level policy, and Step has no bang methods of its own either (see its
|
|
32
|
+
# comment on exactly this). Adding them here now would mean rebuilding them
|
|
33
|
+
# once that policy exists, same as Step's own stated reasoning.
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Belay
|
|
6
|
+
# Backs ctx.llm(..., cacheable: true) - see Context#llm. Not a general-
|
|
7
|
+
# purpose cache for anything else in the gem; the only two callers are
|
|
8
|
+
# Context#llm's own fetch-then-store pair.
|
|
9
|
+
class CacheEntry < ApplicationRecord
|
|
10
|
+
self.table_name = "belay_cache_entries"
|
|
11
|
+
|
|
12
|
+
# SHA256 of [provider class name, model, prompt, schema].inspect - exact,
|
|
13
|
+
# literal match only, the same .inspect-of-an-array-then-hash pattern
|
|
14
|
+
# Workflow.workflow_version already uses. Deliberately NOT normalized
|
|
15
|
+
# (schema/prompt symbol vs string key differences hash differently,
|
|
16
|
+
# on purpose) - "exact-match cache" means exactly that, not "semantically
|
|
17
|
+
# equivalent"; normalizing would be extra logic in service of a kind of
|
|
18
|
+
# match this feature was never meant to offer.
|
|
19
|
+
#
|
|
20
|
+
# provider.class.name, not the model: string alone - a cached entry from
|
|
21
|
+
# one provider should never satisfy a lookup under a different one
|
|
22
|
+
# (swapping Belay.config.provider mid-lifecycle, or a FakeProvider in
|
|
23
|
+
# tests sharing a database with a real one, shouldn't silently reuse
|
|
24
|
+
# whatever the other provider once returned for an identical-looking call).
|
|
25
|
+
def self.key_for(provider:, model:, prompt:, schema:)
|
|
26
|
+
Digest::SHA256.hexdigest([provider.class.name, model, prompt, schema].inspect)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# nil on a miss OR an expired entry - the caller (Context#llm) can't tell
|
|
30
|
+
# those apart from this return value alone, and doesn't need to: either
|
|
31
|
+
# way, it makes a real provider call next. An expired row is left in
|
|
32
|
+
# place, not deleted here - store! below will overwrite it on the next
|
|
33
|
+
# successful call for the same key; there's no background sweep for rows
|
|
34
|
+
# that are never asked for again, same as belay_steps keeps every row
|
|
35
|
+
# forever rather than pruning anything.
|
|
36
|
+
def self.fetch(provider:, model:, prompt:, schema:)
|
|
37
|
+
entry = find_by(key: key_for(provider: provider, model: model, prompt: prompt, schema: schema))
|
|
38
|
+
return nil if entry.nil? || entry.expires_at&.past?
|
|
39
|
+
|
|
40
|
+
entry.increment!(:hits)
|
|
41
|
+
entry.content
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# upsert, not create!/update! - two callers racing to cache the same miss
|
|
45
|
+
# (two workers asking the identical question for the first time at once)
|
|
46
|
+
# both reach here; upsert's ON CONFLICT DO UPDATE means the second one
|
|
47
|
+
# overwrites the first cleanly instead of raising on the unique index,
|
|
48
|
+
# the same reasoning Run#acquire_lease! and friends already lean on
|
|
49
|
+
# Postgres for elsewhere in this gem rather than a read-then-write race.
|
|
50
|
+
#
|
|
51
|
+
# ttl: defaults to the CURRENT Belay.config.cache_ttl at call time, not
|
|
52
|
+
# whatever it was when an older entry for this key was first stored -
|
|
53
|
+
# every store! reads it fresh, so changing the config takes effect on the
|
|
54
|
+
# very next cache miss, not retroactively on rows already written.
|
|
55
|
+
# rubocop:disable-next Metrics/ParameterLists
|
|
56
|
+
def self.store!(provider:, model:, prompt:, schema:, content:, ttl: Belay.config.cache_ttl)
|
|
57
|
+
key_value = key_for(provider: provider, model: model, prompt: prompt, schema: schema)
|
|
58
|
+
now = Time.current
|
|
59
|
+
upsert(
|
|
60
|
+
{ key: key_value, content: content, hits: 0, expires_at: ttl && (now + ttl), created_at: now, updated_at: now },
|
|
61
|
+
unique_by: :key
|
|
62
|
+
)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|