belay 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -0
- data/app/controllers/belay/stats_controller.rb +2 -0
- data/app/views/belay/stats/_budget_banner.html.erb +6 -0
- data/app/views/belay/stats/_table.html.erb +2 -0
- data/app/views/belay/stats/index.html.erb +4 -0
- data/lib/belay/configuration.rb +58 -0
- data/lib/belay/daily_stats.rb +32 -5
- data/lib/belay/engine.rb +9 -0
- data/lib/belay/executor.rb +48 -0
- data/lib/belay/run.rb +13 -0
- data/lib/belay/version.rb +1 -1
- data/lib/belay/webhook_job.rb +110 -0
- data/lib/belay/webhooks.rb +42 -0
- data/lib/belay.rb +2 -0
- data/lib/generators/belay/install/install_generator.rb +11 -0
- data/lib/generators/belay/install/templates/add_budget_alert_sent_at_to_belay_runs.rb.tt +9 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 16138d86493d56b1c9baef13c6b5039519a01a2382679f134dc7db31f721ad39
|
|
4
|
+
data.tar.gz: c044de960b0983f6bbf0d5965983d89bf1b3d6b2923decebc2e2f71907196e91
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 77143cfb6dd262d9b763a0c7c3166dec5d8fc5366fa1e613575f3ea83a1f289c448f578a279ddaf44c45fccba17d89d1fc4e81894be89450bbb3f071460f7d3e
|
|
7
|
+
data.tar.gz: 806fae91af2d064e8458a94f47aea3ea9562dc3b2dc00f454ee0fad2ad14b4b7d540e76348f65e04a8cf3b98906bf653fecd1a6c4ad9de634567c301ad35db6c
|
data/README.md
CHANGED
|
@@ -152,6 +152,18 @@ Belay publishes `ActiveSupport::Notifications` events for the run/step/gate life
|
|
|
152
152
|
|
|
153
153
|
**→ [docs/instrumentation.md](docs/instrumentation.md)** for the full event list and payloads.
|
|
154
154
|
|
|
155
|
+
## Webhooks
|
|
156
|
+
|
|
157
|
+
Built on top of instrumentation: set `config.webhook_url` and a gate pausing, a step getting flagged `needs_review`, or a run completing each POST a JSON body to it - retried automatically, optionally HMAC-signed, no polling or a subscriber block of your own required.
|
|
158
|
+
|
|
159
|
+
**→ [docs/webhooks.md](docs/webhooks.md)** for configuration, signature verification, and retry behavior.
|
|
160
|
+
|
|
161
|
+
## Cost budgets
|
|
162
|
+
|
|
163
|
+
`config.run_cost_budget_cents` fires `belay.run.budget_exceeded` the first time a single run's total cost crosses it; `config.daily_cost_budget_cents` shows a banner on the dashboard's Stats page once today's spend does. Both nil (off) by default.
|
|
164
|
+
|
|
165
|
+
**→ [docs/cost-budgets.md](docs/cost-budgets.md)** for both thresholds and why they're separate.
|
|
166
|
+
|
|
155
167
|
## Roadmap
|
|
156
168
|
|
|
157
169
|
See [ROADMAP.md](ROADMAP.md).
|
|
@@ -9,6 +9,8 @@ module Belay
|
|
|
9
9
|
class StatsController < ApplicationController
|
|
10
10
|
def index
|
|
11
11
|
@daily_stats = Belay::DailyStats.for_range
|
|
12
|
+
@daily_cost_budget_cents = Belay.config.daily_cost_budget_cents
|
|
13
|
+
@today_cost_cents = Belay::DailyStats.today_cost_cents
|
|
12
14
|
end
|
|
13
15
|
end
|
|
14
16
|
end
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
<div class="mb-6 rounded-lg border border-amber-500/25 bg-amber-500/5 p-4" role="alert">
|
|
2
|
+
<p class="text-sm text-amber-200">
|
|
3
|
+
Today's spend (<%= number_to_currency(today_cost_cents / 100.0) %>) has crossed the configured daily budget of
|
|
4
|
+
<%= number_to_currency(daily_cost_budget_cents / 100.0) %>.
|
|
5
|
+
</p>
|
|
6
|
+
</div>
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Runs</th>
|
|
7
7
|
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Real API calls</th>
|
|
8
8
|
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Per 100 runs</th>
|
|
9
|
+
<th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-slate-500">Spend</th>
|
|
9
10
|
</tr>
|
|
10
11
|
</thead>
|
|
11
12
|
<tbody class="divide-y divide-slate-800">
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
<td class="stat-runs px-4 py-2 font-mono text-xs text-slate-500"><%= day.runs %></td>
|
|
16
17
|
<td class="stat-calls px-4 py-2 font-mono text-xs text-slate-500"><%= day.real_calls %></td>
|
|
17
18
|
<td class="stat-rate px-4 py-2 font-mono text-xs font-medium text-slate-100"><%= day.calls_per_100_runs %></td>
|
|
19
|
+
<td class="stat-cost px-4 py-2 font-mono text-xs text-slate-300"><%= number_to_currency(day.cost_cents / 100.0) %></td>
|
|
18
20
|
</tr>
|
|
19
21
|
<% end %>
|
|
20
22
|
</tbody>
|
|
@@ -6,6 +6,10 @@
|
|
|
6
6
|
</p>
|
|
7
7
|
</div>
|
|
8
8
|
|
|
9
|
+
<% if @daily_cost_budget_cents && @today_cost_cents > @daily_cost_budget_cents %>
|
|
10
|
+
<%= render "budget_banner", today_cost_cents: @today_cost_cents, daily_cost_budget_cents: @daily_cost_budget_cents %>
|
|
11
|
+
<% end %>
|
|
12
|
+
|
|
9
13
|
<% if @daily_stats.any? %>
|
|
10
14
|
<%= render "chart", daily_stats: @daily_stats %>
|
|
11
15
|
<%= render "table", daily_stats: @daily_stats %>
|
data/lib/belay/configuration.rb
CHANGED
|
@@ -47,7 +47,57 @@ module Belay
|
|
|
47
47
|
# own comment), so the host app owns wherever this string actually comes from.
|
|
48
48
|
attr_accessor :custom_css
|
|
49
49
|
|
|
50
|
+
# Where Belay::WebhookJob POSTs a JSON body ({event:, payload:}) for every event
|
|
51
|
+
# in webhook_events below. nil by default - opt-in, same reasoning as provider/
|
|
52
|
+
# authenticate_with: a host app that never sets this gets no webhook behavior at
|
|
53
|
+
# all, not a silently-failing POST to nowhere. Read at delivery time (inside the
|
|
54
|
+
# subscriber block and inside WebhookJob#perform), not captured when the
|
|
55
|
+
# subscription is installed - so it can be set in config/initializers/belay.rb
|
|
56
|
+
# regardless of initializer load order relative to Belay::Engine's own.
|
|
57
|
+
attr_accessor :webhook_url
|
|
58
|
+
|
|
59
|
+
# Optional HMAC-SHA256 secret. When set, every webhook request carries an
|
|
60
|
+
# X-Belay-Signature header (hex digest of the raw JSON body) so the receiving
|
|
61
|
+
# endpoint can verify the request actually came from this app, the same pattern
|
|
62
|
+
# Stripe/GitHub webhooks use. nil by default - signing is extra integration work
|
|
63
|
+
# for a receiver that may not need it (e.g. a private endpoint on a trusted
|
|
64
|
+
# network); unsigned is still a reasonable default, matching webhook_url's own
|
|
65
|
+
# opt-in-everything posture.
|
|
66
|
+
attr_accessor :webhook_secret
|
|
67
|
+
|
|
68
|
+
# Which belay.* events actually get POSTed. A plain Array a host app can
|
|
69
|
+
# overwrite wholesale (push onto it, or replace it entirely) - not every event
|
|
70
|
+
# Belay fires is something worth waking up a human/on-call for; this starts
|
|
71
|
+
# narrow (gate pause, needs_review, run completion - exactly what
|
|
72
|
+
# FUTURE_ROADMAP.md's webhooks item asks for) rather than defaulting to "every
|
|
73
|
+
# belay.* event," which would surprise anyone who just wants "tell me when a
|
|
74
|
+
# human is needed or a run finishes," not every auto-approved gate too.
|
|
75
|
+
attr_accessor :webhook_events
|
|
76
|
+
|
|
77
|
+
# Cents. nil by default - opt-in, same posture as every other threshold here. Once
|
|
78
|
+
# a single run's total cost_cents (every step's, summed - see Run#cost_cents)
|
|
79
|
+
# crosses this, Executor fires belay.run.budget_exceeded exactly once for that
|
|
80
|
+
# run (not once per step that happens to still be over budget afterward). Not in
|
|
81
|
+
# webhook_events by default even once set - a host app that wants this pushed to
|
|
82
|
+
# its webhook adds "belay.run.budget_exceeded" itself; setting a budget purely
|
|
83
|
+
# for the dashboard banner below shouldn't silently start firing webhooks nobody
|
|
84
|
+
# asked for.
|
|
85
|
+
attr_accessor :run_cost_budget_cents
|
|
86
|
+
|
|
87
|
+
# Cents. nil by default. Purely a dashboard concern (unlike run_cost_budget_cents
|
|
88
|
+
# above, this fires no event) - Belay::StatsController compares it against
|
|
89
|
+
# today's total cost_cents (Belay::DailyStats.for_range) and the Stats page shows
|
|
90
|
+
# a banner once today's total crosses it.
|
|
91
|
+
attr_accessor :daily_cost_budget_cents
|
|
92
|
+
|
|
50
93
|
def initialize
|
|
94
|
+
assign_execution_defaults!
|
|
95
|
+
assign_webhook_and_budget_defaults!
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def assign_execution_defaults!
|
|
51
101
|
@lease_duration = 300
|
|
52
102
|
@heartbeat_interval = 30
|
|
53
103
|
@provider = nil
|
|
@@ -56,5 +106,13 @@ module Belay
|
|
|
56
106
|
@rule_approval_threshold = 3
|
|
57
107
|
@custom_css = nil
|
|
58
108
|
end
|
|
109
|
+
|
|
110
|
+
def assign_webhook_and_budget_defaults!
|
|
111
|
+
@webhook_url = nil
|
|
112
|
+
@webhook_secret = nil
|
|
113
|
+
@webhook_events = %w[belay.gate.paused belay.step.needs_review belay.run.completed]
|
|
114
|
+
@run_cost_budget_cents = nil
|
|
115
|
+
@daily_cost_budget_cents = nil
|
|
116
|
+
end
|
|
59
117
|
end
|
|
60
118
|
end
|
data/lib/belay/daily_stats.rb
CHANGED
|
@@ -7,7 +7,7 @@ module Belay
|
|
|
7
7
|
# ActiveRecord model of its own: there's no belay_daily_stats table, this is
|
|
8
8
|
# a read-only view over belay_runs/belay_steps' own created_at timestamps.
|
|
9
9
|
class DailyStats
|
|
10
|
-
Row = Struct.new(:date, :runs, :real_calls, :calls_per_100_runs, keyword_init: true)
|
|
10
|
+
Row = Struct.new(:date, :runs, :real_calls, :calls_per_100_runs, :cost_cents, keyword_init: true)
|
|
11
11
|
|
|
12
12
|
# Calls are grouped by the STEP's own day, not the owning run's - a run
|
|
13
13
|
# can pause on a gate for a real, unbounded amount of time (nothing times
|
|
@@ -25,9 +25,12 @@ module Belay
|
|
|
25
25
|
def self.for_range
|
|
26
26
|
calls_by_day = calls_by_day_counts
|
|
27
27
|
runs_by_day = runs_by_day_counts
|
|
28
|
+
cost_by_day = cost_cents_by_day
|
|
28
29
|
|
|
29
|
-
days = (calls_by_day.keys + runs_by_day.keys).uniq.sort
|
|
30
|
-
days.map
|
|
30
|
+
days = (calls_by_day.keys + runs_by_day.keys + cost_by_day.keys).uniq.sort
|
|
31
|
+
days.map do |date|
|
|
32
|
+
build_row(date, runs_by_day.fetch(date, 0), calls_by_day.fetch(date, 0), cost_by_day.fetch(date, 0))
|
|
33
|
+
end
|
|
31
34
|
end
|
|
32
35
|
|
|
33
36
|
def self.calls_by_day_counts
|
|
@@ -40,6 +43,19 @@ module Belay
|
|
|
40
43
|
end
|
|
41
44
|
private_class_method :runs_by_day_counts
|
|
42
45
|
|
|
46
|
+
# Grouped by the step's own day, same reasoning calls_by_day_counts' comment
|
|
47
|
+
# above already gives for calls - a step's cost is incurred when the call it
|
|
48
|
+
# paid for actually happened, not whenever its owning run started. Not scoped to
|
|
49
|
+
# llm_call/cache_hit: false the way calls_by_day_counts is - cost_cents is only
|
|
50
|
+
# ever non-nil on a real, billed llm_call anyway (a cache hit's own cost_cents is
|
|
51
|
+
# explicitly 0, per Context#fetch_cached_llm_response's own comment, not nil), so
|
|
52
|
+
# a plain SUM across every step's cost_cents already excludes everything that
|
|
53
|
+
# shouldn't count without a separate WHERE.
|
|
54
|
+
def self.cost_cents_by_day
|
|
55
|
+
normalized_counts(Step.group("DATE(created_at)").sum(:cost_cents))
|
|
56
|
+
end
|
|
57
|
+
private_class_method :cost_cents_by_day
|
|
58
|
+
|
|
43
59
|
# .group("DATE(...)").count's hash keys come back as plain Strings on some
|
|
44
60
|
# adapter/Rails version combinations, real Date objects on others - a raw
|
|
45
61
|
# SQL group-by fragment (not an actual typed column) gives ActiveRecord
|
|
@@ -51,8 +67,19 @@ module Belay
|
|
|
51
67
|
end
|
|
52
68
|
private_class_method :normalized_counts
|
|
53
69
|
|
|
54
|
-
|
|
55
|
-
|
|
70
|
+
# Deliberately NOT @daily_stats.find { |row| row.date == Date.current } at the
|
|
71
|
+
# call site - every date above comes from Postgres's own DATE(created_at), not
|
|
72
|
+
# Ruby's Date.current, and the two can disagree for part of the day whenever the
|
|
73
|
+
# app's Time.zone differs from the DB session's own timezone setting. Asking
|
|
74
|
+
# Postgres's own CURRENT_DATE here instead of Ruby's Date.current means both
|
|
75
|
+
# sides of "is this today" are computed the same way, in the same session - they
|
|
76
|
+
# can never disagree, regardless of which timezone either happens to be in.
|
|
77
|
+
def self.today_cost_cents
|
|
78
|
+
Step.where("DATE(created_at) = CURRENT_DATE").sum(:cost_cents)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def self.build_row(date, runs, calls, cost_cents)
|
|
82
|
+
Row.new(date: date, runs: runs, real_calls: calls, cost_cents: cost_cents,
|
|
56
83
|
calls_per_100_runs: runs.zero? ? 0.0 : (calls.to_f / runs * 100).round(1))
|
|
57
84
|
end
|
|
58
85
|
private_class_method :build_row
|
data/lib/belay/engine.rb
CHANGED
|
@@ -20,6 +20,15 @@ module Belay
|
|
|
20
20
|
class Engine < ::Rails::Engine
|
|
21
21
|
isolate_namespace Belay
|
|
22
22
|
|
|
23
|
+
# Installed once per process, at boot - not per-request. Belay::Webhooks.deliver
|
|
24
|
+
# (the callback this installs) checks Belay.config.webhook_url/webhook_events
|
|
25
|
+
# itself on every call, so it's a safe no-op in any app that never sets
|
|
26
|
+
# webhook_url, and initializer load order relative to a host app's own
|
|
27
|
+
# config/initializers/belay.rb never matters (see Webhooks' own comment).
|
|
28
|
+
initializer "belay.webhooks" do
|
|
29
|
+
Belay::Webhooks.subscribe!
|
|
30
|
+
end
|
|
31
|
+
|
|
23
32
|
# The dashboard's CSS, inlined into the layout via <style> rather than
|
|
24
33
|
# served through a <link> and whichever asset pipeline (Sprockets,
|
|
25
34
|
# Propshaft, none at all) the host app happens to have configured - that's
|
data/lib/belay/executor.rb
CHANGED
|
@@ -245,6 +245,14 @@ module Belay
|
|
|
245
245
|
settle_succeeded!(step_run, step, context, output)
|
|
246
246
|
rescue StandardError => e
|
|
247
247
|
settle_failed!(step_run, step, context, e)
|
|
248
|
+
ensure
|
|
249
|
+
# One call site, not one in each settle_*! below, covering success, failure,
|
|
250
|
+
# AND (harmlessly) the AUTO_SETTLED early return above - check_run_cost_budget!
|
|
251
|
+
# itself no-ops immediately for anything that isn't a step with a real,
|
|
252
|
+
# just-persisted cost_cents (a gate never has one), so running it unconditionally
|
|
253
|
+
# here is equivalent to calling it explicitly from every exit path, without the
|
|
254
|
+
# duplication (and the risk of a future settle_*! forgetting to call it).
|
|
255
|
+
check_run_cost_budget!(step)
|
|
248
256
|
end
|
|
249
257
|
|
|
250
258
|
def settle_succeeded!(step_run, step, context, output)
|
|
@@ -275,6 +283,46 @@ module Belay
|
|
|
275
283
|
)
|
|
276
284
|
end
|
|
277
285
|
|
|
286
|
+
# Fires belay.run.budget_exceeded the FIRST time this run's total cost_cents
|
|
287
|
+
# (every step's, summed - Run#cost_cents) crosses config.run_cost_budget_cents,
|
|
288
|
+
# never again for the same run - a run that's already over budget doesn't need
|
|
289
|
+
# the same alert repeated on every subsequent step that's merely still over it.
|
|
290
|
+
# step.cost_cents.nil? short-circuits before the budget/alert_sent_at checks
|
|
291
|
+
# below even run: a step with no recorded cost (no pricing: configured, or not
|
|
292
|
+
# an llm_call at all) can never be the step that newly crosses a budget, so
|
|
293
|
+
# there's nothing to check.
|
|
294
|
+
def check_run_cost_budget!(step)
|
|
295
|
+
return if step.cost_cents.nil?
|
|
296
|
+
|
|
297
|
+
budget = Belay.config.run_cost_budget_cents
|
|
298
|
+
return if budget.nil? || @run.budget_alert_sent_at
|
|
299
|
+
|
|
300
|
+
total = @run.cost_cents
|
|
301
|
+
return if total <= budget
|
|
302
|
+
|
|
303
|
+
claim_budget_alert!(total, budget)
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# A conditional UPDATE, not read-then-write - same reasoning as Run#acquire_
|
|
307
|
+
# lease!'s own comment: nothing here takes the same kind of lease
|
|
308
|
+
# run_attempt's own caller assumes holds for the whole run (Workflow.start's
|
|
309
|
+
# inline path takes none at all; flag_needs_review!'s own comment already
|
|
310
|
+
# flags that no lease mechanism exists to tell two workers apart), so two
|
|
311
|
+
# Executor instances racing on the same run could otherwise both read
|
|
312
|
+
# budget_alert_sent_at as nil and both fire this event. update_all's return
|
|
313
|
+
# value (rows actually matched) is the only thing trusted to decide that;
|
|
314
|
+
# @run's in-memory attribute is only set after a successful claim, never used
|
|
315
|
+
# to decide one.
|
|
316
|
+
def claim_budget_alert!(total, budget)
|
|
317
|
+
claimed = Run.where(id: @run.id, budget_alert_sent_at: nil).update_all(budget_alert_sent_at: Time.current)
|
|
318
|
+
return if claimed.zero?
|
|
319
|
+
|
|
320
|
+
@run.budget_alert_sent_at = Time.current
|
|
321
|
+
ActiveSupport::Notifications.instrument(
|
|
322
|
+
"belay.run.budget_exceeded", run_id: @run.id, total_cost_cents: total, budget_cents: budget
|
|
323
|
+
)
|
|
324
|
+
end
|
|
325
|
+
|
|
278
326
|
# Wrapped in @run.heartbeat_while, not called bare - this is the one call in
|
|
279
327
|
# the whole step/gate lifecycle that can plausibly run longer than
|
|
280
328
|
# lease_duration (a model call, or a gate's own when: doing something
|
data/lib/belay/run.rb
CHANGED
|
@@ -82,6 +82,19 @@ module Belay
|
|
|
82
82
|
steps.needs_review.first
|
|
83
83
|
end
|
|
84
84
|
|
|
85
|
+
# SQL SUM, not steps.sum(&:cost_cents) in Ruby - a plain SUM already treats every
|
|
86
|
+
# NULL row (every non-llm_call step, and any llm_call step whose provider had no
|
|
87
|
+
# pricing: configured for that model) as 0 and returns 0, never nil, for a run
|
|
88
|
+
# with no costed steps at all - exactly the "we don't know the cost" vs. "the
|
|
89
|
+
# cost is zero" distinction Provider#cost_cents' own nil already draws at the
|
|
90
|
+
# single-step level, just rolled up: a run where NOTHING is priced costs "0
|
|
91
|
+
# cents we know about," the same honest answer a single unpriced step's own nil
|
|
92
|
+
# would give if summed the Ruby way (0, via Array#sum's +0 default) anyway - so
|
|
93
|
+
# there's no real information lost by using SQL's NULL-safe SUM here instead.
|
|
94
|
+
def cost_cents
|
|
95
|
+
steps.sum(:cost_cents)
|
|
96
|
+
end
|
|
97
|
+
|
|
85
98
|
# DESIGN_DECISIONS #5: "a worker claims a run with a conditional UPDATE on
|
|
86
99
|
# lease_owner/lease_expires_at." The condition - no owner yet, or the existing
|
|
87
100
|
# lease has expired - is evaluated by Postgres in the same statement that claims
|
data/lib/belay/version.rb
CHANGED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_job"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "json"
|
|
6
|
+
require "uri"
|
|
7
|
+
require "openssl"
|
|
8
|
+
|
|
9
|
+
module Belay
|
|
10
|
+
# POSTs a single belay.* event to Belay.config.webhook_url, enqueued by
|
|
11
|
+
# Belay::Webhooks.deliver (never called directly by application code). A separate
|
|
12
|
+
# job, not an inline HTTP call from inside the instrument block that fired the
|
|
13
|
+
# event, so a slow or down webhook receiver can never add latency to - or fail - a
|
|
14
|
+
# gate pausing, a step getting flagged needs_review, or a run completing. Same
|
|
15
|
+
# reasoning as ResumeRunJob/JanitorJob already being jobs rather than inline calls.
|
|
16
|
+
#
|
|
17
|
+
# event/payload, not the Run/Step records themselves: ActiveJob serializes a plain
|
|
18
|
+
# Hash (with its original String/Symbol keys preserved) without needing GlobalID,
|
|
19
|
+
# and every value already in an instrumentation payload (ids, strings, booleans) is
|
|
20
|
+
# plain-JSON-safe as-is - no serializer concern to design around. (Webhooks.deliver
|
|
21
|
+
# itself is responsible for never handing this job a payload containing the raw
|
|
22
|
+
# :exception_object ActiveSupport::Notifications::Instrumenter adds when a wrapped
|
|
23
|
+
# block raises - see that module's own comment - so this job never has to guard
|
|
24
|
+
# against an unserializable argument.)
|
|
25
|
+
#
|
|
26
|
+
# webhook_url/webhook_secret are both read here, at perform time, not captured at
|
|
27
|
+
# enqueue time - so clearing webhook_url between enqueue and perform (an app
|
|
28
|
+
# deliberately turning webhooks off mid-flight) makes this job a clean no-op rather
|
|
29
|
+
# than delivering to a URL the app just unset.
|
|
30
|
+
#
|
|
31
|
+
# OPEN_TIMEOUT/READ_TIMEOUT are set explicitly (unlike AnthropicProvider/
|
|
32
|
+
# OpenAIProvider/GeminiProvider's own Net::HTTP calls, which set neither): those
|
|
33
|
+
# three call a trusted vendor's own API; webhook_url is an arbitrary URL the host
|
|
34
|
+
# app configured, which could point at a receiver that hangs indefinitely without
|
|
35
|
+
# ever closing the connection. Net::HTTP's stdlib default (60s/60s) would otherwise
|
|
36
|
+
# let one hung receiver tie up a job worker for minutes across all 5 retry_on
|
|
37
|
+
# attempts; a short, fixed timeout here caps that per attempt.
|
|
38
|
+
#
|
|
39
|
+
# retry_on, not discard_on: a webhook receiver having a bad minute is exactly the
|
|
40
|
+
# case retrying helps with, and letting it eventually raise once attempts are
|
|
41
|
+
# exhausted surfaces the failure in whatever the host app's own job backend already
|
|
42
|
+
# does with a failed job (Sidekiq's dead set, GoodJob's errors, etc.) - silently
|
|
43
|
+
# swallowing a permanently-failing webhook forever would hide a real integration
|
|
44
|
+
# break from the one app that could actually fix it. A malformed webhook_url is
|
|
45
|
+
# deliberately NOT in this list, below - retrying the exact same unparseable string
|
|
46
|
+
# five times, with increasing delay, can never succeed; #perform raises a plain
|
|
47
|
+
# Belay::Error for that case instead, immediately, once.
|
|
48
|
+
class WebhookJob < ActiveJob::Base
|
|
49
|
+
OPEN_TIMEOUT = 5
|
|
50
|
+
READ_TIMEOUT = 5
|
|
51
|
+
|
|
52
|
+
# Raised on any non-2xx response. The receiving endpoint's own response body is
|
|
53
|
+
# preserved verbatim in the message, same reasoning as every Provider's own
|
|
54
|
+
# RequestError.
|
|
55
|
+
class RequestError < Error
|
|
56
|
+
attr_reader :status
|
|
57
|
+
|
|
58
|
+
def initialize(status:, body:)
|
|
59
|
+
@status = status
|
|
60
|
+
super("Belay webhook POST failed with status #{status}: #{body}")
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
retry_on RequestError, wait: :polynomially_longer, attempts: 5
|
|
65
|
+
retry_on Net::OpenTimeout, Net::ReadTimeout, SocketError, wait: :polynomially_longer, attempts: 5
|
|
66
|
+
|
|
67
|
+
def perform(event:, payload:)
|
|
68
|
+
url = Belay.config.webhook_url
|
|
69
|
+
return if url.nil?
|
|
70
|
+
|
|
71
|
+
body = JSON.generate(event: event, payload: payload)
|
|
72
|
+
response = post(parse_webhook_url!(url), body)
|
|
73
|
+
status = response.code.to_i
|
|
74
|
+
raise RequestError.new(status: status, body: response.body) unless status.between?(200, 299)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
# URI() alone doesn't catch every malformed URL - a scheme-less string like
|
|
80
|
+
# "example.com/hook" parses successfully into a URI::Generic with host: nil
|
|
81
|
+
# (it's treated as a relative reference), rather than raising. Checking host
|
|
82
|
+
# explicitly catches that case too, so it fails the same clear, immediate,
|
|
83
|
+
# non-retried way as a string URI() itself rejects, instead of surfacing later
|
|
84
|
+
# as a cryptic Net::HTTP connection error against a nil host.
|
|
85
|
+
def parse_webhook_url!(url)
|
|
86
|
+
uri = URI(url)
|
|
87
|
+
raise URI::InvalidURIError, "no host" if uri.host.nil?
|
|
88
|
+
|
|
89
|
+
uri
|
|
90
|
+
rescue URI::InvalidURIError => e
|
|
91
|
+
raise Error, "Belay.config.webhook_url (#{url.inspect}) is not a valid URL: #{e.message}"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def post(uri, body)
|
|
95
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https",
|
|
96
|
+
open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) do |http|
|
|
97
|
+
request = Net::HTTP::Post.new(uri, request_headers(body))
|
|
98
|
+
request.body = body
|
|
99
|
+
http.request(request)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def request_headers(body)
|
|
104
|
+
headers = { "content-type" => "application/json" }
|
|
105
|
+
secret = Belay.config.webhook_secret
|
|
106
|
+
headers["X-Belay-Signature"] = OpenSSL::HMAC.hexdigest("SHA256", secret, body) if secret
|
|
107
|
+
headers
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Belay
|
|
4
|
+
# Bridges ActiveSupport::Notifications (docs/instrumentation.md) to
|
|
5
|
+
# Belay::WebhookJob. .subscribe! is called exactly once, from Belay::Engine's own
|
|
6
|
+
# initializer, at process boot - not per-request, not re-subscribed if config
|
|
7
|
+
# changes later.
|
|
8
|
+
#
|
|
9
|
+
# Subscribes to every belay.* event unconditionally, then filters on
|
|
10
|
+
# Belay.config.webhook_events inside the single callback, rather than
|
|
11
|
+
# subscribing/unsubscribing per event as config.webhook_events changes - config can
|
|
12
|
+
# be (re)assigned any time after boot (a test's Belay.reset_config!, a console
|
|
13
|
+
# session), and re-deriving the subscription from a config value that might not
|
|
14
|
+
# exist yet at initializer time would make this fragile to initializer ordering.
|
|
15
|
+
# Checking config inside the callback instead means order never matters: by the
|
|
16
|
+
# time any belay.* event actually fires, config has long since been set.
|
|
17
|
+
module Webhooks
|
|
18
|
+
def self.subscribe!
|
|
19
|
+
ActiveSupport::Notifications.subscribe(/^belay\./) do |name, _start, _finish, _id, payload|
|
|
20
|
+
deliver(name, payload)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.deliver(name, payload)
|
|
25
|
+
return if Belay.config.webhook_url.nil?
|
|
26
|
+
return unless Belay.config.webhook_events.include?(name)
|
|
27
|
+
# Run#transition! wraps update! in ActiveSupport::Notifications.instrument(...) {
|
|
28
|
+
# }; if that block raises (e.g. a validation/locking error), Instrumenter still
|
|
29
|
+
# notifies every subscriber (the `ensure` in its own #instrument), but first
|
|
30
|
+
# mutates payload in place to add :exception (a 2-element String Array - fine)
|
|
31
|
+
# AND :exception_object (the raw exception instance - not ActiveJob-serializable,
|
|
32
|
+
# would raise ActiveJob::SerializationError from inside this very call). Skip it
|
|
33
|
+
# entirely rather than strip just the unserializable key: the transition this
|
|
34
|
+
# payload describes didn't actually happen, so forwarding it at all - "run
|
|
35
|
+
# completed," when update! raised - would be reporting something false, not
|
|
36
|
+
# just a serialization hazard.
|
|
37
|
+
return if payload.key?(:exception)
|
|
38
|
+
|
|
39
|
+
WebhookJob.perform_later(event: name, payload: payload)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
data/lib/belay.rb
CHANGED
|
@@ -25,6 +25,8 @@ module Belay
|
|
|
25
25
|
require_relative "belay/executor"
|
|
26
26
|
require_relative "belay/resume_run_job"
|
|
27
27
|
require_relative "belay/janitor_job"
|
|
28
|
+
require_relative "belay/webhook_job"
|
|
29
|
+
require_relative "belay/webhooks"
|
|
28
30
|
require_relative "belay/engine"
|
|
29
31
|
|
|
30
32
|
# Raised by Run#start!/pause!/resume!/complete!/fail!/cancel! when the current
|
|
@@ -43,6 +43,17 @@ module Belay
|
|
|
43
43
|
def create_belay_rules_migration
|
|
44
44
|
migration_template "create_belay_rules.rb.tt", "db/migrate/create_belay_rules.rb"
|
|
45
45
|
end
|
|
46
|
+
|
|
47
|
+
# Added after 0.2.0 shipped, as a new migration - never editing
|
|
48
|
+
# create_belay_runs.rb.tt above once it's shipped (see that file's own
|
|
49
|
+
# history). A host app upgrading the gem and re-running `rails generate
|
|
50
|
+
# belay:install` only gets this one copied in: migration_template skips
|
|
51
|
+
# every migration whose class already exists in db/migrate, so the five
|
|
52
|
+
# above are untouched no-ops on a re-run.
|
|
53
|
+
def add_budget_alert_sent_at_to_belay_runs_migration
|
|
54
|
+
migration_template "add_budget_alert_sent_at_to_belay_runs.rb.tt",
|
|
55
|
+
"db/migrate/add_budget_alert_sent_at_to_belay_runs.rb"
|
|
56
|
+
end
|
|
46
57
|
end
|
|
47
58
|
end
|
|
48
59
|
end
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
class AddBudgetAlertSentAtToBelayRuns < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
2
|
+
def change
|
|
3
|
+
# Set the one time Executor fires belay.run.budget_exceeded for this run (once
|
|
4
|
+
# its total cost_cents crosses config.run_cost_budget_cents) - nil forever for
|
|
5
|
+
# a run that never has a budget configured, or never crosses it. Guards against
|
|
6
|
+
# firing the same event again on every later step that's still over budget.
|
|
7
|
+
add_column :belay_runs, :budget_alert_sent_at, :datetime
|
|
8
|
+
end
|
|
9
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: belay
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Swakhar Dey
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-07 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: actionpack
|
|
@@ -164,6 +164,7 @@ files:
|
|
|
164
164
|
- app/views/belay/runs/show.html.erb
|
|
165
165
|
- app/views/belay/shared/_flash.html.erb
|
|
166
166
|
- app/views/belay/shared/_sidebar.html.erb
|
|
167
|
+
- app/views/belay/stats/_budget_banner.html.erb
|
|
167
168
|
- app/views/belay/stats/_chart.html.erb
|
|
168
169
|
- app/views/belay/stats/_table.html.erb
|
|
169
170
|
- app/views/belay/stats/index.html.erb
|
|
@@ -190,8 +191,11 @@ files:
|
|
|
190
191
|
- lib/belay/run.rb
|
|
191
192
|
- lib/belay/step.rb
|
|
192
193
|
- lib/belay/version.rb
|
|
194
|
+
- lib/belay/webhook_job.rb
|
|
195
|
+
- lib/belay/webhooks.rb
|
|
193
196
|
- lib/belay/workflow.rb
|
|
194
197
|
- lib/generators/belay/install/install_generator.rb
|
|
198
|
+
- lib/generators/belay/install/templates/add_budget_alert_sent_at_to_belay_runs.rb.tt
|
|
195
199
|
- lib/generators/belay/install/templates/create_belay_approvals.rb.tt
|
|
196
200
|
- lib/generators/belay/install/templates/create_belay_cache_entries.rb.tt
|
|
197
201
|
- lib/generators/belay/install/templates/create_belay_rules.rb.tt
|