belay 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -1
- data/lib/belay/executor.rb +35 -4
- data/lib/belay/gemini_provider.rb +158 -0
- data/lib/belay/openai_provider.rb +246 -0
- data/lib/belay/run.rb +31 -8
- data/lib/belay/version.rb +1 -1
- data/lib/belay.rb +2 -0
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 52f50e4537c30e2b3eb43793cc383813ca063aa24f158c8fb094e4aa79e5a932
|
|
4
|
+
data.tar.gz: 1b058a13b621cbc6eee5b44b478a3416aea6c7d2e27dd967031ca9657887bd2b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2954f88a1817e5a6e5b57f556157279b74bd30b8bd0769b451f5b662b72ff10127731d9bf3121823137a69fda172dff4b79acf46d78079378a251e2048dac078
|
|
7
|
+
data.tar.gz: 99b29ce7dd493c0218a1e8e94600108370ec74fc7b1e0fba9307eb7ce9f4793af11ee67eb4b9c644da0f2f4bb78cd9157c5af9fc286250399bab90cbdb96cfaa
|
data/README.md
CHANGED
|
@@ -132,7 +132,7 @@ ctx.llm(model:, prompt:, schema: nil, cacheable: false)
|
|
|
132
132
|
|
|
133
133
|
Calls `Belay.config.provider`, validates the response against `schema:` provider-agnostically, and returns `response.content` directly as the step's own output.
|
|
134
134
|
|
|
135
|
-
**→ [docs/providers.md](docs/providers.md)** -
|
|
135
|
+
**→ [docs/providers.md](docs/providers.md)** - ships `AnthropicProvider`, `OpenAIProvider`, and `GeminiProvider`; bring your own for anything else.
|
|
136
136
|
**→ [docs/caching.md](docs/caching.md)** - `cacheable: true` and when it's actually safe to use.
|
|
137
137
|
|
|
138
138
|
## Dashboard
|
|
@@ -146,6 +146,12 @@ A runs index, a run detail page with the full step timeline, a replay diff view,
|
|
|
146
146
|
|
|
147
147
|
**→ [docs/dashboard.md](docs/dashboard.md)** for mounting, wiring in your own auth (Devise/Pundit/CanCanCan/anything), and what each page shows.
|
|
148
148
|
|
|
149
|
+
## Instrumentation
|
|
150
|
+
|
|
151
|
+
Belay publishes `ActiveSupport::Notifications` events for the run/step/gate lifecycle (`belay.run.started`, `belay.step.completed`, `belay.gate.paused`, and others) - the same mechanism Rails itself uses for `sql.active_record`, so you can pipe what Belay is doing into your own logs/APM without polling the dashboard.
|
|
152
|
+
|
|
153
|
+
**→ [docs/instrumentation.md](docs/instrumentation.md)** for the full event list and payloads.
|
|
154
|
+
|
|
149
155
|
## Roadmap
|
|
150
156
|
|
|
151
157
|
See [ROADMAP.md](ROADMAP.md).
|
data/lib/belay/executor.rb
CHANGED
|
@@ -186,6 +186,8 @@ module Belay
|
|
|
186
186
|
# human. Run#pause! already exists (running -> paused) for exactly this shape
|
|
187
187
|
# of "stopped, not failed, waiting on something external."
|
|
188
188
|
@run.pause!
|
|
189
|
+
ActiveSupport::Notifications.instrument("belay.step.needs_review", run_id: @run.id, step_id: existing.id,
|
|
190
|
+
name: existing.name, reason: reason)
|
|
189
191
|
raise Error, "#{reason} - flagged needs_review; the run is paused until a human resolves it"
|
|
190
192
|
end
|
|
191
193
|
|
|
@@ -240,12 +242,37 @@ module Belay
|
|
|
240
242
|
output = run_block(step_run, context, step)
|
|
241
243
|
return [nil, nil] if output.equal?(AUTO_SETTLED) # auto_settle_gate! already wrote this row itself
|
|
242
244
|
|
|
243
|
-
|
|
244
|
-
[output, nil]
|
|
245
|
+
settle_succeeded!(step_run, step, context, output)
|
|
245
246
|
rescue StandardError => e
|
|
246
|
-
|
|
247
|
+
settle_failed!(step_run, step, context, e)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def settle_succeeded!(step_run, step, context, output)
|
|
251
|
+
status = settled_status(step_run)
|
|
252
|
+
step.update!(status: status, output: output, **extra_step_attrs(step_run, context))
|
|
253
|
+
instrument_step_completed(step, status: status)
|
|
254
|
+
[output, nil]
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def settle_failed!(step_run, step, context, error)
|
|
258
|
+
step.update!(status: "failed", error: { "class" => error.class.name, "message" => error.message },
|
|
247
259
|
**extra_step_attrs(step_run, context))
|
|
248
|
-
|
|
260
|
+
instrument_step_completed(step, status: "failed")
|
|
261
|
+
[nil, error]
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# One event per settled attempt (success or failure alike), not separate
|
|
265
|
+
# started/completed events - the same consolidation ActionController's own
|
|
266
|
+
# process_action.action_controller notification makes (one event, status: in
|
|
267
|
+
# the payload), rather than one event per outcome. A step's own row already
|
|
268
|
+
# records attempt: - a retried step fires this once per attempt, same as it
|
|
269
|
+
# gets its own Step row per attempt.
|
|
270
|
+
def instrument_step_completed(step, status:)
|
|
271
|
+
ActiveSupport::Notifications.instrument(
|
|
272
|
+
"belay.step.completed",
|
|
273
|
+
run_id: @run.id, step_id: step.id, position: step.position, name: step.name,
|
|
274
|
+
kind: step.kind, status: status, attempt: step.attempt
|
|
275
|
+
)
|
|
249
276
|
end
|
|
250
277
|
|
|
251
278
|
# Wrapped in @run.heartbeat_while, not called bare - this is the one call in
|
|
@@ -311,6 +338,8 @@ module Belay
|
|
|
311
338
|
def auto_settle_gate!(step, context_key, rule)
|
|
312
339
|
step.update!(status: "succeeded", output: nil, input: { "context_key" => context_key },
|
|
313
340
|
rule_applied_id: rule.id)
|
|
341
|
+
ActiveSupport::Notifications.instrument("belay.gate.auto_approved", run_id: @run.id, step_id: step.id,
|
|
342
|
+
name: step.name, rule_id: rule.id)
|
|
314
343
|
AUTO_SETTLED
|
|
315
344
|
end
|
|
316
345
|
|
|
@@ -347,6 +376,8 @@ module Belay
|
|
|
347
376
|
@run.pause!
|
|
348
377
|
@run.release_lease!
|
|
349
378
|
end
|
|
379
|
+
ActiveSupport::Notifications.instrument("belay.gate.paused", run_id: @run.id, step_id: step.id,
|
|
380
|
+
name: step.name)
|
|
350
381
|
throw :gate_paused
|
|
351
382
|
end
|
|
352
383
|
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Belay
|
|
8
|
+
# Calls Google's Gemini API directly over Net::HTTP (stdlib) - same
|
|
9
|
+
# reasoning as AnthropicProvider/OpenAIProvider: no SDK gem, so no second
|
|
10
|
+
# gem's own version matrix for this gem to track for a provider this
|
|
11
|
+
# central.
|
|
12
|
+
#
|
|
13
|
+
# Endpoint and field names verified against Google's own REST reference
|
|
14
|
+
# (ai.google.dev/api/generate-content) as of 2026-10-28.
|
|
15
|
+
#
|
|
16
|
+
# The API key goes in the request URL's own ?key= query param, not a
|
|
17
|
+
# header - that's Gemini's own convention, unlike Anthropic's x-api-key or
|
|
18
|
+
# OpenAI's Authorization: Bearer. URI.encode_www_form escapes it rather
|
|
19
|
+
# than interpolating it directly into the URL string, purely so a key
|
|
20
|
+
# containing a URL-meaningful character (unlikely, but API keys are
|
|
21
|
+
# opaque strings this gem didn't generate) can't corrupt the request line.
|
|
22
|
+
#
|
|
23
|
+
# schema: is passed through to generationConfig.responseSchema completely
|
|
24
|
+
# unvalidated and untranslated - Gemini's schema dialect uses its own,
|
|
25
|
+
# UPPERCASE type names ("OBJECT", "STRING", "INTEGER", ...), not standard
|
|
26
|
+
# JSON Schema's lowercase ones ("object", "string"), and doesn't support
|
|
27
|
+
# every JSON Schema keyword. A schema written for AnthropicProvider or
|
|
28
|
+
# OpenAIProvider will not work here unchanged - same spirit as
|
|
29
|
+
# AnthropicProvider's additionalProperties: false requirement, just a
|
|
30
|
+
# different provider-specific dialect. See docs/providers.md.
|
|
31
|
+
#
|
|
32
|
+
# finishReason is mapped to the same StructuredOutputError reason: values
|
|
33
|
+
# the other two providers use, so a caller can branch on reason: the same
|
|
34
|
+
# way regardless of which provider is configured: "SAFETY"/"RECITATION"
|
|
35
|
+
# (the model declined to produce matching content) -> "refusal";
|
|
36
|
+
# "MAX_TOKENS" -> "max_tokens". Like AnthropicProvider/OpenAIProvider, this
|
|
37
|
+
# check only runs when schema: was given - see OpenAIProvider's own comment
|
|
38
|
+
# for why a refusal/truncation on a plain-text call isn't this gem's call
|
|
39
|
+
# to second-guess.
|
|
40
|
+
class GeminiProvider < Provider
|
|
41
|
+
API_BASE = "https://generativelanguage.googleapis.com/v1beta/models"
|
|
42
|
+
|
|
43
|
+
# Raised on any non-2xx response. Gemini's own error body (JSON, usually
|
|
44
|
+
# {"error": {"code":, "message":, "status":}}) is preserved verbatim in
|
|
45
|
+
# the message, same reasoning as AnthropicProvider::RequestError.
|
|
46
|
+
class RequestError < Error
|
|
47
|
+
attr_reader :status
|
|
48
|
+
|
|
49
|
+
def initialize(status:, body:)
|
|
50
|
+
@status = status
|
|
51
|
+
super("Gemini API request failed with status #{status}: #{body}")
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Raised when schema: was given but the response can't be trusted to
|
|
56
|
+
# contain valid, complete schema-matching JSON. reason: is "refusal" or
|
|
57
|
+
# "max_tokens" - see the class comment above for which finishReason maps
|
|
58
|
+
# to each.
|
|
59
|
+
class StructuredOutputError < Error
|
|
60
|
+
attr_reader :reason, :response
|
|
61
|
+
|
|
62
|
+
def initialize(reason:, response:)
|
|
63
|
+
@reason = reason
|
|
64
|
+
@response = response
|
|
65
|
+
super("Gemini did not return schema-matching output (reason: #{reason.inspect}): #{response.inspect}")
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# pricing: {"gemini-..." => { input_per_million: <USD Float>, output_per_million: <USD Float> }, ...}
|
|
70
|
+
# Deliberately not pre-filled with Google's current rates - same
|
|
71
|
+
# reasoning as AnthropicProvider's own pricing: comment.
|
|
72
|
+
def initialize(api_key: ENV.fetch("GEMINI_API_KEY", nil), max_output_tokens: 4096, pricing: {})
|
|
73
|
+
super()
|
|
74
|
+
raise ArgumentError, "#{self.class} requires api_key: (or GEMINI_API_KEY)" if api_key.nil?
|
|
75
|
+
|
|
76
|
+
@api_key = api_key
|
|
77
|
+
@max_output_tokens = max_output_tokens
|
|
78
|
+
@pricing = pricing
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def call(model:, prompt:, schema: nil)
|
|
82
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
83
|
+
status, body = perform_request(model, request_body(prompt: prompt, schema: schema))
|
|
84
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
85
|
+
|
|
86
|
+
raise RequestError.new(status: status, body: body) unless status.between?(200, 299)
|
|
87
|
+
|
|
88
|
+
build_response(JSON.parse(body), model: model, schema: schema, latency_ms: latency_ms)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def request_body(prompt:, schema:)
|
|
94
|
+
body = {
|
|
95
|
+
contents: [{ parts: [{ text: prompt }] }],
|
|
96
|
+
generationConfig: { maxOutputTokens: @max_output_tokens }
|
|
97
|
+
}
|
|
98
|
+
return body if schema.nil?
|
|
99
|
+
|
|
100
|
+
body[:generationConfig] = body[:generationConfig].merge(responseMimeType: "application/json",
|
|
101
|
+
responseSchema: schema)
|
|
102
|
+
body
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def perform_request(model, body)
|
|
106
|
+
uri = URI("#{API_BASE}/#{model}:generateContent")
|
|
107
|
+
uri.query = URI.encode_www_form(key: @api_key)
|
|
108
|
+
response = Net::HTTP.post(uri, JSON.generate(body), { "content-type" => "application/json" })
|
|
109
|
+
[response.code.to_i, response.body]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def build_response(response, model:, schema:, latency_ms:)
|
|
113
|
+
usage = response["usageMetadata"] || {}
|
|
114
|
+
input_tokens, output_tokens = usage.values_at("promptTokenCount", "candidatesTokenCount")
|
|
115
|
+
|
|
116
|
+
ProviderResponse.new(
|
|
117
|
+
content: extract_content(response, schema: schema),
|
|
118
|
+
model: response["modelVersion"] || model,
|
|
119
|
+
input_tokens: input_tokens,
|
|
120
|
+
output_tokens: output_tokens,
|
|
121
|
+
cost_cents: cost_cents(model: model, input_tokens: input_tokens, output_tokens: output_tokens),
|
|
122
|
+
latency_ms: latency_ms
|
|
123
|
+
)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def extract_content(response, schema:)
|
|
127
|
+
schema ? extract_structured_output(response) : extract_text(response)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def extract_structured_output(response)
|
|
131
|
+
finish_reason = response.dig("candidates", 0, "finishReason")
|
|
132
|
+
if %w[SAFETY RECITATION].include?(finish_reason)
|
|
133
|
+
raise StructuredOutputError.new(reason: "refusal", response: response)
|
|
134
|
+
end
|
|
135
|
+
raise StructuredOutputError.new(reason: "max_tokens", response: response) if finish_reason == "MAX_TOKENS"
|
|
136
|
+
|
|
137
|
+
JSON.parse(extract_text(response))
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def extract_text(response)
|
|
141
|
+
part = response.dig("candidates", 0, "content", "parts", 0)
|
|
142
|
+
raise Error, "Gemini response had no text content part" if part.nil? || part["text"].nil?
|
|
143
|
+
|
|
144
|
+
part["text"]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def cost_cents(model:, input_tokens:, output_tokens:)
|
|
148
|
+
return nil if input_tokens.nil? || output_tokens.nil?
|
|
149
|
+
|
|
150
|
+
rates = @pricing[model]
|
|
151
|
+
return nil if rates.nil?
|
|
152
|
+
|
|
153
|
+
input_cost = input_tokens * rates.fetch(:input_per_million)
|
|
154
|
+
output_cost = output_tokens * rates.fetch(:output_per_million)
|
|
155
|
+
((input_cost + output_cost) / 1_000_000.0 * 100).round
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "json"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Belay
|
|
8
|
+
# Calls OpenAI's API directly over Net::HTTP (stdlib) - same reasoning as
|
|
9
|
+
# AnthropicProvider: no SDK gem, so no second gem's own version matrix for
|
|
10
|
+
# this gem to track for a provider this central. POST a JSON body, read a
|
|
11
|
+
# JSON body back is small enough that stdlib Net::HTTP + JSON covers it.
|
|
12
|
+
#
|
|
13
|
+
# api: picks which of OpenAI's two current text-generation APIs to call -
|
|
14
|
+
# :responses (default, OpenAI's own forward-looking API, POST /v1/responses)
|
|
15
|
+
# or :chat_completions (the older, still fully supported POST
|
|
16
|
+
# /v1/chat/completions). Both are real, current APIs, not a deprecated/
|
|
17
|
+
# replacement pair, and their request/response shapes differ enough
|
|
18
|
+
# (structured-output config lives at text.format vs response_format; usage
|
|
19
|
+
# keys are input_tokens/output_tokens vs prompt_tokens/completion_tokens;
|
|
20
|
+
# truncation and refusal surface completely differently - see
|
|
21
|
+
# extract_responses_content/extract_chat_completions_content below) that
|
|
22
|
+
# this class branches internally on @api rather than guessing one and
|
|
23
|
+
# forcing callers who need the other to go write their own provider. The
|
|
24
|
+
# public interface (#call) and everything around it (errors, cost, latency)
|
|
25
|
+
# is identical either way.
|
|
26
|
+
#
|
|
27
|
+
# Endpoint and field names verified against OpenAI's own public OpenAPI spec
|
|
28
|
+
# (github.com/openai/openai-openapi) as of 2026-10-28.
|
|
29
|
+
#
|
|
30
|
+
# schema: on :responses sets text.format: {type: "json_schema", name:,
|
|
31
|
+
# schema:, strict: true}; on :chat_completions it's response_format:
|
|
32
|
+
# {type: "json_schema", json_schema: {name:, schema:, strict: true}}.
|
|
33
|
+
# Unlike Anthropic, OpenAI requires a name: on the schema either way - a
|
|
34
|
+
# fixed, gem-chosen value (SCHEMA_NAME) is used throughout, since nothing
|
|
35
|
+
# about the schema format itself needs that name to be caller-visible.
|
|
36
|
+
# strict: true is always passed (not left to OpenAI's own default: false) -
|
|
37
|
+
# a non-strict schema is advisory only, and this gem's own
|
|
38
|
+
# StructuredOutputError handling below assumes a schema that was actually
|
|
39
|
+
# enforced.
|
|
40
|
+
#
|
|
41
|
+
# refusal/truncation are each mapped to the same StructuredOutputError
|
|
42
|
+
# reason: values AnthropicProvider uses ("refusal"/"max_tokens"), so a
|
|
43
|
+
# caller can branch on reason: the same way regardless of which provider is
|
|
44
|
+
# configured:
|
|
45
|
+
# :responses - a "refusal" content item on the output message ->
|
|
46
|
+
# "refusal"; status: "incomplete" with
|
|
47
|
+
# incomplete_details.reason == "max_output_tokens" ->
|
|
48
|
+
# "max_tokens"
|
|
49
|
+
# :chat_completions - message.refusal present -> "refusal";
|
|
50
|
+
# finish_reason == "length" -> "max_tokens"
|
|
51
|
+
# Both checks, like AnthropicProvider's own, only run when schema: was
|
|
52
|
+
# given - a refusal or truncation on a plain-text call isn't this gem's
|
|
53
|
+
# problem to second-guess; it surfaces as whatever text (or lack of it)
|
|
54
|
+
# the author's own code then has to deal with, same as any other provider.
|
|
55
|
+
# Genuinely two providers' worth of request/response handling behind one #call
|
|
56
|
+
# interface (see api: above), not one class that grew unchecked - same
|
|
57
|
+
# ClassLength exemption reasoning as Executor::StepRunner (executor.rb).
|
|
58
|
+
class OpenAIProvider < Provider # rubocop:disable Metrics/ClassLength
|
|
59
|
+
API_URLS = {
|
|
60
|
+
responses: URI("https://api.openai.com/v1/responses"),
|
|
61
|
+
chat_completions: URI("https://api.openai.com/v1/chat/completions")
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
SCHEMA_NAME = "belay_response"
|
|
65
|
+
|
|
66
|
+
# Raised on any non-2xx response. OpenAI's own error body (JSON, usually
|
|
67
|
+
# {"error": {"type":, "message":}}) is preserved verbatim in the message,
|
|
68
|
+
# same reasoning as AnthropicProvider::RequestError.
|
|
69
|
+
class RequestError < Error
|
|
70
|
+
attr_reader :status
|
|
71
|
+
|
|
72
|
+
def initialize(status:, body:)
|
|
73
|
+
@status = status
|
|
74
|
+
super("OpenAI API request failed with status #{status}: #{body}")
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Raised when schema: was given but the response can't be trusted to
|
|
79
|
+
# contain valid, complete schema-matching JSON. reason: is "refusal" or
|
|
80
|
+
# "max_tokens" - see the class comment above for exactly what maps to
|
|
81
|
+
# each, per api:.
|
|
82
|
+
class StructuredOutputError < Error
|
|
83
|
+
attr_reader :reason, :response
|
|
84
|
+
|
|
85
|
+
def initialize(reason:, response:)
|
|
86
|
+
@reason = reason
|
|
87
|
+
@response = response
|
|
88
|
+
super("OpenAI did not return schema-matching output (reason: #{reason.inspect}): #{response.inspect}")
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# pricing: {"gpt-..." => { input_per_million: <USD Float>, output_per_million: <USD Float> }, ...}
|
|
93
|
+
# Deliberately not pre-filled with OpenAI's current rates - same reasoning
|
|
94
|
+
# as AnthropicProvider's own pricing: comment.
|
|
95
|
+
def initialize(api_key: ENV.fetch("OPENAI_API_KEY", nil), api: :responses, max_tokens: 4096, pricing: {})
|
|
96
|
+
super()
|
|
97
|
+
raise ArgumentError, "#{self.class} requires api_key: (or OPENAI_API_KEY)" if api_key.nil?
|
|
98
|
+
raise ArgumentError, "api: must be one of #{API_URLS.keys.inspect}, got #{api.inspect}" unless API_URLS.key?(api)
|
|
99
|
+
|
|
100
|
+
@api_key = api_key
|
|
101
|
+
@api = api
|
|
102
|
+
@max_tokens = max_tokens
|
|
103
|
+
@pricing = pricing
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def call(model:, prompt:, schema: nil)
|
|
107
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
108
|
+
status, body = perform_request(request_body(model: model, prompt: prompt, schema: schema))
|
|
109
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
110
|
+
|
|
111
|
+
raise RequestError.new(status: status, body: body) unless status.between?(200, 299)
|
|
112
|
+
|
|
113
|
+
build_response(JSON.parse(body), model: model, schema: schema, latency_ms: latency_ms)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
def request_body(model:, prompt:, schema:)
|
|
119
|
+
return responses_request_body(model, prompt, schema) if @api == :responses
|
|
120
|
+
|
|
121
|
+
chat_completions_request_body(model, prompt, schema)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def responses_request_body(model, prompt, schema)
|
|
125
|
+
body = { model: model, input: prompt, max_output_tokens: @max_tokens }
|
|
126
|
+
return body if schema.nil?
|
|
127
|
+
|
|
128
|
+
body.merge(text: { format: { type: "json_schema", name: SCHEMA_NAME, schema: schema, strict: true } })
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def chat_completions_request_body(model, prompt, schema)
|
|
132
|
+
body = { model: model, max_completion_tokens: @max_tokens, messages: [{ role: "user", content: prompt }] }
|
|
133
|
+
return body if schema.nil?
|
|
134
|
+
|
|
135
|
+
body.merge(response_format: { type: "json_schema",
|
|
136
|
+
json_schema: { name: SCHEMA_NAME, schema: schema, strict: true } })
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def perform_request(body)
|
|
140
|
+
response = Net::HTTP.post(API_URLS.fetch(@api), JSON.generate(body), request_headers)
|
|
141
|
+
[response.code.to_i, response.body]
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def request_headers
|
|
145
|
+
{ "Authorization" => "Bearer #{@api_key}", "content-type" => "application/json" }
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def build_response(response, model:, schema:, latency_ms:)
|
|
149
|
+
usage = response["usage"] || {}
|
|
150
|
+
input_tokens, output_tokens = usage_tokens(usage)
|
|
151
|
+
|
|
152
|
+
ProviderResponse.new(
|
|
153
|
+
content: extract_content(response, schema: schema),
|
|
154
|
+
model: response["model"] || model,
|
|
155
|
+
input_tokens: input_tokens,
|
|
156
|
+
output_tokens: output_tokens,
|
|
157
|
+
cost_cents: cost_cents(model: model, input_tokens: input_tokens, output_tokens: output_tokens),
|
|
158
|
+
latency_ms: latency_ms
|
|
159
|
+
)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def usage_tokens(usage)
|
|
163
|
+
keys = @api == :responses ? %w[input_tokens output_tokens] : %w[prompt_tokens completion_tokens]
|
|
164
|
+
usage.values_at(*keys)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def extract_content(response, schema:)
|
|
168
|
+
return extract_responses_content(response, schema: schema) if @api == :responses
|
|
169
|
+
|
|
170
|
+
extract_chat_completions_content(response, schema: schema)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def extract_responses_content(response, schema:)
|
|
174
|
+
message = find_output_message(response)
|
|
175
|
+
return extract_responses_structured_output(response, message) if schema
|
|
176
|
+
|
|
177
|
+
text = output_text(message)
|
|
178
|
+
raise Error, "OpenAI response had no output_text content" if text.nil?
|
|
179
|
+
|
|
180
|
+
text
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def extract_responses_structured_output(response, message)
|
|
184
|
+
raise StructuredOutputError.new(reason: "refusal", response: response) if refusal?(message)
|
|
185
|
+
raise StructuredOutputError.new(reason: "max_tokens", response: response) if responses_truncated?(response)
|
|
186
|
+
|
|
187
|
+
text = output_text(message)
|
|
188
|
+
raise Error, "OpenAI response had no output_text content" if text.nil?
|
|
189
|
+
|
|
190
|
+
JSON.parse(text)
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def find_output_message(response)
|
|
194
|
+
message = (response["output"] || []).find { |item| item["type"] == "message" }
|
|
195
|
+
raise Error, "OpenAI response had no output message" if message.nil?
|
|
196
|
+
|
|
197
|
+
message
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def output_text(message)
|
|
201
|
+
text_block = (message["content"] || []).find { |c| c["type"] == "output_text" }
|
|
202
|
+
text_block && text_block["text"]
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def refusal?(message)
|
|
206
|
+
(message["content"] || []).any? { |c| c["type"] == "refusal" }
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def responses_truncated?(response)
|
|
210
|
+
response["status"] == "incomplete" && response.dig("incomplete_details", "reason") == "max_output_tokens"
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def extract_chat_completions_content(response, schema:)
|
|
214
|
+
return extract_chat_completions_structured_output(response) if schema
|
|
215
|
+
|
|
216
|
+
content = response.dig("choices", 0, "message", "content")
|
|
217
|
+
raise Error, "OpenAI response had no message content" if content.nil?
|
|
218
|
+
|
|
219
|
+
content
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def extract_chat_completions_structured_output(response)
|
|
223
|
+
message = response.dig("choices", 0, "message") || {}
|
|
224
|
+
raise StructuredOutputError.new(reason: "refusal", response: response) if message["refusal"]
|
|
225
|
+
|
|
226
|
+
finish_reason = response.dig("choices", 0, "finish_reason")
|
|
227
|
+
raise StructuredOutputError.new(reason: "max_tokens", response: response) if finish_reason == "length"
|
|
228
|
+
|
|
229
|
+
content = message["content"]
|
|
230
|
+
raise Error, "OpenAI response had no message content" if content.nil?
|
|
231
|
+
|
|
232
|
+
JSON.parse(content)
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def cost_cents(model:, input_tokens:, output_tokens:)
|
|
236
|
+
return nil if input_tokens.nil? || output_tokens.nil?
|
|
237
|
+
|
|
238
|
+
rates = @pricing[model]
|
|
239
|
+
return nil if rates.nil?
|
|
240
|
+
|
|
241
|
+
input_cost = input_tokens * rates.fetch(:input_per_million)
|
|
242
|
+
output_cost = output_tokens * rates.fetch(:output_per_million)
|
|
243
|
+
((input_cost + output_cost) / 1_000_000.0 * 100).round
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
end
|
data/lib/belay/run.rb
CHANGED
|
@@ -36,15 +36,15 @@ module Belay
|
|
|
36
36
|
validates :workflow_version, presence: true
|
|
37
37
|
|
|
38
38
|
def start!
|
|
39
|
-
transition!(from: "pending", to: "running", started_at: Time.current)
|
|
39
|
+
transition!(from: "pending", to: "running", event: "started", started_at: Time.current)
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
def pause!
|
|
43
|
-
transition!(from: "running", to: "paused")
|
|
43
|
+
transition!(from: "running", to: "paused", event: "paused")
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
def resume!
|
|
47
|
-
transition!(from: "paused", to: "running")
|
|
47
|
+
transition!(from: "paused", to: "running", event: "resumed")
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
# None of complete!/fail!/cancel! below clear lease_owner/lease_expires_at - a
|
|
@@ -53,15 +53,16 @@ module Belay
|
|
|
53
53
|
# first, so a stale lease on a terminal run is never actually read. Would need
|
|
54
54
|
# fixing if anything ever queries leases without that status filter.
|
|
55
55
|
def complete!
|
|
56
|
-
transition!(from: "running", to: "completed", finished_at: Time.current)
|
|
56
|
+
transition!(from: "running", to: "completed", event: "completed", finished_at: Time.current)
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
def fail!(error: nil)
|
|
60
|
-
transition!(from: %w[pending running paused], to: "failed",
|
|
60
|
+
transition!(from: %w[pending running paused], to: "failed", event: "failed",
|
|
61
|
+
finished_at: Time.current, error: error)
|
|
61
62
|
end
|
|
62
63
|
|
|
63
64
|
def cancel!
|
|
64
|
-
transition!(from: %w[pending running paused], to: "cancelled", finished_at: Time.current)
|
|
65
|
+
transition!(from: %w[pending running paused], to: "cancelled", event: "cancelled", finished_at: Time.current)
|
|
65
66
|
end
|
|
66
67
|
|
|
67
68
|
# At most one, by construction: Executor stops at the FIRST gate whose when:
|
|
@@ -251,6 +252,13 @@ module Belay
|
|
|
251
252
|
# answer wasn't quite right, which is the opposite of "safe to auto-settle
|
|
252
253
|
# identically next time."
|
|
253
254
|
record_rule_decision!(gate, approved: true) if edited_payload.nil?
|
|
255
|
+
# Last, same ordering principle as ResumeRunJob/record_rule_decision! above: a
|
|
256
|
+
# subscriber that raises (ActiveSupport::Notifications runs subscribers
|
|
257
|
+
# synchronously, in-process - a slow or broken one is this gem's problem the
|
|
258
|
+
# moment it's subscribed) must never be able to leave the gate settled with no
|
|
259
|
+
# job enqueued, which placing this any earlier would risk.
|
|
260
|
+
ActiveSupport::Notifications.instrument("belay.gate.approved", run_id: id, step_id: gate.id, name: gate.name,
|
|
261
|
+
by: by, edited: !edited_payload.nil?)
|
|
254
262
|
end
|
|
255
263
|
|
|
256
264
|
# Rejects this run's pending gate: records a rejected Belay::Approval, settles
|
|
@@ -273,6 +281,8 @@ module Belay
|
|
|
273
281
|
end
|
|
274
282
|
|
|
275
283
|
record_rule_decision!(gate, approved: false)
|
|
284
|
+
ActiveSupport::Notifications.instrument("belay.gate.rejected", run_id: id, step_id: gate.id, name: gate.name,
|
|
285
|
+
by: by)
|
|
276
286
|
end
|
|
277
287
|
|
|
278
288
|
# The two outcomes a human can give a needs_review step (flag_needs_review!,
|
|
@@ -615,7 +625,19 @@ module Belay
|
|
|
615
625
|
# No concurrency guard: this reads status in memory, then does a plain update!. Two
|
|
616
626
|
# processes racing on the same run (stale in-memory copies) can both pass the check.
|
|
617
627
|
# Deferred to the leases work (Phase 1f) rather than solved here.
|
|
618
|
-
|
|
628
|
+
#
|
|
629
|
+
# event: is a separate argument from to:, not derived from it (e.g. "belay.run.#{to}") -
|
|
630
|
+
# to: "running" is reached by both start! and resume!, and those are two meaningfully
|
|
631
|
+
# different events for anything listening (ActiveSupport::Notifications.instrument
|
|
632
|
+
# below; see docs/instrumentation.md), the same way the comment above this method
|
|
633
|
+
# already explains why dispatch can't collapse them into one target-keyed graph either.
|
|
634
|
+
#
|
|
635
|
+
# Wrapping update! in .instrument, not instrumenting after the fact, is deliberate: a
|
|
636
|
+
# subscriber gets the real duration of the UPDATE for free (same convention
|
|
637
|
+
# ActiveRecord's own sql.active_record notification uses), and a failed update! never
|
|
638
|
+
# fires the event at all - nothing here is misreported as a transition that didn't
|
|
639
|
+
# actually happen.
|
|
640
|
+
def transition!(from:, to:, event:, **attrs)
|
|
619
641
|
allowed_from = Array(from)
|
|
620
642
|
unless allowed_from.include?(status)
|
|
621
643
|
raise InvalidTransitionError,
|
|
@@ -623,7 +645,8 @@ module Belay
|
|
|
623
645
|
"(expected current status to be #{allowed_from.inspect})"
|
|
624
646
|
end
|
|
625
647
|
|
|
626
|
-
|
|
648
|
+
payload = { run_id: id, workflow: workflow, workflow_version: workflow_version, from: status, to: to }
|
|
649
|
+
ActiveSupport::Notifications.instrument("belay.run.#{event}", payload) { update!(**attrs, status: to) }
|
|
627
650
|
end
|
|
628
651
|
end
|
|
629
652
|
end
|
data/lib/belay/version.rb
CHANGED
data/lib/belay.rb
CHANGED
|
@@ -20,6 +20,8 @@ module Belay
|
|
|
20
20
|
require_relative "belay/provider"
|
|
21
21
|
require_relative "belay/fake_provider"
|
|
22
22
|
require_relative "belay/anthropic_provider"
|
|
23
|
+
require_relative "belay/openai_provider"
|
|
24
|
+
require_relative "belay/gemini_provider"
|
|
23
25
|
require_relative "belay/executor"
|
|
24
26
|
require_relative "belay/resume_run_job"
|
|
25
27
|
require_relative "belay/janitor_job"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: belay
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Swakhar Dey
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
11
|
+
date: 2026-10-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: actionpack
|
|
@@ -180,7 +180,9 @@ files:
|
|
|
180
180
|
- lib/belay/engine.rb
|
|
181
181
|
- lib/belay/executor.rb
|
|
182
182
|
- lib/belay/fake_provider.rb
|
|
183
|
+
- lib/belay/gemini_provider.rb
|
|
183
184
|
- lib/belay/janitor_job.rb
|
|
185
|
+
- lib/belay/openai_provider.rb
|
|
184
186
|
- lib/belay/provider.rb
|
|
185
187
|
- lib/belay/provider_response.rb
|
|
186
188
|
- lib/belay/resume_run_job.rb
|