belay 0.1.0

data/lib/belay/configuration.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Belay
+  class Configuration
+    # Seconds a worker holds a run before the janitor may reclaim it. See
+    # DESIGN_DECISIONS.md #12 for why 300/30.
+    attr_accessor :lease_duration
+
+    # Seconds between heartbeats while a worker has a step in flight.
+    attr_accessor :heartbeat_interval
+
+    # The Belay::Provider ctx.llm calls. nil by default - a workflow with no
+    # llm-calling steps never touches this, but a step that calls ctx.llm without
+    # one configured gets a clear error instead of a confusing NoMethodError on nil.
+    attr_accessor :provider
+
+    # Gates who can load Belay::Engine's mounted dashboard: a ->(request) { boolean }
+    # proc, called on every request. nil by default - deliberately NOT "open to
+    # anyone" by default, since the dashboard shows prompts, responses, costs and
+    # approve/reject actions. Belay::ApplicationController raises a clear error if
+    # this is still nil when a request actually arrives, rather than quietly
+    # serving it unauthenticated. See lib/belay/application_controller.rb.
+    attr_accessor :authenticate_with
+
+    # How long a ctx.llm(..., cacheable: true) entry stays valid, in seconds.
+    # nil means never expires. One global setting, not a per-call override -
+    # cacheable: is already the one knob a step author needs; a second
+    # ttl: keyword on every ctx.llm call would be more API surface for a
+    # distinction most workflows don't need to make per-step.
+    attr_accessor :cache_ttl
+
+    # How many consistent, unedited approve!s of the exact same gate occurrence
+    # (same workflow_version, gate_name, and input/outputs - see Belay::Rule)
+    # it takes before that occurrence is learned as a rule. One global setting,
+    # same reasoning as cache_ttl above - a per-gate override is more API
+    # surface than any workflow has asked for yet. Does not by itself make
+    # anything auto-approve; see feat(rules): auto-settle a gate when a
+    # learned rule matches, before ever pausing.
+    attr_accessor :rule_approval_threshold
+
+    # Extra CSS appended, in the layout's own <style> tag, right after the
+    # dashboard's compiled Tailwind build - not a replacement for it. Appended
+    # rather than merged means a same-specificity rule (redeclaring .btn-primary,
+    # say) wins by ordinary cascade order, no !important required; nil by
+    # default renders nothing extra. A plain String, not a path or an asset -
+    # this gem has no asset pipeline opinion to lean on (see Engine.stylesheet's
+    # own comment), so the host app owns wherever this string actually comes from.
+    attr_accessor :custom_css
+
+    def initialize
+      @lease_duration = 300
+      @heartbeat_interval = 30
+      @provider = nil
+      @authenticate_with = nil
+      @cache_ttl = 7 * 24 * 60 * 60 # 7 days
+      @rule_approval_threshold = 3
+      @custom_css = nil
+    end
+  end
+end

data/lib/belay/context.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "json_schemer"
+
+module Belay
+  # Passed as `ctx` to a step block (see README's Target API). Exposes the run's input
+  # and prior steps' outputs as dotted method calls - ctx.invoice_id, ctx.extract - built
+  # from two plain hashes, with no coupling to a specific Belay::Workflow class. Also
+  # exposes step_idempotency_key: a plain reader, not a method_missing-served dynamic
+  # key, since it's one fixed value for whichever step this Context was built for, not
+  # a named entry in input or outputs. Belay::Executor is the only thing that passes a
+  # real one (run_id + position, DESIGN_DECISIONS #3); it defaults to nil so Context
+  # stays constructible standalone, as every existing spec already does.
+  #
+  # The freeze below is shallow: @input/@outputs are frozen hashes, but a value inside
+  # them (e.g. ctx.extract returning a Hash) is not. If the executor ever reuses one
+  # Context, or shares an output value, across more than one step, a step mutating
+  # its own ctx.extract in place could corrupt what a later step reads. Not guarded
+  # here - the executor's construction pattern (one Context per step? shared?) isn't
+  # decided yet, and deep-freezing arbitrary step output is real added cost to pay for
+  # a risk that depends on that undecided design.
+  class Context
+    # Raised when schema: was given to ctx.llm but the provider's response doesn't
+    # match it. errors is JSONSchemer's own list of validation error hashes, not a
+    # single summarized string, so a caller can inspect exactly what failed.
+    class SchemaValidationError < Error
+      attr_reader :errors
+
+      def initialize(errors:, content:, schema:)
+        @errors = errors
+        messages = errors.map { |e| e["error"] }.join("; ")
+        super("ctx.llm's response did not match schema: #{messages} (content: #{content.inspect}, " \
+              "schema: #{schema.inspect})")
+      end
+    end
+
+    attr_reader :input, :outputs, :step_idempotency_key
+
+    # model_override: is Run#replay(model:)'s whole mechanism - nil for every
+    # ordinary run (the common case: #llm just uses whatever model: the step's
+    # own block passed). Executor::StepRunner#build_context is the only real
+    # caller that ever passes one, reading it off @run.replay_model.
+    def initialize(input: {}, outputs: {}, step_idempotency_key: nil, model_override: nil)
+      @input = symbolize_without_collision!(input, "input")
+      @outputs = symbolize_without_collision!(outputs, "prior step output")
+      @step_idempotency_key = step_idempotency_key
+      @model_override = model_override
+      # Mutated in place by #llm below, not reassigned - this object freezes itself
+      # at the end of #initialize, which would make `@llm_metadata = ...` raise
+      # FrozenError from inside #llm. Mutating the Hash this ivar already points to
+      # is a different, allowed thing: freezing Context doesn't freeze objects it
+      # merely references.
+      @llm_metadata = {}
+      validate_no_name_collisions!
+      freeze
+    end
+
+    # Calls Belay.config.provider, validates the response against schema: (if given)
+    # with JSONSchemer, and returns response.content - a step block's return value
+    # becomes its checkpointed output, so #llm's return value IS the step's output,
+    # not a wrapper around it.
+    #
+    # Validated here, provider-agnostically, rather than inside each Provider:
+    # AnthropicProvider's own schema: already asks Anthropic's API to constrain its
+    # output, but FakeProvider enforces nothing at all - without this, a spec's
+    # scripted response could silently not match the schema: it claims to, and
+    # nothing would catch that until production. This is the one place that's true
+    # for every provider.
+    #
+    # schema: must use string keys for its own keywords (symbol-keyed schemas are
+    # deep-stringified below first) and set additionalProperties: false on every
+    # object, per Anthropic's own requirement for the same schema when a provider
+    # like AnthropicProvider forwards it as output_config.format.schema.
+    # model: passed here is a default, not the final word - model_override
+    # (set only on a replay's shadow run) takes precedence, so a replayed
+    # llm_call step actually calls the overriding model without the workflow
+    # body itself needing to know anything about replay.
+    #
+    # cacheable: false by default - opt-in, exact-match caching (DESIGN_
+    # DECISIONS #10, Belay::CacheEntry). Deliberately not inferred from
+    # kind: or anything else about the step: "decision steps only" can't be
+    # enforced mechanically (kind: "llm_call" covers both a classification
+    # AND an extraction call; Belay has no way to tell them apart), so this
+    # is a plain, explicit choice the step author makes, the same contract
+    # idempotent: true already uses elsewhere - only set this on a step
+    # whose ENTIRE prompt text fully and uniquely determines the correct
+    # answer. A short, self-contained classification prompt usually
+    # qualifies; an extraction prompt that references a large document
+    # (especially over a vision/file attachment, whose bytes aren't part of
+    # what gets hashed) usually doesn't - two different documents could
+    # share a cache key and get back the wrong answer.
+    def llm(model:, prompt:, schema: nil, cacheable: false)
+      effective_model = @model_override || model
+      provider = resolve_provider!
+      call_args = { provider: provider, model: effective_model, prompt: prompt, schema: schema }
+
+      cached = fetch_cached_llm_response(**call_args) if cacheable
+      return cached unless cached.nil?
+
+      response = provider.call(model: effective_model, prompt: prompt, schema: schema)
+      record_llm_metadata!(provider: provider, prompt: prompt, response: response)
+      validate_against_schema!(response.content, schema) if schema
+      store_cached_llm_response!(**call_args, content: response.content) if cacheable
+      response.content
+    end
+
+    # {} if #llm was never called this step - nothing to persist onto the step row,
+    # same as any step that made no provider call. Frozen dup, same reasoning as
+    # Workflow#steps/#inputs and FakeProvider#calls: a caller reaching in and
+    # mutating this would corrupt what StepRunner later reads to update the row.
+    def llm_metadata
+      @llm_metadata.dup.freeze
+    end
+
+    def method_missing(name, *args)
+      return super unless args.empty? && !block_given?
+
+      if @input.key?(name)
+        @input[name]
+      elsif @outputs.key?(name)
+        @outputs[name]
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      @input.key?(name) || @outputs.key?(name) || super
+    end
+
+    private
+
+    def resolve_provider!
+      Belay.config.provider || raise(Error, "Belay.config.provider is not set - configure one before calling " \
+                                            "ctx.llm, e.g. Belay.configure { |c| c.provider = " \
+                                            "Belay::AnthropicProvider.new }")
+    end
+
+    # nil on a cache miss (or an expired entry) - the caller falls through to
+    # a real provider call either way, same as CacheEntry.fetch's own nil
+    # means to its two callers. Assumes a cached response's content is never
+    # itself nil, which ProviderResponse's own validation already guarantees
+    # (content is required) - there's no cacheable response this could ever
+    # mistake for a miss.
+    #
+    # input_tokens/output_tokens/cost_cents/latency_ms are 0, not nil, for a
+    # hit - nil already means something specific elsewhere (e.g. cost_cents
+    # is nil when no pricing: was configured for a REAL call - "we don't
+    # know"), and a hit's true cost really is zero, not unknown.
+    def fetch_cached_llm_response(provider:, model:, prompt:, schema:)
+      content = CacheEntry.fetch(provider: provider, model: model, prompt: prompt, schema: schema)
+      return nil if content.nil?
+
+      @llm_metadata.merge!(provider: provider.class.name, input: prompt, model: model, cache_hit: true,
+                           input_tokens: 0, output_tokens: 0, cost_cents: 0, latency_ms: 0)
+      content
+    end
+
+    # Called only after validate_against_schema! has already passed (see #llm) -
+    # a response that fails validation never reaches here, so no cacheable: true
+    # key is ever stuck returning a bad answer forever.
+    def store_cached_llm_response!(provider:, model:, prompt:, schema:, content:)
+      CacheEntry.store!(provider: provider, model: model, prompt: prompt, schema: schema, content: content)
+    end
+
+    # Recorded before validating, not after: provider.call above already made a real,
+    # billed request by the time we know whether its content matches schema:. A response
+    # that fails validation raises below and this step's row ends up "failed" -
+    # run_attempt's dual-merge (see executor.rb) persists whatever's in @llm_metadata onto
+    # that failed row regardless, which only works if the merge already happened before
+    # the raise, not after it.
+    #
+    # input: prompt, not :prompt - matches belay_steps' own column name exactly (same
+    # reasoning as every other key here: extra_step_attrs below merges this whole hash
+    # straight into step.update!, with no translation step in between), so the dashboard's
+    # run page has something real to show as "prompt" - the exact value passed to
+    # ctx.llm, not a summary of it.
+    def record_llm_metadata!(provider:, prompt:, response:)
+      fields = response.to_h.slice(:model, :input_tokens, :output_tokens, :cost_cents, :latency_ms)
+      @llm_metadata.merge!(provider: provider.class.name, input: prompt, cache_hit: false, **fields)
+    end
+
+    def validate_against_schema!(content, schema)
+      document = json_normalize(content)
+      schemer = JSONSchemer.schema(json_normalize(schema))
+      return if schemer.valid?(document)
+
+      raise SchemaValidationError.new(errors: schemer.validate(document).to_a, content: content, schema: schema)
+    end
+
+    # Hash#deep_stringify_keys only converts hash keys, not a bare Symbol used as a
+    # VALUE - required: [:total_cents] (a real, common JSON Schema shape, not an
+    # edge case) comes through untouched, so "total_cents" (the stringified
+    # document's actual key) never matches :total_cents. JSON itself has no symbol
+    # type; normalizing every symbol anywhere in the structure, key or value, is
+    # what makes a Ruby-idiomatic symbol-keyed/valued schema or FakeProvider script
+    # behave the same as what the real API actually returns (plain JSON.parse'd
+    # strings throughout).
+    def json_normalize(value)
+      case value
+      when Hash
+        value.to_h { |k, v| [k.to_s, json_normalize(v)] }
+      when Array
+        value.map { |v| json_normalize(v) }
+      when Symbol
+        value.to_s
+      else
+        value
+      end
+    end
+
+    def validate_no_name_collisions!
+      name_overlap = @input.keys & @outputs.keys
+      unless name_overlap.empty?
+        raise ArgumentError, "input and prior step output names collide: #{name_overlap.inspect} - rename one"
+      end
+
+      # self.class.method_defined?, not respond_to?: respond_to? would consult
+      # respond_to_missing? above, which checks @input/@outputs themselves - every key
+      # would "collide" with its own dynamic accessor. method_defined? only sees real,
+      # statically-defined methods: #input/#outputs (attr_reader) and inherited
+      # Object/Kernel ones (class, hash, freeze, send, ...) - not method_missing-served
+      # ones. A key literally named "input" would otherwise shadow the reader above
+      # with no error.
+      reserved = (@input.keys + @outputs.keys).select { |key| self.class.method_defined?(key) }
+      return if reserved.empty?
+
+      raise ArgumentError,
+            "input/output name(s) #{reserved.inspect} collide with an existing method on #{self.class} - rename them"
+    end
+
+    def symbolize_without_collision!(hash, label)
+      symbolized = hash.transform_keys(&:to_sym)
+      if symbolized.size != hash.size
+        raise ArgumentError, "two #{label} keys collapse to the same name once symbolized: #{hash.keys.inspect}"
+      end
+
+      symbolized.freeze
+    end
+  end
+end

data/lib/belay/daily_stats.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Belay
+  # How many REAL provider calls (an llm_call step where cache_hit: false - a
+  # hit or a rule-auto-settled gate made no call at all) this install made per
+  # 100 runs, by day. Backs the dashboard's Stats page - plain Ruby, not an
+  # ActiveRecord model of its own: there's no belay_daily_stats table, this is
+  # a read-only view over belay_runs/belay_steps' own created_at timestamps.
+  class DailyStats
+    Row = Struct.new(:date, :runs, :real_calls, :calls_per_100_runs, keyword_init: true)
+
+    # Calls are grouped by the STEP's own day, not the owning run's - a run
+    # can pause on a gate for a real, unbounded amount of time (nothing times
+    # a pending approval out; Phase 4's own rule-learning needs N separate,
+    # time-separated approvals to even happen), so a run created Jan 1 whose
+    # gate isn't approved until Jan 15 makes its post-gate steps' real calls
+    # on Jan 15, not Jan 1. Grouping by the run's day instead would silently
+    # mutate Jan 1's historical count every time an old run resumes - exactly
+    # backwards for a chart whose whole purpose is "is this week's cost down,"
+    # and would make a past day's bar change on a later page load. Runs are
+    # still grouped by their OWN created_at below - "how many runs started
+    # that day" is a legitimately different question from "how many real
+    # calls happened that day," and conflating them was the bug this comment
+    # replaced.
+    def self.for_range
+      calls_by_day = calls_by_day_counts
+      runs_by_day = runs_by_day_counts
+
+      days = (calls_by_day.keys + runs_by_day.keys).uniq.sort
+      days.map { |date| build_row(date, runs_by_day.fetch(date, 0), calls_by_day.fetch(date, 0)) }
+    end
+
+    def self.calls_by_day_counts
+      normalized_counts(Step.llm_call.where(cache_hit: false).group("DATE(created_at)").count)
+    end
+    private_class_method :calls_by_day_counts
+
+    def self.runs_by_day_counts
+      normalized_counts(Run.group("DATE(created_at)").count)
+    end
+    private_class_method :runs_by_day_counts
+
+    # .group("DATE(...)").count's hash keys come back as plain Strings on some
+    # adapter/Rails version combinations, real Date objects on others - a raw
+    # SQL group-by fragment (not an actual typed column) gives ActiveRecord
+    # nothing to infer the cast from. Normalized here, once, so every lookup
+    # and every Row's date below is reliably a Date, regardless of which
+    # shape the DB adapter actually returned.
+    def self.normalized_counts(counts_by_raw_date)
+      counts_by_raw_date.transform_keys { |key| key.is_a?(Date) ? key : Date.parse(key.to_s) }
+    end
+    private_class_method :normalized_counts
+
+    def self.build_row(date, runs, calls)
+      Row.new(date: date, runs: runs, real_calls: calls,
+              calls_per_100_runs: runs.zero? ? 0.0 : (calls.to_f / runs * 100).round(1))
+    end
+    private_class_method :build_row
+  end
+end

data/lib/belay/engine.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "rails/engine"
+
+module Belay
+  # Makes Belay mountable in a host app: mount Belay::Engine => "/belay" in
+  # config/routes.rb. isolate_namespace keeps every route helper, controller and
+  # view path under the Belay:: namespace, so a host app's own RunsController or
+  # route named "runs" never collides with this gem's. isolate_namespace also
+  # gives the engine its own Zeitwerk autoloader for app/controllers and
+  # app/views - unlike every plain Ruby class under lib/belay/ (Run, Step,
+  # Approval, ...), which is deliberately require_relative'd from lib/belay.rb
+  # instead, with no Rails autoloading convention to lean on (see those files'
+  # own comments on this). Controllers are different: Rails engines are BUILT to
+  # autoload app/controllers this way, so this is the one place in the gem that
+  # actually uses it, via the conventional app/controllers + app/views +
+  # config/routes.rb directory layout at the gem root (not under lib/ -
+  # belay.gemspec's spec.files lists app/**/* and config/routes.rb alongside
+  # lib/**/* for exactly this).
+  class Engine < ::Rails::Engine
+    isolate_namespace Belay
+
+    # The dashboard's CSS, inlined into the layout via <style> rather than
+    # served through a <link> and whichever asset pipeline (Sprockets,
+    # Propshaft, none at all) the host app happens to have configured - that's
+    # exactly the kind of host-app assumption Belay has never made anywhere
+    # else (no Zeitwerk assumption, explicit class_name: throughout, routes
+    # drawn from the engine's own config/routes.rb). app/assets/belay/
+    # application.css is a committed, precompiled Tailwind build (see
+    # package.json's build:css) - there is no runtime build step, and this
+    # gem's own end users never run npm.
+    #
+    # Memoized at the class level (not per-request, unlike an instance
+    # variable in a helper) - the file never changes while the process is
+    # running, so re-reading it on every single request would be pure waste.
+    def self.stylesheet
+      @stylesheet ||= root.join("app/assets/belay/application.css").read
+    end
+  end
+end