prompt_canary 0.3.0

data/lib/prompt_canary/cli/commands/history.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class CLI
+    module Commands
+      module History
+        private
+
+        def history(args)
+          options = {}
+          OptionParser.new do |opts|
+            opts.on("--since PERIOD") { |p| options[:since] = p }
+          end.parse!(args)
+
+          prompt_name = args.first
+          if prompt_name.nil?
+            warn "Usage: prompt_canary history PROMPT_CLASS [--since Nd]"
+            exit 1
+          end
+
+          if options[:since] && options[:since] !~ /\A[1-9]\d*d\z/
+            warn "Invalid --since format: #{options[:since].inspect}. Expected format: Nd (e.g. 7d)"
+            exit 1
+          end
+
+          require "prompt_canary/storage/active_record_adapter" unless defined?(PromptCanary::PromptEvent)
+          history_scope(prompt_name, options[:since]).each { |e| puts format_event(e) }
+        end
+
+        def history_scope(prompt_name, since_period)
+          scope = PromptCanary::PromptEvent.where(prompt: prompt_name).order(:recorded_at)
+          return scope unless since_period
+
+          scope.where("recorded_at >= ?", Time.now - (since_period.to_i * 24 * 60 * 60))
+        end
+
+        def format_event(event)
+          parts = [event.recorded_at.strftime("%Y-%m-%d %H:%M"),
+                   event.event.upcase.ljust(12),
+                   event.version,
+                   event_change(event),
+                   "[#{event.triggered_by}]",
+                   event_metric(event)].compact
+          parts.join("  ").strip
+        end
+
+        def event_change(event)
+          if event.previous_status || event.new_status
+            "#{event.previous_status} → #{event.new_status}"
+          elsif !event.previous_percent.nil? && !event.new_percent.nil?
+            "#{event.previous_percent}% → #{event.new_percent}%"
+          end
+        end
+
+        def event_metric(event)
+          return unless event.triggering_metric
+
+          "#{event.triggering_metric} #{event.triggering_value} > #{event.triggering_threshold}"
+        end
+      end
+    end
+  end
+end

data/lib/prompt_canary/cli/commands/status.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class CLI
+    module Commands
+      module Status
+        private
+
+        def status(args)
+          prompt_name = args.first
+          if prompt_name.nil?
+            warn "Usage: prompt_canary status PROMPT_CLASS"
+            exit 1
+          end
+
+          prompt_class = begin
+            Object.const_get(prompt_name)
+          rescue NameError
+            warn "Unknown prompt class: #{prompt_name}"
+            exit 1
+          end
+
+          primary_name = prompt_class.primary_version.name
+          puts "#{prompt_name}:"
+          prompt_class.versions.each do |v|
+            label = version_status(prompt_class.name, v.name, primary_name)
+            percent = v.rollout.fetch(:percent, 0)
+            traffic = percent.positive? ? "#{percent}% traffic (canary)" : "no canary traffic"
+            puts "  #{v.name.ljust(6)}  #{label.ljust(12)}  #{traffic}"
+          end
+        end
+
+        def version_status(prompt_name, version_name, primary_name)
+          if demoted_in_db?(prompt_name, version_name)
+            "DEMOTED"
+          elsif version_name == primary_name
+            "PRIMARY"
+          else
+            "CANDIDATE"
+          end
+        end
+
+        def demoted_in_db?(prompt_name, version_name)
+          return false unless defined?(PromptCanary::RolloutOverride)
+
+          PromptCanary::RolloutOverride
+            .where(prompt: prompt_name, version: version_name, rollout_override: 0)
+            .exists?
+        rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+          false
+        end
+      end
+    end
+  end
+end

data/lib/prompt_canary/cli.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "cli/commands/history"
+require_relative "cli/commands/status"
+
+module PromptCanary
+  class CLI
+    include Commands::History
+    include Commands::Status
+
+    def run(args)
+      subcommand = args.shift
+      case subcommand
+      when "demote"   then demote(args)
+      when "promote"  then promote(args)
+      when "history"  then history(args)
+      when "status"   then status(args)
+      else
+        warn "Unknown command: #{subcommand}"
+        exit 1
+      end
+    end
+
+    private
+
+    def promote(args)
+      options = {}
+      OptionParser.new do |opts|
+        opts.on("--reason REASON") { |r| options[:reason] = r }
+      end.parse!(args)
+
+      prompt_name, version_name = args
+      if prompt_name.nil? || version_name.nil?
+        warn "Usage: prompt_canary promote PROMPT_CLASS VERSION [--reason REASON]"
+        exit 1
+      end
+
+      prompt_class = begin
+        Object.const_get(prompt_name)
+      rescue NameError
+        warn "Unknown prompt class: #{prompt_name}"
+        exit 1
+      end
+      PromptCanary.promote(prompt_class, version_name, reason: options[:reason])
+    end
+
+    def demote(args)
+      options = {}
+      OptionParser.new do |opts|
+        opts.on("--reason REASON") { |r| options[:reason] = r }
+      end.parse!(args)
+
+      prompt_name, version_name = args
+      if prompt_name.nil? || version_name.nil?
+        warn "Usage: prompt_canary demote PROMPT_CLASS VERSION [--reason REASON]"
+        exit 1
+      end
+
+      prompt_class = begin
+        Object.const_get(prompt_name)
+      rescue NameError
+        warn "Unknown prompt class: #{prompt_name}"
+        exit 1
+      end
+      PromptCanary.demote(prompt_class, version_name, reason: options[:reason])
+    end
+  end
+end

data/lib/prompt_canary/configuration.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Configuration
+    VALID_ADAPTERS = %i[anthropic].freeze
+    VALID_STORAGE  = %i[memory sqlite active_record].freeze
+
+    attr_reader :adapter, :storage
+
+    def adapter=(value)
+      unless VALID_ADAPTERS.include?(value)
+        raise ConfigurationError, "Unknown adapter: #{value.inspect}. Valid adapters: #{VALID_ADAPTERS.join(", ")}"
+      end
+
+      @adapter = value
+    end
+
+    def storage=(value)
+      unless VALID_STORAGE.include?(value)
+        raise ConfigurationError, "Unknown storage: #{value.inspect}. Valid storage: #{VALID_STORAGE.join(", ")}"
+      end
+
+      @storage = value
+    end
+
+    def validate!
+      raise ConfigurationError, "adapter is required" if adapter.nil?
+      raise ConfigurationError, "storage is required" if storage.nil?
+    end
+  end
+end

data/lib/prompt_canary/deployment.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  # rubocop:disable Metrics/ModuleLength
+  module Deployment
+    def set_canary(prompt_class, version_name, percent)
+      assert_valid_canary_percent!(percent)
+      assert_version_registered!(prompt_class, version_name)
+
+      if ar_storage?
+        require "prompt_canary/storage/active_record_adapter"
+        if RolloutOverride.where(prompt: prompt_class.name, version: version_name, rollout_override: 0).exists?
+          raise DemotedVersionError,
+                "#{version_name.inspect} is demoted — call restore before adjusting traffic"
+        end
+
+        prev_percent = effective_canary_percent(prompt_class, version_name)
+        override = RolloutOverride.find_or_initialize_by(prompt: prompt_class.name, version: version_name)
+        override.created_at ||= Time.now
+        override.update!(rollout_override: percent)
+        record_event(prompt: prompt_class.name, version: version_name, event: "canary_set",
+                     previous_percent: prev_percent, new_percent: percent, triggered_by: "manual")
+      else
+        version = prompt_class.versions.find { |v| v.name == version_name }
+        version.set_rollout!(percent)
+      end
+    end
+
+    def promote(prompt_class, version_name, reason: nil)
+      assert_version_registered!(prompt_class, version_name)
+
+      if ar_storage?
+        require "prompt_canary/storage/active_record_adapter"
+        prev_primary = effective_primary_name(prompt_class)
+        prev_status  = current_version_status(prompt_class, version_name)
+        override = PrimaryOverride.find_or_initialize_by(prompt: prompt_class.name)
+        override.created_at ||= Time.now
+        override.update!(version: version_name)
+        record_event(prompt: prompt_class.name, version: version_name, event: "promoted",
+                     previous_status: prev_status, new_status: "primary",
+                     reason: reason, triggered_by: "manual")
+        record_superseded_event(prompt_class, prev_primary, version_name)
+      else
+        prompt_class.promote_to_primary!(version_name)
+      end
+      publish("prompt_canary.promoted", prompt: prompt_class.name, version: version_name, reason: reason)
+    end
+
+    def demote(prompt_class, version_name, reason: nil, triggered_by: "manual",
+               triggering_metric: nil, triggering_value: nil, triggering_threshold: nil)
+      assert_can_demote_primary!(prompt_class, version_name)
+
+      if ar_storage?
+        require "prompt_canary/storage/active_record_adapter"
+        prev_percent = effective_canary_percent(prompt_class, version_name)
+        override = RolloutOverride.find_or_initialize_by(prompt: prompt_class.name, version: version_name)
+        override.created_at ||= Time.now
+        override.update!(rollout_override: 0)
+        record_event(
+          prompt: prompt_class.name, version: version_name, event: "demoted",
+          previous_percent: prev_percent, new_percent: 0, new_status: "demoted",
+          reason: reason, triggered_by: triggered_by,
+          triggering_metric: triggering_metric&.to_s,
+          triggering_value: triggering_value,
+          triggering_threshold: triggering_threshold
+        )
+      else
+        version = prompt_class.versions.find { |v| v.name == version_name }
+        version&.demote!
+      end
+      publish("prompt_canary.demoted", prompt: prompt_class.name, version: version_name, reason: reason)
+    end
+
+    def restore(prompt_class, version_name)
+      if ar_storage?
+        require "prompt_canary/storage/active_record_adapter"
+        RolloutOverride.where(prompt: prompt_class.name, version: version_name).delete_all
+        record_event(prompt: prompt_class.name, version: version_name, event: "restored",
+                     previous_status: "demoted", new_status: "candidate", triggered_by: "manual")
+      else
+        version = prompt_class.versions.find { |v| v.name == version_name }
+        version&.restore!
+      end
+      publish("prompt_canary.restored", prompt: prompt_class.name, version: version_name)
+    end
+
+    private
+
+    def record_superseded_event(prompt_class, prev_primary, promoted_name)
+      return unless prev_primary && prev_primary != promoted_name
+
+      record_event(prompt: prompt_class.name, version: prev_primary, event: "superseded",
+                   previous_status: "primary", new_status: "candidate", triggered_by: "manual")
+    end
+
+    def current_version_status(prompt_class, version_name)
+      return "primary" if effective_primary_name(prompt_class) == version_name
+      return "demoted" if RolloutOverride.where(
+        prompt: prompt_class.name, version: version_name, rollout_override: 0
+      ).exists?
+
+      "candidate"
+    rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+      "candidate"
+    end
+
+    def assert_version_registered!(prompt_class, version_name)
+      return if prompt_class.versions.any? { |v| v.name == version_name }
+
+      raise UnknownVersionError, "#{version_name.inspect} is not a registered version of #{prompt_class}"
+    end
+
+    def assert_valid_canary_percent!(percent)
+      raise ArgumentError, "percent must be positive — use `demote` to stop traffic" if percent.zero?
+    end
+
+    def assert_can_demote_primary!(prompt_class, version_name)
+      return unless effective_primary_name(prompt_class) == version_name
+      return unless no_viable_candidate?(prompt_class, version_name)
+
+      raise CannotDemotePrimaryError,
+            "Cannot demote #{version_name.inspect} — primary version with no viable " \
+            "candidate. Promote another version first."
+    end
+
+    def effective_primary_name(prompt_class)
+      if ar_storage? && defined?(PromptCanary::PrimaryOverride)
+        override = PrimaryOverride.find_by(prompt: prompt_class.name)
+        return override.version if override
+      end
+      prompt_class.primary_version.name
+    rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+      prompt_class.primary_version.name
+    end
+
+    def no_viable_candidate?(prompt_class, version_name)
+      others = prompt_class.versions.reject { |v| v.name == version_name }
+      return true if others.empty?
+
+      if ar_storage?
+        require "prompt_canary/storage/active_record_adapter"
+        others.all? do |v|
+          RolloutOverride.where(prompt: prompt_class.name, version: v.name, rollout_override: 0).exists?
+        end
+      else
+        others.all?(&:demoted?)
+      end
+    end
+
+    def ar_storage?
+      defined?(configuration) && configuration.storage == :active_record
+    end
+
+    def effective_canary_percent(prompt_class, version_name)
+      existing = RolloutOverride.find_by(
+        "prompt = ? AND version = ? AND rollout_override > 0",
+        prompt_class.name, version_name
+      )
+      return existing.rollout_override if existing
+
+      version = prompt_class.versions.find { |v| v.name == version_name }
+      version&.rollout&.fetch(:percent, 0) || 0
+    end
+
+    def record_event(payload)
+      return unless defined?(PromptCanary::PromptEvent)
+
+      PromptEvent.create!(
+        prompt: payload[:prompt],
+        version: payload[:version],
+        event: payload[:event],
+        previous_percent: payload[:previous_percent],
+        new_percent: payload[:new_percent],
+        previous_status: payload[:previous_status],
+        new_status: payload[:new_status],
+        reason: payload[:reason],
+        triggered_by: payload.fetch(:triggered_by, "manual"),
+        triggering_metric: payload[:triggering_metric],
+        triggering_value: payload[:triggering_value],
+        triggering_threshold: payload[:triggering_threshold],
+        recorded_at: Time.now
+      )
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/lib/prompt_canary/engine.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Engine < Rails::Engine
+    isolate_namespace PromptCanary
+  end
+end

data/lib/prompt_canary/monitor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Monitor
+    def initialize(recorder:)
+      @recorder = recorder
+    end
+
+    def evaluate(prompt_class)
+      prompt_class.versions.each do |version|
+        version.rollback_rules.each do |rule|
+          value = @recorder.public_send(rule.metric,
+                                        prompt: prompt_class.name,
+                                        version: version.name,
+                                        over: rule.window)
+
+          next unless rule.violated_by?(value)
+
+          PromptCanary.demote(
+            prompt_class, version.name,
+            triggered_by: "monitor",
+            triggering_metric: rule.metric.to_s,
+            triggering_value: value,
+            triggering_threshold: rule.threshold
+          )
+        end
+      end
+    end
+  end
+end

data/lib/prompt_canary/monitor_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class MonitorJob < ActiveJob::Base
+    queue_as :default
+
+    def perform
+      recorder = Recorder.new(storage: StorageFactory.build(PromptCanary.configuration.storage))
+      monitor  = Monitor.new(recorder: recorder)
+      PromptCanary.registered_prompts.each { |klass| monitor.evaluate(klass) }
+    end
+  end
+end

data/lib/prompt_canary/prompt.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Prompt
+    include Promptable
+
+    def self.inherited(subclass)
+      warn "[PromptCanary] Inheriting from PromptCanary::Prompt is deprecated. " \
+           "Use `include PromptCanary::Promptable` instead."
+      super
+    end
+  end
+end

data/lib/prompt_canary/prompt_executor.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class PromptExecutor
+    def initialize(prompt_class:, adapter: nil, recorder: nil)
+      @prompt_class = prompt_class
+      @adapter      = adapter  || AdapterFactory.build(PromptCanary.configuration.adapter)
+      @recorder     = recorder || Recorder.new(storage: StorageFactory.build(PromptCanary.configuration.storage))
+    end
+
+    def call(context: {}, **args)
+      version   = Router.choose(@prompt_class, context)
+      telemetry = @adapter.call(version: version, args: args)
+      @recorder.record(prompt: @prompt_class.name, version: version, telemetry: telemetry)
+
+      Result.new(
+        text: telemetry[:text],
+        version_used: version.name,
+        model: version.model,
+        latency_ms: telemetry[:latency_ms],
+        tokens: telemetry[:tokens],
+        error: telemetry[:error],
+        recorded_at: Time.now
+      )
+    end
+  end
+end

data/lib/prompt_canary/promptable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  module Promptable
+    def self.included(base)
+      base.extend(ClassMethods)
+      PromptCanary.register_prompt(base)
+    end
+
+    module ClassMethods
+      def version(name, &block)
+        if versions.any? { |v| v.name == name }
+          raise DuplicateVersionError, "Version #{name.inspect} is already registered on #{self}"
+        end
+
+        builder = VersionBuilder.new(name)
+        builder.instance_eval(&block)
+        versions << builder.build
+      end
+
+      def versions
+        @versions ||= []
+      end
+
+      def primary_version
+        raise NoPrimaryVersionError, "#{self} has no versions registered" if versions.empty?
+
+        if @primary_override
+          versions.find { |v| v.name == @primary_override } || versions.first
+        else
+          versions.first
+        end
+      end
+
+      def promote_to_primary!(version_name)
+        @primary_override = version_name
+      end
+
+      def call(context: {}, adapter: nil, recorder: nil, **args)
+        PromptExecutor.new(prompt_class: self, adapter: adapter, recorder: recorder)
+                      .call(context: context, **args)
+      end
+
+      def reset_registry!
+        @versions = []
+        @primary_override = nil
+      end
+    end
+  end
+end

data/lib/prompt_canary/railtie.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "monitor_job"
+
+module PromptCanary
+  class Railtie < Rails::Railtie
+    initializer "prompt_canary.load_prompt_classes" do |_app|
+      ActiveSupport.on_load(:after_initialize) do
+        PromptCanary.load_prompt_classes(Rails.root.join("app", "prompts").to_s)
+      end
+    end
+
+    initializer "prompt_canary.check_storage_config" do
+      ActiveSupport.on_load(:after_initialize) do
+        PromptCanary.check_storage_config!(Rails.logger)
+      end
+    end
+
+    initializer "prompt_canary.log_registered_prompts" do
+      ActiveSupport.on_load(:after_initialize) do
+        next unless PromptCanary.registered_prompts.any?
+
+        names = PromptCanary.registered_prompts.map(&:name).join(", ")
+        Rails.logger.info("[PromptCanary] Registered prompts: #{names}")
+      end
+    end
+  end
+end

data/lib/prompt_canary/recorder.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Recorder
+    attr_reader :storage
+
+    def initialize(storage:)
+      @storage = storage
+    end
+
+    def latency_p95(prompt:, version:, over:)
+      records = @storage.read_recent(prompt: prompt, version: version, limit: over)
+      return 0 if records.empty?
+
+      latencies = records.map { |r| r[:latency_ms] }.sort
+      index = (latencies.length * 0.95).ceil - 1
+      latencies[index]
+    end
+
+    def error_rate(prompt:, version:, over:)
+      records = @storage.read_recent(prompt: prompt, version: version, limit: over)
+      return 0.0 if records.empty?
+
+      errored = records.count { |r| !r[:error].nil? }
+      (errored.to_f / records.length).round(2)
+    end
+
+    def stats(prompt:, version:, over:)
+      records = @storage.read_recent(prompt: prompt, version: version, limit: over)
+      return { call_count: 0, error_rate: 0.0, latency_p95: nil, last_called_at: nil } if records.empty?
+
+      latencies   = records.map { |r| r[:latency_ms] }.compact.sort
+      error_count = records.count { |r| !r[:error].nil? }
+      p95_index   = (latencies.size * 0.95).ceil - 1
+
+      {
+        call_count: records.size,
+        error_rate: (error_count.to_f / records.size).round(2),
+        latency_p95: latencies[p95_index],
+        last_called_at: records.last[:recorded_at]
+      }
+    end
+
+    def record(prompt:, version:, telemetry:)
+      @storage.write(
+        prompt: prompt,
+        version: version.name,
+        latency_ms: telemetry[:latency_ms],
+        tokens: telemetry[:tokens],
+        error: telemetry[:error],
+        recorded_at: Time.now
+      )
+    end
+  end
+end

data/lib/prompt_canary/result.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Result
+    attr_reader :text, :version_used, :model, :latency_ms, :tokens, :error, :recorded_at
+
+    def initialize(text:, version_used:, model:, latency_ms:, tokens:, error:, recorded_at:)
+      @text = text
+      @version_used = version_used
+      @model = model
+      @latency_ms = latency_ms
+      @tokens = tokens
+      @error = error
+      @recorded_at = recorded_at
+      freeze
+    end
+  end
+end

data/lib/prompt_canary/rollback_rule.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class RollbackRule
+    attr_reader :metric, :threshold, :comparator, :window
+
+    def initialize(metric:, threshold:, comparator:, window:)
+      @metric     = metric
+      @threshold  = threshold
+      @comparator = comparator
+      @window     = window
+      freeze
+    end
+
+    def violated_by?(value)
+      case comparator
+      when :greater_than then value > threshold
+      when :less_than    then value < threshold
+      end
+    end
+  end
+end