prompt_canary 0.3.0

data/lib/prompt_canary/router.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Router
+    def self.choose(prompt_class, context)
+      primary = db_primary(prompt_class) || prompt_class.primary_version
+      partial = prompt_class.versions.find { |v| v.partial_rollout? || v.predicate? }
+      return primary unless partial
+      return primary if demoted?(prompt_class.name, partial.name)
+
+      route_partial(partial, primary, context, prompt_class.name)
+    end
+
+    def self.route_partial(partial, primary, context, prompt_name)
+      return partial if partial.matches_predicate?(context)
+
+      call_id = context[:call_id]
+      return primary unless call_id
+
+      percent = canary_percent(prompt_name, partial.name, partial.rollout.fetch(:percent, 0))
+      Zlib.crc32(call_id.to_s) % 100 < percent ? partial : primary
+    end
+    private_class_method :route_partial
+
+    def self.canary_percent(prompt_name, version_name, default)
+      return default unless defined?(PromptCanary::RolloutOverride)
+
+      override = PromptCanary::RolloutOverride
+                 .where(prompt: prompt_name, version: version_name)
+                 .where("rollout_override > 0")
+                 .first
+      override ? override.rollout_override : default
+    rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+      default
+    end
+    private_class_method :canary_percent
+
+    def self.db_primary(prompt_class)
+      return nil unless defined?(PromptCanary::PrimaryOverride)
+
+      override = PromptCanary::PrimaryOverride.find_by(prompt: prompt_class.name)
+      return nil unless override
+
+      prompt_class.versions.find { |v| v.name == override.version }
+    rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+      nil
+    end
+    private_class_method :db_primary
+
+    def self.demoted?(prompt_name, version_name)
+      return false unless defined?(PromptCanary::RolloutOverride)
+
+      PromptCanary::RolloutOverride
+        .where(prompt: prompt_name, version: version_name, rollout_override: 0)
+        .exists?
+    rescue ::ActiveRecord::ConnectionNotEstablished, ::ActiveRecord::StatementInvalid
+      false
+    end
+    private_class_method :demoted?
+  end
+end

data/lib/prompt_canary/storage/active_record_adapter.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "json"
+
+module PromptCanary
+  class Call < ::ActiveRecord::Base
+    self.table_name = "prompt_canary_calls"
+  end
+
+  class RolloutOverride < ::ActiveRecord::Base
+    self.table_name = "prompt_canary_rollout_overrides"
+  end
+
+  class PrimaryOverride < ::ActiveRecord::Base
+    self.table_name = "prompt_canary_primary_overrides"
+  end
+
+  class PromptEvent < ::ActiveRecord::Base
+    self.table_name = "prompt_canary_events"
+  end
+
+  module Storage
+    class ActiveRecord
+      def write(record)
+        Call.create!(
+          prompt: record[:prompt],
+          version: record[:version],
+          latency_ms: record[:latency_ms],
+          tokens: record[:tokens]&.to_json,
+          error: record[:error]&.message,
+          recorded_at: record[:recorded_at]
+        )
+      end
+
+      def read_recent(prompt:, version:, limit:)
+        Call.where(prompt: prompt, version: version)
+            .order(recorded_at: :desc)
+            .limit(limit)
+            .reverse
+            .map { |row| deserialize(row) }
+      end
+
+      private
+
+      def deserialize(row)
+        {
+          prompt: row.prompt,
+          version: row.version,
+          latency_ms: row.latency_ms,
+          tokens: row.tokens ? JSON.parse(row.tokens, symbolize_names: true) : nil,
+          error: row.error ? StandardError.new(row.error) : nil,
+          recorded_at: row.recorded_at
+        }
+      end
+    end
+  end
+end

data/lib/prompt_canary/storage/memory.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  module Storage
+    class Memory
+      def initialize
+        @records = []
+      end
+
+      def write(record)
+        @records << record
+      end
+
+      def read_recent(prompt:, version:, limit:)
+        @records
+          .select { |r| r[:prompt] == prompt && r[:version] == version }
+          .last(limit)
+      end
+    end
+  end
+end

data/lib/prompt_canary/storage/sqlite.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+
+module PromptCanary
+  module Storage
+    class SQLite
+      def initialize(path: "prompt_canary.db")
+        @db = ::SQLite3::Database.new(path)
+        @db.results_as_hash = true
+        create_table
+      end
+
+      def write(record)
+        @db.execute(
+          "INSERT INTO calls (prompt, version, latency_ms, tokens, error, recorded_at) VALUES (?, ?, ?, ?, ?, ?)",
+          [
+            record[:prompt],
+            record[:version],
+            record[:latency_ms],
+            JSON.dump(record[:tokens]),
+            record[:error]&.message,
+            record[:recorded_at].iso8601
+          ]
+        )
+      end
+
+      def read_recent(prompt:, version:, limit:)
+        rows = @db.execute(
+          "SELECT * FROM calls WHERE prompt = ? AND version = ? ORDER BY recorded_at DESC LIMIT ?",
+          [prompt, version, limit]
+        )
+
+        rows.reverse.map do |row|
+          {
+            prompt: row["prompt"],
+            version: row["version"],
+            latency_ms: row["latency_ms"],
+            tokens: row["tokens"] ? JSON.parse(row["tokens"], symbolize_names: true) : nil,
+            error: row["error"] ? StandardError.new(row["error"]) : nil,
+            recorded_at: Time.parse(row["recorded_at"])
+          }
+        end
+      end
+
+      private
+
+      def create_table
+        @db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS calls (
+            id          INTEGER PRIMARY KEY AUTOINCREMENT,
+            prompt      TEXT    NOT NULL,
+            version     TEXT    NOT NULL,
+            latency_ms  INTEGER,
+            tokens      TEXT,
+            error       TEXT,
+            recorded_at TEXT    NOT NULL
+          )
+        SQL
+      end
+    end
+  end
+end

data/lib/prompt_canary/storage_factory.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class StorageFactory
+    REGISTRY = {
+      memory: -> { Storage::Memory.new },
+      sqlite: lambda {
+        require "prompt_canary/storage/sqlite"
+        Storage::SQLite.new
+      },
+      active_record: lambda {
+        require "prompt_canary/storage/active_record_adapter"
+        Storage::ActiveRecord.new
+      }
+    }.freeze
+
+    def self.build(storage_name)
+      builder = REGISTRY[storage_name]
+      raise ConfigurationError, "Unknown storage: #{storage_name.inspect}" unless builder
+
+      builder.call
+    end
+  end
+end

data/lib/prompt_canary/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  VERSION = "0.3.0"
+end

data/lib/prompt_canary/version_builder.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class VersionBuilder
+    attr_reader :_name, :_model, :_system, :_rollout
+
+    def initialize(name)
+      @_name = name
+      @_rollout = {}
+      @_rollback_rules = []
+    end
+
+    def rollout(value)
+      @_rollout = value
+    end
+
+    def rollout_to(&block)
+      @_predicate = block
+    end
+
+    def rollback_if(metric, over:, greater_than: nil, less_than: nil)
+      comparator = if greater_than
+                     :greater_than
+                   elsif less_than
+                     :less_than
+                   else
+                     raise ArgumentError, "rollback_if requires greater_than: or less_than:"
+                   end
+      threshold = greater_than || less_than
+      @_rollback_rules << RollbackRule.new(metric: metric, threshold: threshold, comparator: comparator, window: over)
+    end
+
+    def model(value)
+      @_model = value
+    end
+
+    def system(value = nil, &block)
+      @_system = block || value
+    end
+
+    def build
+      Version.new(
+        name: _name,
+        model: _model,
+        system: _system,
+        rollout: _rollout,
+        predicate: @_predicate,
+        rollback_rules: @_rollback_rules
+      )
+    end
+  end
+end

data/lib/prompt_canary/version_object.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module PromptCanary
+  class Version
+    attr_reader :name, :model, :system, :rollout, :rollback_rules
+
+    def initialize(name:, model:, system:, rollout:, predicate: nil, rollback_rules: [])
+      @name = name
+      @model = model
+      @system = system
+      @rollout = rollout
+      @predicate = predicate
+      @rollback_rules = rollback_rules
+    end
+
+    def system_for(args = {})
+      @system.respond_to?(:call) ? @system.call(args) : @system
+    end
+
+    def predicate?
+      !@predicate.nil?
+    end
+
+    def matches_predicate?(context)
+      return false unless @predicate
+
+      @predicate.call(context)
+    rescue StandardError
+      false
+    end
+
+    def partial_rollout?
+      rollout.fetch(:percent, 0).positive?
+    end
+
+    def routes?(key)
+      roll = Zlib.crc32(key.to_s) % 100
+      roll < rollout.fetch(:percent, 0)
+    end
+
+    def set_rollout!(percent)
+      @rollout = { percent: percent }
+    end
+
+    def demoted?
+      @demoted || false
+    end
+
+    def demote!
+      @demoted = true
+      @previous_rollout = @rollout.fetch(:percent, 0)
+      @rollout = { percent: 0 }
+    end
+
+    def restore!
+      @demoted = false
+      @rollout = { percent: @previous_rollout || 0 }
+      @previous_rollout = nil
+    end
+  end
+end

data/lib/prompt_canary.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module PromptCanary
+  class Error < StandardError; end
+  class ConfigurationError < Error; end
+  class DuplicateVersionError < Error; end
+  class NoPrimaryVersionError < Error; end
+  class UnknownVersionError < Error; end
+  class CannotDemotePrimaryError < Error; end
+  class DemotedVersionError < Error; end
+end
+
+require_relative "prompt_canary/deployment"
+
+module PromptCanary
+  class << self
+    include Deployment
+
+    def configure
+      yield configuration
+      configuration.validate!
+    end
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def register_prompt(klass)
+      registered_prompts << klass
+    end
+
+    def registered_prompts
+      @registered_prompts ||= []
+    end
+
+    def reset_configuration!
+      @configuration = nil
+      @registered_prompts = nil
+    end
+
+    def load_prompt_classes(path, loader: method(:require))
+      return unless File.directory?(path)
+
+      Dir[File.join(path, "**", "*.rb")].sort.each { |f| loader.call(f) }
+    end
+
+    def check_storage_config!(logger)
+      return unless configuration.storage == :sqlite
+
+      logger.warn(
+        "[PromptCanary] storage: :sqlite is not recommended for multi-process Rails deployments. " \
+        "Run `rails generate prompt_canary:install && rails db:migrate` " \
+        "and set `storage: :active_record`."
+      )
+    end
+
+    def stats(prompt_class, version_name, over: 100)
+      recorder = Recorder.new(storage: StorageFactory.build(configuration.storage))
+      recorder.stats(prompt: prompt_class.name, version: version_name, over: over)
+    end
+
+    def subscribe(event, &block)
+      subscribers[event] << block
+    end
+
+    def reset_subscribers!
+      @subscribers = nil
+    end
+
+    private
+
+    def publish(event, payload = {})
+      subscribers[event].each { |sub| sub.call(payload) }
+    end
+
+    def subscribers
+      @subscribers ||= Hash.new { |h, k| h[k] = [] }
+    end
+  end
+end
+
+require_relative "prompt_canary/version"
+require_relative "prompt_canary/configuration"
+require_relative "prompt_canary/rollback_rule"
+require_relative "prompt_canary/result"
+require_relative "prompt_canary/version_object"
+require_relative "prompt_canary/version_builder"
+require_relative "prompt_canary/promptable"
+require_relative "prompt_canary/prompt"
+require_relative "prompt_canary/router"
+require_relative "prompt_canary/adapters/base"
+require_relative "prompt_canary/adapters/anthropic"
+require_relative "prompt_canary/adapter_factory"
+require_relative "prompt_canary/storage/memory"
+require_relative "prompt_canary/storage_factory"
+require_relative "prompt_canary/prompt_executor"
+require_relative "prompt_canary/recorder"
+require_relative "prompt_canary/monitor"
+require_relative "prompt_canary/cli"
+require_relative "prompt_canary/railtie" if defined?(Rails::Railtie)
+require_relative "prompt_canary/engine" if defined?(Rails::Engine)

data/sig/prompt_canary.rbs

@@ -0,0 +1,4 @@
+module PromptCanary
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: prompt_canary
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Rockwell Windsor Rice
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-06-04 00:00:00.000000000 Z
+dependencies: []
+description: Declare prompts as Ruby classes with versioned configurations, route
+  traffic by percentage or predicate, record telemetry, and automatically roll back
+  misbehaving versions.
+email:
+- rockwellwindsor@gmail.com
+executables:
+- prompt_canary
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/controllers/prompt_canary/application_controller.rb
+- app/controllers/prompt_canary/dashboard/prompts_controller.rb
+- app/views/layouts/prompt_canary/application.html.erb
+- app/views/prompt_canary/dashboard/prompts/index.html.erb
+- app/views/prompt_canary/dashboard/prompts/show.html.erb
+- config/routes.rb
+- examples/auto_rollback.rb
+- examples/demo.rb
+- exe/prompt_canary
+- lib/generators/prompt_canary/install_generator.rb
+- lib/generators/prompt_canary/templates/create_prompt_canary_calls.rb
+- lib/prompt_canary.rb
+- lib/prompt_canary/adapter_factory.rb
+- lib/prompt_canary/adapters/anthropic.rb
+- lib/prompt_canary/adapters/base.rb
+- lib/prompt_canary/cli.rb
+- lib/prompt_canary/cli/commands/history.rb
+- lib/prompt_canary/cli/commands/status.rb
+- lib/prompt_canary/configuration.rb
+- lib/prompt_canary/deployment.rb
+- lib/prompt_canary/engine.rb
+- lib/prompt_canary/monitor.rb
+- lib/prompt_canary/monitor_job.rb
+- lib/prompt_canary/prompt.rb
+- lib/prompt_canary/prompt_executor.rb
+- lib/prompt_canary/promptable.rb
+- lib/prompt_canary/railtie.rb
+- lib/prompt_canary/recorder.rb
+- lib/prompt_canary/result.rb
+- lib/prompt_canary/rollback_rule.rb
+- lib/prompt_canary/router.rb
+- lib/prompt_canary/storage/active_record_adapter.rb
+- lib/prompt_canary/storage/memory.rb
+- lib/prompt_canary/storage/sqlite.rb
+- lib/prompt_canary/storage_factory.rb
+- lib/prompt_canary/version.rb
+- lib/prompt_canary/version_builder.rb
+- lib/prompt_canary/version_object.rb
+- sig/prompt_canary.rbs
+homepage: https://github.com/rockwellwindsor/prompt_canary
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/rockwellwindsor/prompt_canary
+  source_code_uri: https://github.com/rockwellwindsor/prompt_canary
+  changelog_uri: https://github.com/rockwellwindsor/prompt_canary/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: Canary deploys and automatic rollback for LLM prompts in Ruby.
+test_files: []