prompt_warden 0.1.0 → 0.1.1

data/lib/prompt_warden/cli.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+require 'time'
+
+module PromptWarden
+  module CLI
+    class EventFormatter
+      def self.format(event, json: false)
+        if json
+          event.to_h.to_json
+        else
+          format_human_readable(event)
+        end
+      end
+
+      private
+
+      def self.format_human_readable(event)
+        timestamp = event.timestamp ? Time.parse(event.timestamp).strftime('%H:%M:%S') : 'N/A'
+        model = event.model || 'unknown'
+        cost = event.cost_usd ? "$#{event.cost_usd}" : 'N/A'
+        status = event.status || 'unknown'
+
+        # Format alerts
+        alert_text = if event.alerts&.any?
+          alerts = event.alerts.map do |alert|
+            case alert['type']
+            when 'regex'
+              "⚠️ #{alert['rule']}"
+            when 'cost'
+              "💰 >$#{alert['limit']}"
+            else
+              "❓ #{alert['type']}"
+            end
+          end
+          " [#{alerts.join(', ')}]"
+        else
+          ""
+        end
+
+        # Truncate prompt for display
+        prompt_preview = event.prompt ? event.prompt[0..50] : 'N/A'
+        prompt_preview += '...' if event.prompt && event.prompt.length > 50
+
+        "#{timestamp} #{model} #{cost} #{status}#{alert_text} | #{prompt_preview}"
+      end
+    end
+
+    class EventFilter
+      def self.matches?(event, options)
+        return false if options[:alerts] && (!event.alerts || event.alerts.empty?)
+        return false if options[:model] && event.model != options[:model]
+        return false if options[:cost] && (!event.cost_usd || event.cost_usd < options[:cost])
+        return false if options[:status] && event.status != options[:status]
+        true
+      end
+    end
+
+    class CLIBuffer
+      def initialize
+        @events = []
+        @listeners = []
+      end
+
+      def push(event)
+        @events << event
+        @listeners.each { |listener| listener.call(event) }
+      end
+
+      def on_event(&block)
+        @listeners << block
+      end
+
+      def recent_events(limit: 10)
+        @events.last(limit)
+      end
+    end
+
+    class Tail
+      def self.run(args = ARGV)
+        options = parse_options(args)
+
+        # Configure PromptWarden if not already configured
+        configure_prompt_warden
+
+        # Run the tail command
+        new(options).run
+      end
+
+      def initialize(options)
+        @options = options
+      end
+
+      def run
+        puts "🔍 PromptWarden CLI - Real-time event monitoring"
+        puts "Press Ctrl+C to exit"
+        puts "-" * 80
+
+        # Create a mock buffer for CLI mode
+        @cli_buffer = CLIBuffer.new
+
+        # Override the real buffer temporarily
+        @original_buffer = PromptWarden.send(:buffer)
+        PromptWarden.define_singleton_method(:buffer) { @cli_buffer }
+
+        # Set up event listener
+        @event_count = 0
+        @cli_buffer.on_event do |event|
+          next unless EventFilter.matches?(event, @options)
+
+          @event_count += 1
+          puts EventFormatter.format(event, json: @options[:json])
+
+          exit 0 if @options[:limit] && @event_count >= @options[:limit]
+        end
+
+        # Show recent events if not following
+        unless @options[:follow]
+          show_recent_events
+          return
+        end
+
+        # Follow mode - wait for events
+        if @options[:follow]
+          puts "Waiting for events... (use --no-follow to see recent events only)"
+
+          # Keep the process alive
+          loop do
+            sleep 1
+          end
+        end
+
+      rescue Interrupt
+        puts "\n👋 Goodbye!"
+      rescue => e
+        puts "❌ Error: #{e.message}"
+        puts e.backtrace if ENV['DEBUG']
+        exit 1
+      ensure
+        # Restore original buffer
+        PromptWarden.define_singleton_method(:buffer) { @original_buffer }
+      end
+
+      private
+
+      def show_recent_events
+        recent_events = @cli_buffer.recent_events(limit: @options[:limit] || 10)
+        recent_events.each do |event|
+          next unless EventFilter.matches?(event, @options)
+          puts EventFormatter.format(event, json: @options[:json])
+        end
+      end
+
+      def self.parse_options(args)
+        options = {
+          alerts: false,
+          model: nil,
+          cost: nil,
+          status: nil,
+          limit: nil,
+          json: false,
+          follow: true
+        }
+
+        OptionParser.new do |opts|
+          opts.banner = "Usage: pw_tail [options]"
+
+          opts.on('--alerts', 'Show only events with alerts') { options[:alerts] = true }
+          opts.on('--model MODEL', 'Filter by model') { |m| options[:model] = m }
+          opts.on('--cost COST', Float, 'Show events above cost (USD)') { |c| options[:cost] = c }
+          opts.on('--status STATUS', 'Filter by status (ok, failed)') { |s| options[:status] = s }
+          opts.on('--limit N', Integer, 'Limit number of events to show') { |l| options[:limit] = l }
+          opts.on('--json', 'Output in JSON format') { options[:json] = true }
+          opts.on('--no-follow', 'Don\'t follow new events') { options[:follow] = false }
+          opts.on('-h', '--help', 'Show this help message') do
+            puts opts
+            exit
+          end
+        end.parse!(args)
+
+        options
+      end
+
+      def self.configure_prompt_warden
+        begin
+          PromptWarden.configuration
+        rescue
+          # Set minimal configuration for CLI
+          PromptWarden.configure do |config|
+            config.project_token = 'cli_mode'
+            config.api_url = 'http://localhost:3000'  # Won't actually upload in CLI mode
+          end
+        end
+      end
+    end
+  end
+end

data/lib/prompt_warden/configuration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module PromptWarden
+  class Configuration
+    attr_accessor :project_token, :flush_interval, :batch_bytes,
+                  :max_retries,   :logger, :api_url,
+                  :price_overrides
+
+    def initialize
+      @project_token  = ENV['PROMPT_WARDEN_TOKEN']
+      @flush_interval = 1.0
+      @batch_bytes    = 256 * 1024
+      @max_retries    = 3
+      @logger         = defined?(Rails) ? Rails.logger : Logger.new($stdout)
+      @api_url        = ENV['PROMPT_WARDEN_API'] || 'https://staging.promptwarden.dev/api/v1/ingest'
+      @price_overrides = {}   # e.g., { "gpt-4o" => 0.005 }
+      @adapter_blocks  = {}   # lazily‑executed registration procs
+    end
+
+    # --- validation ------------------------------------------------------
+    def validate!
+      raise ArgumentError, 'project_token is required' if @project_token.to_s.empty?
+    end
+
+    # --- adapter registration -------------------------------------------
+    # Called by host app for custom SDKs
+    def register_adapter(key, &block)
+      @adapter_blocks[key] = block
+      block.call if Gem.loaded_specs.key?(key.to_s) # execute immediately if gem already loaded
+    end
+
+    # Invoked by internal loader after all built‑in auto‑detects
+    def run_pending_adapters
+      @adapter_blocks.each_value(&:call)
+    end
+  end
+end

data/lib/prompt_warden/cost_calculator.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module PromptWarden
+  class CostCalculator
+    # Model pricing (per 1K tokens) - updated as of 2024
+    MODEL_PRICING = {
+      # OpenAI Models
+      'gpt-4o' => { input: 0.0025, output: 0.01 },
+      'gpt-4o-mini' => { input: 0.00015, output: 0.0006 },
+      'gpt-4-turbo' => { input: 0.01, output: 0.03 },
+      'gpt-4' => { input: 0.03, output: 0.06 },
+      'gpt-3.5-turbo' => { input: 0.0005, output: 0.0015 },
+
+      # Anthropic Models
+      'claude-3-opus-20240229' => { input: 0.015, output: 0.075 },
+      'claude-3-sonnet-20240229' => { input: 0.003, output: 0.015 },
+      'claude-3-haiku-20240307' => { input: 0.00025, output: 0.00125 },
+      'claude-3-5-sonnet-20241022' => { input: 0.003, output: 0.015 },
+
+      # Default fallback
+      'default' => { input: 0.001, output: 0.002 }
+    }.freeze
+
+    class << self
+      def calculate_cost(prompt:, model:, response_tokens: nil)
+        input_tokens = count_tokens(prompt, model)
+        output_tokens = response_tokens || estimate_output_tokens(prompt, model)
+
+        pricing = get_model_pricing(model)
+        input_cost = (input_tokens / 1000.0) * pricing[:input]
+        output_cost = (output_tokens / 1000.0) * pricing[:output]
+
+        (input_cost + output_cost).round(6)
+      end
+
+      def count_tokens(text, model = nil)
+        return 0 if text.nil? || text.empty?
+
+        # Use tiktoken for OpenAI models if available
+        if model&.start_with?('gpt-') && defined?(Tiktoken)
+          count_tokens_tiktoken(text, model)
+        else
+          # Fallback to approximate counting
+          count_tokens_approximate(text)
+        end
+      end
+
+      def estimate_output_tokens(prompt, model)
+        return 0 if prompt.nil? || prompt.empty?
+
+        # Rough estimation based on prompt length and model
+        base_tokens = prompt.length / 4  # ~4 chars per token
+
+        # Adjust based on model type
+        case model
+        when /gpt-4o/
+          (base_tokens * 0.8).round  # More efficient
+        when /claude-3-opus/
+          (base_tokens * 1.2).round  # More verbose
+        when /claude-3-sonnet/
+          (base_tokens * 1.0).round  # Standard
+        when /claude-3-haiku/
+          (base_tokens * 0.7).round  # Concise
+        else
+          base_tokens.round
+        end
+      end
+
+      def get_model_pricing(model)
+        return MODEL_PRICING['default'] unless model
+
+        # Try exact match first
+        return MODEL_PRICING[model] if MODEL_PRICING[model]
+
+        # Try partial matches for model variants
+        MODEL_PRICING.each do |key, pricing|
+          next if key == 'default'
+          return pricing if model.include?(key.split('-').first) || key.include?(model.split('-').first)
+        end
+
+        MODEL_PRICING['default']
+      end
+
+      private
+
+      def count_tokens_tiktoken(text, model)
+        # Use tiktoken gem for accurate OpenAI token counting
+        encoding = Tiktoken.encoding_for_model(model)
+        encoding.encode(text).length
+      rescue => e
+        # Fallback if tiktoken fails
+        PromptWarden.configuration.logger.warn("Tiktoken failed for #{model}: #{e.message}")
+        count_tokens_approximate(text)
+      end
+
+      def count_tokens_approximate(text)
+        # Approximate token counting (roughly 4 characters per token)
+        # This is a simplified approximation - tiktoken is more accurate
+        (text.length / 4.0).round
+      end
+    end
+  end
+end

data/lib/prompt_warden/event.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module PromptWarden
+  Event = Struct.new(
+    :id, :prompt, :response, :model,
+    :latency_ms, :cost_usd, :status, :rule, :timestamp, :alerts,
+    keyword_init: true
+  ) do
+    def to_h
+      to_h = super
+      to_h[:timestamp] ||= Time.now.utc.iso8601
+      to_h[:alerts] ||= []
+      to_h
+    end
+  end
+end

data/lib/prompt_warden/instrumentation/anthropic.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module PromptWarden
+  module Instrumentation
+    module Anthropic
+      def messages(params = {}, opts = {})
+        _pw_wrap(:messages, params) { super }
+      end
+
+      def complete(params = {}, opts = {})
+        _pw_wrap(:complete, params) { super }
+      end
+
+      private
+
+      def _pw_wrap(_method_sym, params)
+        cfg    = PromptWarden.configuration
+        prompt = _extract_prompt(params)
+        model  = params[:model] || params['model']
+
+        # Use enhanced cost calculation
+        cost_estimate = PromptWarden.calculate_cost(prompt: prompt, model: model)
+
+        # Check for policy alerts before making the request
+        alerts = PromptWarden::Policy.instance.check_alerts(prompt: prompt, cost_estimate: cost_estimate)
+
+        start  = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        resp   = yield
+        latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
+
+        # Calculate actual cost with response tokens if available
+        response_content = _extract_response(resp)
+        response_tokens = resp.dig('usage', 'output_tokens') || resp.dig(:usage, :output_tokens)
+        actual_cost = PromptWarden.calculate_cost(
+          prompt: prompt,
+          model: model,
+          response_tokens: response_tokens
+        )
+
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          prompt: prompt,
+          response: response_content,
+          model: model,
+          latency_ms: latency,
+          cost_usd: actual_cost,
+          status: 'ok',
+          alerts: alerts
+        )
+        resp
+      rescue StandardError
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          model: model,
+          prompt: prompt,
+          status: 'failed',
+          alerts: alerts
+        )
+        raise
+      end
+
+      def _extract_prompt(params)
+        if params[:messages]
+          params[:messages].map { |m| m[:content] || m['content'] }.join("\n")
+        else
+          params[:prompt] || params['prompt'] || ''
+        end
+      end
+
+      def _extract_response(resp)
+        if resp[:content]
+          resp[:content].map { |c| c[:text] || c['text'] }.join("\n")
+        elsif resp['content']
+          resp['content'].map { |c| c[:text] || c['text'] }.join("\n")
+        else
+          resp['completion'] || resp[:completion] || ''
+        end
+      end
+    end
+  end
+end
+
+Anthropic::Client.prepend(PromptWarden::Instrumentation::Anthropic) if defined?(Anthropic::Client)

data/lib/prompt_warden/instrumentation/langchain.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module PromptWarden
+  module Instrumentation
+    module Langchain
+      def chat(prompt, **opts)
+        _pw_wrap(:chat, { prompt: prompt, **opts }) { super }
+      end
+
+      def complete(prompt, **opts)
+        _pw_wrap(:complete, { prompt: prompt, **opts }) { super }
+      end
+
+      private
+
+      def _pw_wrap(method_sym, params)
+        cfg    = PromptWarden.configuration
+        prompt = params[:prompt] || params['prompt']
+        model  = _extract_model(params)
+
+        # Use enhanced cost calculation
+        cost_estimate = PromptWarden.calculate_cost(prompt: prompt, model: model)
+
+        # Check for policy alerts before making the request
+        alerts = PromptWarden::Policy.instance.check_alerts(prompt: prompt, cost_estimate: cost_estimate)
+
+        start  = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        resp   = yield
+        latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
+
+        # Calculate actual cost (Langchain doesn't provide token counts)
+        actual_cost = PromptWarden.calculate_cost(prompt: prompt, model: model)
+
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          prompt: prompt,
+          response: resp.is_a?(String) ? resp : resp.to_s,
+          model: model,
+          latency_ms: latency,
+          cost_usd: actual_cost,
+          status: 'ok',
+          alerts: alerts
+        )
+        resp
+      rescue StandardError
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          model: model,
+          prompt: prompt,
+          status: 'failed',
+          alerts: alerts
+        )
+        raise
+      end
+
+      def _extract_model(params)
+        # Try to extract model from various sources
+        model = params[:model] ||
+                params['model'] ||
+                self.class.name ||
+                name ||
+                'default'
+
+        # Clean up class names to model names
+        model = model.gsub(/^Langchain::LLM::/, '').downcase if model.include?('Langchain::LLM::')
+        model
+      end
+    end
+  end
+end
+
+if defined?(Langchain::LLM::Base)
+  Langchain::LLM::Base.prepend(PromptWarden::Instrumentation::Langchain)
+end

data/lib/prompt_warden/instrumentation/openai.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'time'
+
+module PromptWarden
+  module Instrumentation
+    module OpenAI
+      # --- wrappers ------------------------------------------------------
+      def chat(params = {}, opts = {})
+        _pw_wrap(:chat, params) { super }
+      end
+
+      def completions(params = {}, opts = {})
+        _pw_wrap(:completions, params) { super }
+      end
+
+      private
+
+      def _pw_wrap(_method_sym, params)
+        cfg    = PromptWarden.configuration
+        prompt = _extract_prompt(params)
+        model  = params[:model]
+
+        # Use enhanced cost calculation
+        cost_estimate = PromptWarden.calculate_cost(prompt: prompt, model: model)
+
+        # Check for policy alerts before making the request
+        alerts = PromptWarden::Policy.instance.check_alerts(prompt: prompt, cost_estimate: cost_estimate)
+
+        start  = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        resp   = yield
+        latency = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000).round
+
+        # Calculate actual cost with response tokens if available
+        response_content = resp.dig('choices', 0, 'message', 'content')
+        response_tokens = resp.dig('usage', 'completion_tokens')
+        actual_cost = PromptWarden.calculate_cost(
+          prompt: prompt,
+          model: model,
+          response_tokens: response_tokens
+        )
+
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          prompt: prompt,
+          response: response_content,
+          model: model,
+          latency_ms: latency,
+          cost_usd: actual_cost,
+          status: 'ok',
+          alerts: alerts
+        )
+        resp
+      rescue StandardError
+        PromptWarden.record(
+          id: SecureRandom.uuid,
+          model: params[:model],
+          prompt: prompt,
+          status: 'failed',
+          alerts: alerts
+        )
+        raise
+      end
+
+      def _extract_prompt(params)
+        if params[:messages]
+          params[:messages].map { |m| m[:content] }.join("\n")
+        else
+          params[:prompt].to_s
+        end
+      end
+    end
+  end
+end
+
+if defined?(OpenAI::Client)
+  OpenAI::Client.prepend(PromptWarden::Instrumentation::OpenAI)
+end

data/lib/prompt_warden/policy.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'singleton'
+
+module PromptWarden
+  class PolicyError < StandardError; end
+
+  class Policy
+    include Singleton
+
+    def initialize
+      path = ENV['PROMPT_WARDEN_POLICY'] || 'config/promptwarden.yml'
+      @rules = File.exist?(path) ? YAML.load_file(path) : {}
+      @rules = @rules.transform_keys(&:to_s)
+    end
+
+    def reload!
+      path = ENV['PROMPT_WARDEN_POLICY'] || 'config/promptwarden.yml'
+      @rules = File.exist?(path) ? YAML.load_file(path) : {}
+      @rules = @rules.transform_keys(&:to_s)
+    end
+
+    def parse_regex(str)
+      # Handles strings like "/pattern/i" or "/pattern/"
+      if str.is_a?(String) && str =~ %r{^/(.*?)/([imx]*)$}
+        pattern = ::Regexp.last_match(1)
+        flags = ::Regexp.last_match(2)
+        options = 0
+        options |= Regexp::IGNORECASE if flags.include?('i')
+        options |= Regexp::MULTILINE  if flags.include?('m')
+        options |= Regexp::EXTENDED   if flags.include?('x')
+        Regexp.new(pattern, options)
+      else
+        Regexp.new(str.to_s)
+      end
+    end
+
+    def check_alerts(prompt:, cost_estimate:)
+      alerts = []
+
+      # Check warn_if_regex patterns
+      Array(@rules['warn_if_regex']).each do |regex|
+        r = parse_regex(regex)
+        if prompt.match?(r)
+          alerts << { type: 'regex', rule: regex, level: 'warn' }
+        end
+      end
+
+      # Check cost limits
+      max_cost = @rules['max_cost_usd'].to_f
+      if max_cost.positive? && cost_estimate > max_cost
+        alerts << { type: 'cost', limit: max_cost, level: 'block' }
+      end
+
+      alerts
+    end
+
+    def check!(prompt:, cost_estimate:)
+      max_cost = @rules['max_cost_usd'].to_f
+      if max_cost.positive? && cost_estimate > max_cost
+        raise PolicyError, "Cost #{cost_estimate}$ exceeds max_cost_usd #{max_cost}"
+      end
+
+      Array(@rules['reject_if_regex']).each do |regex|
+        r = parse_regex(regex)
+        raise PolicyError, "Prompt matches reject regex #{regex}" if prompt.match?(r)
+      end
+
+      :ok
+    end
+  end
+end

data/lib/prompt_warden/railtie.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'rails/railtie'
+
+module PromptWarden
+  class Railtie < Rails::Railtie
+    initializer 'prompt_warden.configure' do
+      # Create `config/initializers/prompt_warden.rb` when user runs generator
+    end
+
+    rake_tasks do
+      # Add future rake tasks here
+    end
+  end
+end