prompt_warden 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e6eca8fc76a1284d35dd490ffc8285ed055fad9c799f3530e5165d085234dda
-  data.tar.gz: 8cebcab3d6b2fbca74c20ef8659a72d3f3f3e3caa5a05faf349c3c00cb1fa702
+  metadata.gz: 664b869ad3c4946de2726eb00ba6530e4eabf8e5f77232275bc3af91bd265199
+  data.tar.gz: b56c3c7e33b6478ea8b9021c348c185e2dd410c7860c89309216a17959c15ecc
 SHA512:
-  metadata.gz: 0a6616bab2c115f45597e175e28af8ed02c18b83ad31587263b369395f52747ced29af3b193fb59f76dc40643b20d6b1bb06eb49be354ef514a646d41edf38ca
-  data.tar.gz: 2ac9bbb2c41a91cb0b3b8b13c37d48696b29b9db4708cdfc1924024d3b710a5acd0e9cf53cd1f7c93b959489ffafa9dbe941e91538c75d756a94e90a5cd40527
+  metadata.gz: 3a8fce56fd182a7cbdf64c88e9895166eb86fbad211d8616d1a034b4b2b81bbafe835cb228553d60773508866948331d42bdc48005304ae05e8697df51c1ac41
+  data.tar.gz: 3ded34fe6807cb4e1d88e295e8551ff664ece1c3736ecdee8c0d299b38d101f074d381ab54becf86ef1f22dc2278a86c2c7a0c292144c0eadade74602c7747db

data/.gitignore

@@ -9,3 +9,6 @@
 
 # rspec failure tracking
 .rspec_status
+*.gem
+/pkg/
+test_console_commands.md

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 ## [Unreleased]
 
-## [0.1.0] - 2025-07-22
+## [0.1.1] - 2025-07-23
 
-- Initial release
+### Added
+- CLI executable (`pw_tail`) for real-time event monitoring
+- Enhanced cost calculation with accurate token counting and model-specific pricing
+- Comprehensive test suite with 106 passing specs
+- Support for major AI models with current pricing (GPT-4o, Claude-3, etc.)
+- Rake task for retrying failed uploads (`rake pw:retry_failed`)
+
+### Features
+- **Cost Calculation**: Accurate token counting with tiktoken integration and fallback
+- **CLI Monitoring**: Real-time event streaming with JSON and human-readable formats
+- **Reliability**: Disk-retry mechanism for failed uploads
+- **Extensibility**: Adapter system for custom SDK integration
+
+### Supported Models
+- OpenAI: gpt-4o, gpt-4o-mini, gpt-4-turbo, gpt-3.5-turbo
+- Anthropic: claude-3-opus, claude-3-sonnet, claude-3-haiku
+- Default fallback pricing for unknown models
+
+## [0.1.0] - 2025-07-23
+
+### Added
+- Automatic SDK instrumentation for OpenAI, Anthropic, and Langchain
+- YAML-based policy configuration with cost limits and regex patterns
+- Policy alert system with non-blocking warnings and blocking rejections
+- Automatic alert recording in events with structured alert data
+- Asynchronous event uploads with gzip compression and disk-retry fallback
+
+### Features
+- **Policy Enforcement**: Cost limits, regex pattern matching, alert generation
+- **Event Capture**: Complete AI interaction data with metadata and alerts
+- **Reliability**: Disk-retry mechanism for failed uploads
+- **Extensibility**: Adapter system for custom SDK integration

data/Gemfile

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in prompt_warden.gemspec
 gemspec
 
-gem "irb"
-gem "rake", "~> 13.0"
+gem 'irb'
+gem 'rake', '~> 13.0'
 
-gem "rspec", "~> 3.0"
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -1,9 +1,11 @@
 PATH
   remote: .
   specs:
-    prompt_warden (0.1.0)
+    prompt_warden (0.1.1)
       activesupport (>= 6.1)
       faraday (~> 2.9)
+      faraday-retry (~> 2.2)
+      yaml
 
 GEM
   remote: https://rubygems.org/
@@ -21,12 +23,17 @@ GEM
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
       uri (>= 0.13.1)
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
     ast (2.4.3)
     base64 (0.3.0)
     benchmark (0.4.1)
     bigdecimal (3.2.2)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
+    crack (1.0.0)
+      bigdecimal
+      rexml
     date (3.4.1)
     diff-lcs (1.6.2)
     drb (2.2.3)
@@ -37,6 +44,9 @@ GEM
       logger
     faraday-net_http (3.4.1)
       net-http (>= 0.5.0)
+    faraday-retry (2.3.2)
+      faraday (~> 2.0)
+    hashdiff (1.2.0)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     io-console (0.8.1)
@@ -62,6 +72,7 @@ GEM
     psych (5.2.6)
       date
       stringio
+    public_suffix (6.0.2)
     racc (1.8.1)
     rainbow (3.1.1)
     rake (13.3.0)
@@ -71,6 +82,7 @@ GEM
     regexp_parser (2.10.0)
     reline (0.6.2)
       io-console (~> 0.5)
+    rexml (3.4.1)
     rspec (3.13.1)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -114,12 +126,18 @@ GEM
     ruby-progressbar (1.13.0)
     securerandom (0.4.1)
     stringio (3.1.7)
+    timecop (0.9.10)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
     uri (1.0.3)
+    webmock (3.25.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    yaml (0.2.1)
 
 PLATFORMS
   arm64-darwin-23
@@ -133,6 +151,8 @@ DEPENDENCIES
   rspec (~> 3.12, ~> 3.0)
   rubocop (~> 1.60)
   rubocop-rspec (~> 2.26)
+  timecop (~> 0.9)
+  webmock (~> 3.20)
 
 BUNDLED WITH
    2.7.0

data/README.md

@@ -1,43 +1,241 @@
 # PromptWarden
 
-TODO: Delete this and the text below, and describe your gem
+**Record, audit, and guard AI prompt usage** with automatic SDK instrumentation, policy enforcement, and real-time monitoring.
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/prompt_warden`. To experiment with that code, run `bin/console` for an interactive prompt.
+## Features
+
+- **Automatic SDK Capture**: Zero-code integration with OpenAI, Anthropic, and Langchain
+- **Policy Guardrails**: YAML-based rules for cost limits, regex patterns, and alerts
+- **Enhanced Cost Calculation**: Accurate token counting and model-specific pricing
+- **Real-time Monitoring**: CLI tool for live event streaming and filtering
+- **Alert System**: Non-blocking warnings and blocking rejections based on patterns
+- **Automatic Alert Recording**: Alerts included in events and uploaded to SaaS
+- **Asynchronous Uploads**: Batched, gzipped events with disk-retry fallback
 
 ## Installation
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+```bash
+gem install prompt_warden
+```
+
+Or add to your Gemfile:
 
-Install the gem and add to the application's Gemfile by executing:
+```ruby
+gem 'prompt_warden'
+```
 
-```bash
-bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+## Quick Start
+
+1. **Configure** (in your app's initializer):
+
+```ruby
+PromptWarden.configure do |config|
+  config.project_token = 'your-project-token'
+  config.api_url = 'https://your-saas.com/api/v1/ingest'
+end
+```
+
+2. **Create Policy** (`config/promptwarden.yml`):
+
+```yaml
+max_cost_usd: 0.50           # Block if projected call cost > $0.50
+reject_if_regex:
+  - /password/i
+  - /(ssn|social\s*security)/i
+warn_if_regex:
+  - /\bETA\b/i
 ```
 
-If bundler is not being used to manage dependencies, install the gem by executing:
+3. **Use AI SDKs** (automatically instrumented):
+
+```ruby
+# OpenAI
+client = OpenAI::Client.new
+response = client.chat(parameters: {
+  model: "gpt-4o",
+  messages: [{ role: "user", content: "What is the ETA?" }]
+})
+
+# Anthropic
+client = Anthropic::Client.new
+response = client.messages(
+  model: "claude-3-opus-20240229",
+  max_tokens: 1000,
+  messages: [{ role: "user", content: "What is the ETA?" }]
+)
+```
+
+## CLI Tool
+
+Monitor events in real-time with the `pw_tail` command:
 
 ```bash
-gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+# Follow all events
+./bin/pw_tail
+
+# Show only events with alerts
+./bin/pw_tail --alerts
+
+# Filter by model
+./bin/pw_tail --model gpt-4o
+
+# Show events above cost threshold
+./bin/pw_tail --cost 0.01
+
+# Filter by status
+./bin/pw_tail --status failed
+
+# Limit number of events
+./bin/pw_tail --limit 10
+
+# Output in JSON format
+./bin/pw_tail --json
+
+# Show recent events without following
+./bin/pw_tail --no-follow
 ```
 
-## Usage
+### CLI Output Format
 
-TODO: Write usage instructions here
+```
+10:30:00 gpt-4o $0.005 ok [⚠️ /ETA/i] | What is the ETA for this project?
+10:31:15 claude-3 $0.75 ok [💰 >$0.5] | How much does this cost?
+10:32:30 gpt-4o $0.001 ok | Simple question without alerts
+```
 
-## Development
+## Policy Features
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+### Cost Limits
+```yaml
+max_cost_usd: 0.50  # Block requests exceeding $0.50
+```
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+### Regex Patterns
+```yaml
+reject_if_regex:     # Block requests matching patterns
+  - /password/i
+  - /(ssn|social\s*security)/i
 
-## Contributing
+warn_if_regex:       # Log warnings for patterns
+  - /\bETA\b/i
+  - /urgent/i
+```
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/prompt_warden. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/prompt_warden/blob/main/CODE_OF_CONDUCT.md).
+### Programmatic Checks
+```ruby
+# Check for alerts (non-blocking)
+alerts = PromptWarden::Policy.instance.check_alerts(
+  prompt: "What is the ETA?",
+  cost_estimate: 0.005
+)
+
+# Check for blocks (raises PolicyError)
+PromptWarden::Policy.instance.check!(
+  prompt: "What is the password?",
+  cost_estimate: 0.001
+)
+```
 
-## License
+## Enhanced Cost Calculation
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+PromptWarden provides accurate cost calculation with:
 
-## Code of Conduct
+- **Model-specific pricing** for OpenAI and Anthropic models
+- **Token counting** with tiktoken integration for OpenAI models
+- **Response token integration** for accurate post-request costs
+- **Fallback estimation** for unknown models
+
+```ruby
+# Calculate cost for a prompt
+cost = PromptWarden.calculate_cost(
+  prompt: "Explain quantum computing",
+  model: "gpt-4o"
+)
+
+# Calculate cost with actual response tokens
+actual_cost = PromptWarden.calculate_cost(
+  prompt: "Explain quantum computing",
+  model: "gpt-4o",
+  response_tokens: 150
+)
+```
+
+### Supported Models
+
+**OpenAI Models:**
+- `gpt-4o` ($0.0025/1K input, $0.01/1K output)
+- `gpt-4o-mini` ($0.00015/1K input, $0.0006/1K output)
+- `gpt-4-turbo` ($0.01/1K input, $0.03/1K output)
+- `gpt-3.5-turbo` ($0.0005/1K input, $0.0015/1K output)
+
+**Anthropic Models:**
+- `claude-3-opus-20240229` ($0.015/1K input, $0.075/1K output)
+- `claude-3-sonnet-20240229` ($0.003/1K input, $0.015/1K output)
+- `claude-3-haiku-20240307` ($0.00025/1K input, $0.00125/1K output)
+
+## Supported SDKs
+
+- **OpenAI**: `openai` gem
+- **Anthropic**: `anthropic` gem
+- **Langchain**: `langchain` gem
+
+## Gem vs SaaS
+
+**PromptWarden Gem** (this repository):
+- Local policy enforcement
+- Event capture and buffering
+- Enhanced cost calculation
+- Asynchronous uploads to SaaS
+- CLI monitoring tool
+- Disk-retry for failed uploads
+
+**PromptWarden SaaS** (separate application):
+- Data storage and retention
+- Analytics dashboards
+- Advanced alerting (Slack, email)
+- User and project management
+- Cost tracking and reporting
+
+## Event Structure
+
+Events are automatically captured and include:
+
+```json
+{
+  "id": "uuid",
+  "prompt": "What is the ETA?",
+  "response": "The ETA is 2 weeks",
+  "model": "gpt-4o",
+  "latency_ms": 1250,
+  "cost_usd": 0.005,
+  "status": "ok",
+  "timestamp": "2024-01-15T10:30:00Z",
+  "alerts": [
+    {
+      "type": "regex",
+      "rule": "/ETA/i",
+      "level": "warn"
+    }
+  ]
+}
+```
+
+## Development
+
+```bash
+# Install dependencies
+bundle install
+
+# Run tests
+bundle exec rspec
+
+# Run CLI tests
+bundle exec rspec spec/cli_spec.rb
+
+# Test cost calculation
+ruby test_cost_calculation.rb
+```
+
+## License
 
-Everyone interacting in the PromptWarden project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/prompt_warden/blob/main/CODE_OF_CONDUCT.md).
+MIT License - see LICENSE file for details.

data/Rakefile

@@ -1,8 +1,25 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task default: :spec
+
+desc 'Retry failed PromptWarden uploads from disk'
+namespace :pw do
+  task :retry_failed do
+    require_relative './lib/prompt_warden'
+
+    # Configure with default values if not already configured
+    unless PromptWarden.configuration.project_token
+      PromptWarden.configure do |c|
+        c.project_token = ENV['PROMPT_WARDEN_TOKEN'] || 'default_token'
+        c.api_url = ENV['PROMPT_WARDEN_API'] || 'https://httpbin.org/post'
+      end
+    end
+
+    PromptWarden::Uploader.retry_failed!
+  end
+end

data/bin/console

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "bundler/setup"
-require "prompt_warden"
+require 'bundler/setup'
+require 'prompt_warden'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/bin/pw_tail

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'prompt_warden'
+require_relative '../lib/prompt_warden/cli'
+
+# Run the CLI
+PromptWarden::CLI::Tail.run

data/examples/policy.yml

@@ -0,0 +1,22 @@
+# PromptWarden Policy Configuration
+# This file configures guard-rails for AI prompt usage
+
+# Cost limits - block if projected call cost exceeds this amount
+max_cost_usd: 0.50
+
+# Reject patterns - block execution if prompt matches any of these regexes
+reject_if_regex:
+  - /password/i
+  - /(ssn|social\s*security)/i
+  - /credit\s*card/i
+  - /api\s*key/i
+
+# Alert patterns - log alerts if prompt matches any of these regexes
+# (does not block execution, but creates alerts in events)
+warn_if_regex:
+  - /\bETA\b/i
+  - /deadline/i
+  - /urgent/i
+  - /asap/i
+  - /confidential/i
+  - /internal\s*use\s*only/i

data/lib/prompt_warden/adapter.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module PromptWarden
+  module Adapter
+    ENTRY = Struct.new(:gem_name, :const_path, :block, :loaded)
+    REGISTRY = []
+
+    # public API ----------------------------------------------------------
+    def self.map(gem_name, const_path, &block)
+      REGISTRY << ENTRY.new(gem_name, const_path, block, false)
+      try_load(gem_name) # run now if gem already active
+    end
+
+    def self.auto_load_all!
+      Gem.loaded_specs.keys.each { |name| try_load(name) }
+      PromptWarden.configuration.run_pending_adapters
+    end
+
+    # internal ------------------------------------------------------------
+    def self.try_load(gem_name)
+      entry = REGISTRY.find { |e| e.gem_name == gem_name }
+      return unless entry && !entry.loaded
+
+      return unless Gem.loaded_specs.key?(gem_name) || const_path_defined?(entry.const_path)
+
+      entry.block.call
+      entry.loaded = true
+    end
+
+    def self.const_path_defined?(path)
+      names = path.split('::')
+      mod   = Object
+      names.each do |name|
+        return false unless mod.const_defined?(name, false)
+
+        mod = mod.const_get(name)
+      end
+      true
+    end
+    private_class_method :const_path_defined?
+  end
+end
+
+# built‑in mappings -------------------------------------------------------
+PromptWarden::Adapter.map('openai', 'OpenAI::Client') do
+  require_relative 'instrumentation/openai'
+rescue Exception
+  raise
+end
+PromptWarden::Adapter.map('anthropic', 'Anthropic::Client') do
+  require_relative 'instrumentation/anthropic'
+rescue Exception
+  raise
+end
+PromptWarden::Adapter.map('langchain', 'Langchain::LLM::Base') do
+  require_relative 'instrumentation/langchain'
+rescue Exception
+  raise
+end

data/lib/prompt_warden/buffer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'zlib'
+
+module PromptWarden
+  class Buffer
+    def initialize(cfg = PromptWarden.configuration)
+      @cfg    = cfg
+      @events = []
+      @bytes  = 0
+      @mutex  = Mutex.new
+      start_timer
+    end
+
+    # -- Public ----------------------------------------------------------
+
+    # Enqueue an Event (or Hash). Flushes automatically when batch_bytes hit.
+    def push(event)
+      json = JSON.generate(event.to_h)
+      should_flush = false
+
+      @mutex.synchronize do
+        @events << json
+        @bytes  += json.bytesize
+        should_flush = @bytes >= @cfg.batch_bytes
+      end
+
+      flush! if should_flush
+    end
+
+    # Manual flush
+    def flush!
+      batch = nil
+
+      @mutex.synchronize do
+        return if @events.empty?
+
+        batch  = @events.join("\n")
+        @events.clear
+        @bytes = 0
+      end
+
+      compressed = Zlib.gzip(batch)
+      Uploader.instance.enqueue(compressed)
+    end
+
+    private
+
+    # Flushes buffer every flush_interval seconds
+    def start_timer
+      Thread.new do
+        loop do
+          sleep @cfg.flush_interval
+          flush!
+        end
+      end.tap { |t| t.name = 'PromptWarden::BufferFlusher' }
+    end
+  end
+end