active_model-validations-ai 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1aa5446b42e458800e2e8a8dcc6c867b4c2cfef578dd51a6e846701a12864b1d
-  data.tar.gz: 00eae2e2a14357ca82b77590ca69d1486ad1a8e31abee028ebcb4961cd7b7dfa
+  metadata.gz: d44eee74a5bd787b4ef3fa979b836d38ab849934a6c475c511a05d21c3d2568b
+  data.tar.gz: da9537a5b8c063d64377f8a9fe6f774903a413833b5201c95e03990e6c762a3e
 SHA512:
-  metadata.gz: 1362deb8b5d7481610d53776755b53ef353848329250b7c050e9caf0701bb89a1ed9e0262985a9d45c83eab847e87da4d59cfe02e207c72056307b4636e57069
-  data.tar.gz: 0f22e4bb56974895367cd70687b52801712b86b46be7abbf256eb4e41f02a8da132e164b602ccf35740a9d23f014e92e7782e12ac274d4b3853da7312f167b0b
+  metadata.gz: 85afbbd6f52c16ed8d61eca71acfaa3f1aa02e3587126e7c1245304f3718e9f4b99572c7c233da1a8d1c3bf9210be61820c5f561b839e5a16d7f820c62a615d5
+  data.tar.gz: b44e93ca478d587b154377bb93ded494c2ab787e681e119b6065eed2e1980324581c388a729c00ec14c45af2c0dba1faa58522e8855198fd82eb54c412c2a969

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2023 Kasper Timm Hansen
+Copyright (c) 2023 Bullet Train, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -24,17 +24,54 @@ end
 
 The policy and attribute value is sent to OpenAI with our prompt to be validated with GPT. AI generally isn't near zero-cost like other APIs, so this can be expensive to run. Use with some caution.
 
-## Installation
+## Configuration
+
+We're using `ruby-openai` as the OpenAI client under the hood, see the documentation here for how to configure the access token.
+
+https://github.com/alexrudall/ruby-openai#with-config
+
+Note: if you're using [Bullet Train](bullettrain.co), fill in
+`ENV["OPENAI_ACCESS_TOKEN"]` and OpenAI will already be configured for you.
+
+### Setting request timeout
+
+For validations we know we're not waiting for a complicated chat response, so we timeout requests after 3 seconds instead of `ruby-openai`'s 120 second default. If you need to, you can change the timeout with this:
+
+```ruby
+ActiveModel::Validations::AI.request_timeout = 5 # seconds
+```
+
+### Handling errors
+
+We capture any `Faraday::Error` exceptions and report them. If we're in a Rails app we report via `Rails.error` otherwise it's via `ActiveSupport::ErrorReporter.new`. This gives you access to report to your existing error reporting service automatically.
+
+You can also change the error reporter if need be:
 
-TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+```ruby
+ActiveModel::Validations::AI.error_reporter = ActiveSupport::ErrorReporter.new
+```
+
+### Testing
+
+We've tested that our requests to OpenAI work so you don't have to pay that cost in your apps tests.
+
+Add this in your `test/test_helper.rb` or `spec/spec_helper.rb`:
+
+```ruby
+ActiveModel::Validations::AI.stub_with valid: true
+```
+
+Which prevent any requests from firing and assumes that any validation from this gem is valid.
+
+## Installation
 
 Install the gem and add to the application's Gemfile by executing:
 
-    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ bundle add active_model-validations-ai
 
 If bundler is not being used to manage dependencies, install the gem by executing:
 
-    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+    $ gem install active_model-validations-ai
 
 ## Development
 

data/lib/active_model/validations/ai/locale/en.yml

@@ -3,3 +3,4 @@ en:
     # The values :model, :attribute and :value are always available for interpolation
     messages:
       noncompliant: "does not comply with our policy '%{policy}'"
+      errored: "validation encountered an error and has been reported"

data/lib/active_model/validations/ai/version.rb

@@ -3,7 +3,7 @@
 module ActiveModel
   module Validations
     module AI
-      VERSION = "0.1.0"
+      VERSION = "0.2.0"
     end
   end
 end

data/lib/active_model/validations/ai.rb

@@ -9,20 +9,64 @@ require_relative "ai/version"
 module ActiveModel
   module Validations
     module AI
+      singleton_class.attr_accessor :error_reporter
+      @error_reporter = ActiveSupport::ErrorReporter.new
+
+      singleton_class.attr_accessor :request_timeout
+      @request_timeout = 3 # seconds
+
       def self.client
-        @client ||= OpenAI::Client.new
+        @client ||= OpenAI::Client.new(request_timeout: request_timeout)
+      end
+
+      def self.stub_with(valid:)
+        old_client, @client = @client, MockGPT.new(request_timeout: request_timeout, valid: valid)
+        yield if block_given?
+      ensure
+        @client = old_client if block_given?
+      end
+
+      class MockGPT
+        singleton_class.attr_reader :response
+
+        def self.set(**response)
+          old_response, @response = @response, response
+          yield
+        ensure
+          @response = old_response
+        end
+
+        def initialize(request_timeout:, **response)
+          @request_timeout, @response = request_timeout, response
+        end
+        attr_reader :request_timeout
+
+        def chat(parameters:)
+          raise Faraday::Error if request_timeout&.zero?
+          # parameters => { model:, messages: [{content:}] } # TODO: Replace with this pattern matching on Ruby 3.0+.
+          parameters.dig(:model) or raise ArgumentError
+          parameters.dig(:messages, 0, :content) or raise ArgumentError
+          { "choices" => [{ "message" => { "content" => response.to_json } }] }
+        end
+
+        private
+
+        def response
+          self.class.response || @response
+        end
       end
     end
 
     class ComplianceValidator < ::ActiveModel::EachValidator
       def validate_each(record, name, value)
-        record.errors.add(name, :noncompliant, **options) unless valid?(value)
+        AI.error_reporter.handle(Faraday::Error, fallback: -> { record.errors.add(name, :errored, **options) }) do
+          record.errors.add(name, :noncompliant, **options) unless valid?(value)
+        end
       end
 
       private
 
       def valid?(value)
-        # TODO: Figure out how to timeout a request or handle downtime, and invalidate the attribute.
         response = AI.client.chat(
           parameters: {
             model: "gpt-3.5-turbo",
@@ -34,7 +78,7 @@ module ActiveModel
       end
 
       def prompt_for(value)
-        "Validate this input '#{value}' against this policy '#{policy}'. Respod only in JSON formatted like this '{ \"valid\": true }'."
+        "Validate this input '#{value}' against this policy '#{policy}'. Respond only in JSON formatted like this '{ \"valid\": true }'."
       end
 
       def policy
@@ -53,3 +97,11 @@ end
 ActiveSupport.on_load :i18n do
   I18n.load_path << File.expand_path("ai/locale/en.yml", __dir__)
 end
+
+if defined?(Rails::Railtie)
+  class ActiveModel::Validations::AI::Railtie < Rails::Railtie
+    initializer "error_reporter.setup" do
+      ActiveModel::Validations::AI.error_reporter = Rails.error
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_model-validations-ai
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kasper Timm Hansen
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-06 00:00:00.000000000 Z
+date: 2023-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="