ruby_llm-evals 0.0.1 → 0.1.0

data/db/migrate/20251022211230_create_ruby_llm_evals_runs.rb

@@ -0,0 +1,21 @@
+class CreateRubyLLMEvalsRuns < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ruby_llm_evals_runs do |t|
+      t.references :ruby_llm_evals_prompt, null: false, foreign_key: true
+      t.string :active_job_id, null: false
+      t.timestamp :started_at
+      t.timestamp :ended_at
+      t.string :provider, null: false
+      t.string :model, null: false
+      t.float :temperature
+      t.json :params
+      t.json :tools
+      t.string :schema
+      t.json :schema_other
+      t.text :instructions
+      t.text :message
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20251022211231_create_ruby_llm_evals_prompt_executions.rb

@@ -0,0 +1,26 @@
+class CreateRubyLLMEvalsPromptExecutions < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ruby_llm_evals_prompt_executions do |t|
+      t.references :ruby_llm_evals_sample, null: false, foreign_key: true, index: { name: "index_rle_prompt_executions_on_rle_sample_id" }
+      t.references :ruby_llm_evals_run, null: false, foreign_key: true, index: { name: "index_rle_prompt_executions_on_rle_run_id" }
+      t.string :eval_type, null: false
+      t.text :expected_output
+      t.json :variables
+      t.integer :input
+      t.integer :output
+      t.text :message
+      t.boolean :passed
+      t.string :active_job_id, null: true
+      t.timestamp :started_at
+      t.timestamp :ended_at
+      t.text :error_message
+      t.string :judge_provider
+      t.string :judge_model
+      t.json :judge_message
+      t.integer :judge_input
+      t.integer :judge_output
+
+      t.timestamps
+    end
+  end
+end

data/lib/activemodel/validations/json_validator.rb

@@ -0,0 +1,14 @@
+module ActiveModel
+  module Validations
+    class JsonValidator < EachValidator
+      def validate_each(record, attribute, value)
+        return if value.blank?
+        return unless value.is_a? String
+
+        record.send :"#{attribute}=", JSON.parse(value)
+      rescue JSON::ParserError
+        record.errors.add attribute, :invalid_json
+      end
+    end
+  end
+end

data/lib/ruby_llm/evals/engine.rb

@@ -1,6 +1,54 @@
-module RubyLlm
+require "importmap-rails"
+require "stimulus-rails"
+require "turbo-rails"
+
+module RubyLLM
   module Evals
     class Engine < ::Rails::Engine
+      isolate_namespace RubyLLM::Evals
+
+      INFLECTION_OVERRIDES = { "ruby_llm" => "RubyLLM" }.freeze
+
+      initializer "ruby_llm_evals.inflector", after: "ruby_llm.inflections", before: :set_autoload_paths do
+        ActiveSupport::Inflector.inflections(:en) do |inflections|
+          # The RubyLLM gem registers "RubyLLM" as an acronym in its railtie,
+          # which breaks underscore conversion (RubyLLM.underscore => "rubyllm").
+          # We need to remove it and use "LLM" as an acronym instead for proper conversion:
+          # * "ruby_llm".camelize => "RubyLLM" (not "RubyLlm")
+          # * "RubyLLM".underscore => "ruby_llm" (not "rubyllm")
+          inflections.acronyms.delete("rubyllm")
+          inflections.acronym("LLM")
+        end
+
+        Rails.autoloaders.each do |loader|
+          loader.inflector.inflect(INFLECTION_OVERRIDES)
+        end
+      end
+
+      initializer "ruby_llm_evals.assets" do |app|
+        app.config.assets.paths << root.join("app/assets/stylesheets")
+        app.config.assets.paths << root.join("app/assets/images")
+        app.config.assets.paths << root.join("app/javascript")
+
+        app.config.assets.precompile += %w[
+          ruby_llm/evals/application.css
+          ruby_llm/evals/bulma.min.css
+          ruby_llm/evals/application.js
+          ruby_llm/evals/controllers/application.js
+          ruby_llm/evals/controllers/index.js
+          ruby_llm/evals/controllers/provider_model_controller.js
+          ruby_llm/evals/controllers/file_input_controller.js
+        ]
+      end
+
+      initializer "ruby_llm_evals.importmap", after: "importmap" do |app|
+        RubyLLM::Evals.importmap.draw(root.join("config/importmap.rb"))
+        RubyLLM::Evals.importmap.cache_sweeper(watches: root.join("app/javascript"))
+
+        ActiveSupport.on_load(:action_controller_base) do
+          before_action { RubyLLM::Evals.importmap.cache_sweeper.execute_if_updated }
+        end
+      end
     end
   end
 end

data/lib/ruby_llm/evals/version.rb

@@ -1,5 +1,5 @@
-module RubyLlm
+module RubyLLM
   module Evals
-    VERSION = "0.0.1"
+    VERSION = "0.1.0"
   end
 end

data/lib/ruby_llm/evals.rb

@@ -1,8 +1,12 @@
-require "ruby_llm/evals/version"
+require "activemodel/validations/json_validator"
+require "liquid"
+require "ruby_llm"
 require "ruby_llm/evals/engine"
+require "ruby_llm/evals/version"
+require "ruby_llm/schema"
 
-module RubyLlm
+module RubyLLM
   module Evals
-    # Your code goes here...
+    mattr_accessor :importmap, default: Importmap::Map.new
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-evals
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Patricio Mac Adden
 - Fernando Martinez
 bindir: bin
 cert_chain: []
-date: 2025-10-23 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: importmap-rails
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 7.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 7.2.0
 - !ruby/object:Gem::Dependency
   name: ruby_llm
   requirement: !ruby/object:Gem::Requirement
@@ -118,11 +118,70 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
+- app/assets/stylesheets/ruby_llm/evals/application.css
+- app/assets/stylesheets/ruby_llm/evals/bulma.min.css
+- app/assets/stylesheets/ruby_llm/evals/json_editor.css
+- app/controllers/concerns/ruby_llm/evals/prompt_executions/prompt_execution_scoped.rb
+- app/controllers/ruby_llm/evals/application_controller.rb
+- app/controllers/ruby_llm/evals/prompt_executions/failures_controller.rb
+- app/controllers/ruby_llm/evals/prompt_executions/passages_controller.rb
+- app/controllers/ruby_llm/evals/prompt_executions/retries_controller.rb
+- app/controllers/ruby_llm/evals/prompts_controller.rb
+- app/controllers/ruby_llm/evals/runs_controller.rb
+- app/helpers/ruby_llm/evals/application_helper.rb
+- app/helpers/ruby_llm/evals/prompt_executions_helper.rb
+- app/helpers/ruby_llm/evals/prompts_helper.rb
+- app/helpers/ruby_llm/evals/runs_helper.rb
+- app/javascript/ruby_llm/evals/application.js
+- app/javascript/ruby_llm/evals/controllers/application.js
+- app/javascript/ruby_llm/evals/controllers/eval_type_selector_controller.js
+- app/javascript/ruby_llm/evals/controllers/file_input_controller.js
+- app/javascript/ruby_llm/evals/controllers/index.js
+- app/javascript/ruby_llm/evals/controllers/json_editor_controller.js
+- app/javascript/ruby_llm/evals/controllers/provider_model_controller.js
+- app/javascript/ruby_llm/evals/controllers/schema_selector_controller.js
+- app/jobs/ruby_llm/evals/application_job.rb
+- app/jobs/ruby_llm/evals/execute_sample_job.rb
+- app/jobs/ruby_llm/evals/perform_run_job.rb
+- app/mailers/ruby_llm/evals/application_mailer.rb
+- app/models/concerns/ruby_llm/evals/job_trackable.rb
+- app/models/ruby_llm/evals/application_record.rb
+- app/models/ruby_llm/evals/page.rb
+- app/models/ruby_llm/evals/prompt.rb
+- app/models/ruby_llm/evals/prompt_execution.rb
+- app/models/ruby_llm/evals/run.rb
+- app/models/ruby_llm/evals/sample.rb
+- app/schemas/ruby_llm/evals/judge_verdict_schema.rb
+- app/views/layouts/ruby_llm/evals/application.html.erb
+- app/views/ruby_llm/evals/application/_flashes.html.erb
+- app/views/ruby_llm/evals/application/_nav.html.erb
+- app/views/ruby_llm/evals/application/_pagination.html.erb
+- app/views/ruby_llm/evals/application/_tabs.html.erb
+- app/views/ruby_llm/evals/prompts/_filters.html.erb
+- app/views/ruby_llm/evals/prompts/_form.html.erb
+- app/views/ruby_llm/evals/prompts/_prompt.html.erb
+- app/views/ruby_llm/evals/prompts/compare.html.erb
+- app/views/ruby_llm/evals/prompts/edit.html.erb
+- app/views/ruby_llm/evals/prompts/index.html.erb
+- app/views/ruby_llm/evals/prompts/new.html.erb
+- app/views/ruby_llm/evals/prompts/show.html.erb
+- app/views/ruby_llm/evals/runs/_filters.html.erb
+- app/views/ruby_llm/evals/runs/_run.html.erb
+- app/views/ruby_llm/evals/runs/index.html.erb
+- app/views/ruby_llm/evals/runs/show.html.erb
+- app/views/ruby_llm/evals/samples/_form.html.erb
+- config/importmap.rb
+- config/locales/en.yml
 - config/routes.rb
+- db/migrate/20251022211228_create_ruby_llm_evals_prompts.rb
+- db/migrate/20251022211229_create_ruby_llm_evals_samples.rb
+- db/migrate/20251022211230_create_ruby_llm_evals_runs.rb
+- db/migrate/20251022211231_create_ruby_llm_evals_prompt_executions.rb
+- lib/activemodel/validations/json_validator.rb
 - lib/ruby_llm/evals.rb
 - lib/ruby_llm/evals/engine.rb
 - lib/ruby_llm/evals/version.rb
-- lib/tasks/ruby_llm/evals_tasks.rake
+- lib/tasks/ruby_llm_evals_tasks.rake
 homepage: https://github.com/sinaptia/ruby_llm-evals
 licenses: []
 metadata:
@@ -142,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: LLM evaluation engine for Rails.
 test_files: []