leva 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 10570fd9a6cdbf979cf9538963033a752ef0cb7d4d1966e9a002408f875a5aa2
+  data.tar.gz: e194a81e5e53ddf22b3aee902aab09191ccf39836385c885692f95b11fbcdd46
+SHA512:
+  metadata.gz: 7b534a0aceff9d67f1a0f00d3b70993fec440633ef20005adf5ac866772bd57899fb7412073a7deaa2a7dfac856def72bc51a94ed6a5eb26a21818ec405fddfc
+  data.tar.gz: 42551d07b819337c1ac780aa542fb41af2218e257c9f1bf5d11d21dd351e64f5c6942d2354cbce2dead1b0232e62140fe16a1c5a308c6ba2b9fe8659e3b5b88b

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright Kieran Klaassen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,152 @@
+# Leva - Flexible Evaluation Framework for Language Models
+
+Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'leva'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+## Usage
+
+### 1. Setting up Datasets
+
+First, create a dataset and add any ActiveRecord records:
+
+```ruby
+dataset = Dataset.create(name: "Sentiment Analysis Dataset")
+
+dataset.records << TextContent.create(text: "I love this product!", expected_label: "Positive")
+dataset.records << TextContent.create(text: "Terrible experience", expected_label: "Negative")
+dataset.records << TextContent.create(text: "I's ok", expected_label: "Neutral")
+```
+
+> In this case the TextContent model is the ActiveRecord model from your own application.
+
+### 2. Implementing Evals
+
+Create evals by adding new files in `app/evals/`. Each eval implements both the evaluation logic and how to run it. Here are some examples:
+
+```bash
+$ rails generate leva:eval Sentiment
+```
+
+#### Sentiment Evaluation (app/evals/sentiment_eval.rb)
+
+```ruby
+class SentimentEval < Leva::BaseEval
+  leva_dataset_record_class "TextContent"
+
+  def run_each(record)
+    prediction = label_sentiment(record.text)
+    score = calculate_score(prediction, record.expected_label)
+
+    Leva::Result.new(
+      label: 'sentiment',
+      prediction: prediction,
+      score: score
+    )
+  end
+
+  private
+
+  def label_sentiment(text)
+    # Simple sentiment analysis logic, use LLM to label the sentiment yourself
+    text = text.downcase
+    if text.include?('love')
+      'Positive'
+    elsif text.include?('terrible')
+      'Negative'
+    else
+      'Neutral'
+    end
+  end
+
+  def calculate_score(prediction, expected)
+    prediction == expected ? 1.0 : 0.0
+  end
+end
+```
+
+### 3. Running Experiments
+
+You can run experiments with different evals:
+
+```ruby
+sentiment_experiment = Experiment.create!(name: "Sentiment Analysis", dataset: dataset)
+SentimentEval.run_experiment(sentiment_experiment)
+```
+
+You can also run an experiment with a prompt so you can use a LLM to evaluate the dataset:
+
+```ruby
+prompt = Leva::Prompt.create!(
+  name: "Sentiment Analysis",
+  version: 1,
+  system_prompt: "You are an expert at analyzing text and returning the sentiment.",
+  user_prompt: "Please analyze the following text and return the sentiment as Positive, Negative, or Neutral.  \n\n {{TEXT}}",
+  metadata: {
+    model: "gpt-4o",
+    temperature: 0.5
+  }
+)
+
+sentiment_experiment = Experiment.create!(
+  name: "Sentiment Analysis with LLM",
+  dataset: dataset,
+  prompt: prompt
+)
+
+SentimentEval.run_experiment(sentiment_experiment)
+```
+
+### 4. Analyzing Results
+
+After the experiments are complete, analyze the results:
+
+```ruby
+
+results = experiment.evaluation_results
+average_score = results.average(:score)
+count = results.count
+
+puts "Experiment: #{experiment.name}"
+puts "Average Score: #{average_score}"
+puts "Number of Evaluations: #{count}"
+```
+
+## Configuration
+
+If your evals require API keys or other configurations, ensure you set these up in your Rails credentials or environment variables.
+
+## Leva's Components
+
+### Classes
+
+- `Leva::BaseEval`: The base class for all evals. Override the `run` method in your eval classes.
+- `Leva::Result`: The result of an evaluation.
+
+### Models
+
+- `Leva::Dataset`: Represents a collection of data to be evaluated.
+- `Leva::DatasetRecord`: Represents individual records within a dataset.
+- `Leva::Experiment`: Represents a single run of an evaluation on a dataset.
+- `Leva::EvaluationResult`: Stores the results of each evaluation.
+- `Leva::Prompt`: Represents a prompt for an LLM.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kieranklaassen/leva.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/assets/config/leva_manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets/leva .css

data/app/assets/stylesheets/leva/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/leva/application_controller.rb

@@ -0,0 +1,4 @@
+module Leva
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/leva/application_helper.rb

@@ -0,0 +1,4 @@
+module Leva
+  module ApplicationHelper
+  end
+end

data/app/jobs/leva/application_job.rb

@@ -0,0 +1,4 @@
+module Leva
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/mailers/leva/application_mailer.rb

@@ -0,0 +1,6 @@
+module Leva
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/leva/application_record.rb

@@ -0,0 +1,5 @@
+module Leva
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/leva/dataset.rb

@@ -0,0 +1,16 @@
+# == Schema Information
+#
+# Table name: leva_datasets
+#
+#  id         :integer          not null, primary key
+#  name       :string
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+module Leva
+  class Dataset < ApplicationRecord
+    has_many :dataset_records, dependent: :destroy
+    has_many :records, through: :dataset_records, source: :recordable
+    has_many :experiments, dependent: :destroy
+  end
+end

data/app/models/leva/dataset_record.rb

@@ -0,0 +1,26 @@
+# == Schema Information
+#
+# Table name: leva_dataset_records
+#
+#  id              :integer          not null, primary key
+#  recordable_type :string           not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#  dataset_id      :integer          not null
+#  recordable_id   :integer          not null
+#
+# Indexes
+#
+#  index_leva_dataset_records_on_dataset_id  (dataset_id)
+#  index_leva_dataset_records_on_recordable  (recordable_type,recordable_id)
+#
+# Foreign Keys
+#
+#  dataset_id  (dataset_id => datasets.id)
+#
+module Leva
+  class DatasetRecord < ApplicationRecord
+    belongs_to :dataset
+    belongs_to :recordable, polymorphic: true
+  end
+end

data/app/models/leva/evaluation_result.rb

@@ -0,0 +1,29 @@
+# == Schema Information
+#
+# Table name: leva_evaluation_results
+#
+#  id                :integer          not null, primary key
+#  label             :string
+#  prediction        :string
+#  score             :float
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  dataset_record_id :integer          not null
+#  experiment_id     :integer          not null
+#
+# Indexes
+#
+#  index_leva_evaluation_results_on_dataset_record_id  (dataset_record_id)
+#  index_leva_evaluation_results_on_experiment_id      (experiment_id)
+#
+# Foreign Keys
+#
+#  dataset_record_id  (dataset_record_id => dataset_records.id)
+#  experiment_id      (experiment_id => experiments.id)
+#
+module Leva
+  class EvaluationResult < ApplicationRecord
+    belongs_to :experiment
+    belongs_to :dataset_record
+  end
+end

data/app/models/leva/experiment.rb

@@ -0,0 +1,29 @@
+# == Schema Information
+#
+# Table name: leva_experiments
+#
+#  id         :integer          not null, primary key
+#  metadata   :text
+#  name       :string
+#  status     :integer
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  dataset_id :integer          not null
+#  prompt_id  :integer          not null
+#
+# Indexes
+#
+#  index_leva_experiments_on_dataset_id  (dataset_id)
+#  index_leva_experiments_on_prompt_id   (prompt_id)
+#
+# Foreign Keys
+#
+#  dataset_id  (dataset_id => datasets.id)
+#  prompt_id   (prompt_id => prompts.id)
+#
+module Leva
+  class Experiment < ApplicationRecord
+    belongs_to :dataset
+    belongs_to :prompt
+  end
+end

data/app/models/leva/prompt.rb

@@ -0,0 +1,17 @@
+# == Schema Information
+#
+# Table name: leva_prompts
+#
+#  id            :integer          not null, primary key
+#  metadata      :text
+#  name          :string
+#  system_prompt :text
+#  user_prompt   :text
+#  version       :integer
+#  created_at    :datetime         not null
+#  updated_at    :datetime         not null
+#
+module Leva
+  class Prompt < ApplicationRecord
+  end
+end

data/app/views/layouts/leva/application.html.erb

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Leva</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= yield :head %>
+
+  <%= stylesheet_link_tag    "leva/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,2 @@
+Leva::Engine.routes.draw do
+end

data/db/migrate/20240813172916_create_leva_datasets.rb

@@ -0,0 +1,9 @@
+class CreateLevaDatasets < ActiveRecord::Migration[7.2]
+  def change
+    create_table :leva_datasets do |t|
+      t.string :name
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20240813173033_create_leva_dataset_records.rb

@@ -0,0 +1,10 @@
+class CreateLevaDatasetRecords < ActiveRecord::Migration[7.2]
+  def change
+    create_table :leva_dataset_records do |t|
+      t.references :dataset, null: false, foreign_key: true
+      t.references :recordable, polymorphic: true, null: false
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20240813173050_create_leva_evaluation_results.rb

@@ -0,0 +1,13 @@
+class CreateLevaEvaluationResults < ActiveRecord::Migration[7.2]
+  def change
+    create_table :leva_evaluation_results do |t|
+      t.references :experiment, null: false, foreign_key: true
+      t.references :dataset_record, null: false, foreign_key: true
+      t.string :prediction
+      t.float :score
+      t.string :label
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20240813173105_create_leva_prompts.rb

@@ -0,0 +1,13 @@
+class CreateLevaPrompts < ActiveRecord::Migration[7.2]
+  def change
+    create_table :leva_prompts do |t|
+      t.string :name
+      t.integer :version
+      t.text :system_prompt
+      t.text :user_prompt
+      t.text :metadata
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20240813173222_create_leva_experiments.rb

@@ -0,0 +1,13 @@
+class CreateLevaExperiments < ActiveRecord::Migration[7.2]
+  def change
+    create_table :leva_experiments do |t|
+      t.string :name
+      t.references :dataset, null: false, foreign_key: true
+      t.references :prompt, null: false, foreign_key: true
+      t.integer :status
+      t.text :metadata
+
+      t.timestamps
+    end
+  end
+end

data/lib/leva/engine.rb

@@ -0,0 +1,5 @@
+module Leva
+  class Engine < ::Rails::Engine
+    isolate_namespace Leva
+  end
+end

data/lib/leva/version.rb

@@ -0,0 +1,3 @@
+module Leva
+  VERSION = "0.1.0"
+end

data/lib/leva.rb

@@ -0,0 +1,6 @@
+require "leva/version"
+require "leva/engine"
+
+module Leva
+  # Your code goes here...
+end

data/lib/tasks/auto_annotate_models.rake

@@ -0,0 +1,59 @@
+# NOTE: only doing this in development as some production environments (Heroku)
+# NOTE: are sensitive to local FS writes, and besides -- it's just not proper
+# NOTE: to have a dev-mode tool do its thing in production.
+if Rails.env.development?
+  require 'annotate'
+  task :set_annotation_options do
+    # You can override any of these by setting an environment variable of the
+    # same name.
+    Annotate.set_defaults(
+      'active_admin'                => 'false',
+      'additional_file_patterns'    => [],
+      'routes'                      => 'false',
+      'models'                      => 'true',
+      'position_in_routes'          => 'before',
+      'position_in_class'           => 'before',
+      'position_in_test'            => 'before',
+      'position_in_fixture'         => 'before',
+      'position_in_factory'         => 'before',
+      'position_in_serializer'      => 'before',
+      'show_foreign_keys'           => 'true',
+      'show_complete_foreign_keys'  => 'false',
+      'show_indexes'                => 'true',
+      'simple_indexes'              => 'false',
+      'model_dir'                   => 'app/models',
+      'root_dir'                    => '',
+      'include_version'             => 'false',
+      'require'                     => '',
+      'exclude_tests'               => 'false',
+      'exclude_fixtures'            => 'false',
+      'exclude_factories'           => 'false',
+      'exclude_serializers'         => 'false',
+      'exclude_scaffolds'           => 'true',
+      'exclude_controllers'         => 'true',
+      'exclude_helpers'             => 'true',
+      'exclude_sti_subclasses'      => 'false',
+      'ignore_model_sub_dir'        => 'false',
+      'ignore_columns'              => nil,
+      'ignore_routes'               => nil,
+      'ignore_unknown_models'       => 'false',
+      'hide_limit_column_types'     => 'integer,bigint,boolean',
+      'hide_default_column_types'   => 'json,jsonb,hstore',
+      'skip_on_db_migrate'          => 'false',
+      'format_bare'                 => 'true',
+      'format_rdoc'                 => 'false',
+      'format_yard'                 => 'false',
+      'format_markdown'             => 'false',
+      'sort'                        => 'false',
+      'force'                       => 'false',
+      'frozen'                      => 'false',
+      'classified_sort'             => 'true',
+      'trace'                       => 'false',
+      'wrapper_open'                => nil,
+      'wrapper_close'               => nil,
+      'with_comment'                => 'true'
+    )
+  end
+
+  Annotate.load_tasks
+end

data/lib/tasks/leva_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :leva do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: leva
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kieran Klaassen
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.2.0
+description: Leva is a Ruby on Rails framework for evaluating Language Models (LLMs)
+  using ActiveRecord datasets. It provides a flexible structure for creating experiments,
+  managing datasets, and implementing various evaluation logic.
+email:
+- kieranklaassen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/config/leva_manifest.js
+- app/assets/stylesheets/leva/application.css
+- app/controllers/leva/application_controller.rb
+- app/helpers/leva/application_helper.rb
+- app/jobs/leva/application_job.rb
+- app/mailers/leva/application_mailer.rb
+- app/models/leva/application_record.rb
+- app/models/leva/dataset.rb
+- app/models/leva/dataset_record.rb
+- app/models/leva/evaluation_result.rb
+- app/models/leva/experiment.rb
+- app/models/leva/prompt.rb
+- app/views/layouts/leva/application.html.erb
+- config/routes.rb
+- db/migrate/20240813172916_create_leva_datasets.rb
+- db/migrate/20240813173033_create_leva_dataset_records.rb
+- db/migrate/20240813173050_create_leva_evaluation_results.rb
+- db/migrate/20240813173105_create_leva_prompts.rb
+- db/migrate/20240813173222_create_leva_experiments.rb
+- lib/leva.rb
+- lib/leva/engine.rb
+- lib/leva/version.rb
+- lib/tasks/auto_annotate_models.rake
+- lib/tasks/leva_tasks.rake
+homepage: https://github.com/kieranklaassen/leva
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/kieranklaassen/leva
+  source_code_uri: https://github.com/kieranklaassen/leva
+  changelog_uri: https://github.com/kieranklaassen/leva/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Flexible Evaluation Framework for Language Models in Rails
+test_files: []