lex-dataset 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5e63278e11eb728b818a7519bc0a33fb713088d3d149100d4398414126e6ced8
+  data.tar.gz: a119bcde6cec1c5f253a46284ead75f5fe672f0cd6a75e7334a6d7b325e61255
+SHA512:
+  metadata.gz: f97ed63b5cc90b64a8773186bdc7d2df964a8ba334c5827dff51653db83030c9e19f89fbd07b7537995fe23677cd22792241bcc53244bcd19bd0e7705c9c6caa
+  data.tar.gz: 71209de12c040ce5c4a09436cf8902fc517352b30f59072a9a0dcfd7db031d9279b2e108c2188d6a108d40c611d68306f67d1503d6aa38ee623d046b93a0d56b

data/lib/legion/extensions/dataset/client.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Dataset
+      class Client
+        include Runners::Dataset
+        include Runners::Experiment
+
+        def initialize(db: nil, **opts)
+          @db   = db
+          @opts = opts
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/dataset/helpers/import_export.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'json'
+
+module Legion
+  module Extensions
+    module Dataset
+      module Helpers
+        module ImportExport
+          module_function
+
+          def import_csv(path)
+            rows = []
+            CSV.foreach(path, headers: true, header_converters: :symbol) do |row|
+              rows << { input: row[:input], expected_output: row[:expected_output], metadata: row[:metadata] }
+            end
+            rows
+          end
+
+          def import_json(path)
+            data = ::JSON.parse(File.read(path), symbolize_names: true)
+            data.is_a?(Array) ? data : []
+          end
+
+          def import_jsonl(path)
+            File.readlines(path).map { |line| ::JSON.parse(line.strip, symbolize_names: true) }
+          end
+
+          def export_csv(rows, path)
+            CSV.open(path, 'w', headers: %w[input expected_output metadata], write_headers: true) do |csv|
+              rows.each { |row| csv << [row[:input], row[:expected_output], row[:metadata]] }
+            end
+          end
+
+          def export_json(rows, path)
+            File.write(path, ::JSON.pretty_generate(rows))
+          end
+
+          def export_jsonl(rows, path)
+            File.open(path, 'w') do |f|
+              rows.each { |row| f.puts(::JSON.generate(row)) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/dataset/runners/dataset.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'json'
+
+module Legion
+  module Extensions
+    module Dataset
+      module Runners
+        module Dataset
+          def create_dataset(name:, description: nil, rows: [], **)
+            ds_id = db[:datasets].insert(name: name, description: description, created_at: Time.now.utc)
+            create_version(ds_id, rows)
+            { created: true, name: name, version: 1, row_count: rows.size }
+          end
+
+          def import_dataset(name:, path:, format: 'json', description: nil, **)
+            rows = case format.to_s
+                   when 'csv'   then Helpers::ImportExport.import_csv(path)
+                   when 'jsonl' then Helpers::ImportExport.import_jsonl(path)
+                   else Helpers::ImportExport.import_json(path)
+                   end
+            create_dataset(name: name, description: description, rows: rows)
+          end
+
+          def export_dataset(name:, path:, format: 'json', version: nil, **)
+            rows = get_rows(name, version)
+            case format.to_s
+            when 'csv'   then Helpers::ImportExport.export_csv(rows, path)
+            when 'jsonl' then Helpers::ImportExport.export_jsonl(rows, path)
+            else Helpers::ImportExport.export_json(rows, path)
+            end
+            { exported: true, path: path, row_count: rows.size }
+          end
+
+          def list_datasets(**)
+            db[:datasets].all.map do |dataset|
+              latest = db[:dataset_versions].where(dataset_id: dataset[:id]).order(Sequel.desc(:version)).first
+              { name: dataset[:name], description: dataset[:description],
+                latest_version: latest ? latest[:version] : nil,
+                row_count:      latest ? latest[:row_count] : 0 }
+            end
+          end
+
+          def get_dataset(name:, version: nil, **)
+            ds = db[:datasets].where(name: name).first
+            return { error: 'not_found' } unless ds
+
+            ver = if version
+                    db[:dataset_versions].where(dataset_id: ds[:id], version: version).first
+                  else
+                    db[:dataset_versions].where(dataset_id: ds[:id]).order(Sequel.desc(:version)).first
+                  end
+            return { error: 'version_not_found' } unless ver
+
+            rows = db[:dataset_rows].where(version_id: ver[:id]).order(:row_index).all
+            { name: name, version: ver[:version], version_id: ver[:id], row_count: ver[:row_count],
+              rows: rows.map { |r| { row_index: r[:row_index], input: r[:input], expected_output: r[:expected_output] } } }
+          end
+
+          private
+
+          def create_version(dataset_id, rows)
+            hash = OpenSSL::Digest.new('SHA256').hexdigest(rows.to_s)
+            ver_num = (db[:dataset_versions].where(dataset_id: dataset_id).max(:version) || 0) + 1
+            ver_id = db[:dataset_versions].insert(
+              dataset_id: dataset_id, version: ver_num, row_count: rows.size,
+              content_hash: hash, created_at: Time.now.utc
+            )
+            rows.each_with_index do |row, idx|
+              db[:dataset_rows].insert(
+                version_id: ver_id, row_index: idx,
+                input:           row[:input].to_s,
+                expected_output: row[:expected_output]&.to_s,
+                metadata:        row[:metadata]&.to_s
+              )
+            end
+            ver_id
+          end
+
+          def get_rows(name, version)
+            result = get_dataset(name: name, version: version)
+            result[:rows] || []
+          end
+
+          def db
+            @db
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/dataset/runners/experiment.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Extensions
+    module Dataset
+      module Runners
+        module Experiment
+          def run_experiment(name:, dataset_name:, task_callable:, dataset_version: nil, evaluators: [], **)
+            ds = get_dataset(name: dataset_name, version: dataset_version)
+            return { error: ds[:error] } if ds[:error]
+
+            exp_id = db[:experiments].insert(
+              name: name, dataset_version_id: ds[:version_id],
+              eval_config: ::JSON.dump(evaluators.map { |e| e.respond_to?(:name) ? e.name : e.to_s }),
+              status: 'running', created_at: Time.now.utc
+            )
+
+            results = ds[:rows].map do |row|
+              start_time = Time.now
+              output = task_callable.call(row[:input])
+              latency = ((Time.now - start_time) * 1000).round
+
+              scores = evaluators.map do |evaluator|
+                evaluator.evaluate(input: row[:input], output: output, expected: row[:expected_output])
+              end
+
+              passed = scores.empty? || scores.all? { |s| s[:passed] }
+              db[:experiment_results].insert(
+                experiment_id: exp_id, row_index: row[:row_index],
+                output: output.to_s, eval_scores: ::JSON.dump(scores),
+                latency_ms: latency, passed: passed
+              )
+              { row_index: row[:row_index], passed: passed, latency_ms: latency }
+            end
+
+            summary = build_summary(results)
+            db[:experiments].where(id: exp_id).update(
+              status: 'completed', summary: ::JSON.dump(summary), completed_at: Time.now.utc
+            )
+            { experiment_id: exp_id, name: name, summary: summary }
+          end
+
+          def compare_experiments(exp1_name:, exp2_name:, **)
+            r1 = load_experiment_results(exp1_name)
+            r2 = load_experiment_results(exp2_name)
+            return { error: 'experiments_not_found' } unless r1 && r2
+
+            pairs = r1.zip(r2).select { |a, b| a && b }
+            regressions = pairs.select { |a, b| a[:passed] && !b[:passed] }.map { |a, _| a[:row_index] }
+            improvements = pairs.select { |a, b| !a[:passed] && b[:passed] }.map { |_, b| b[:row_index] }
+
+            { exp1: exp1_name, exp2: exp2_name, rows_compared: pairs.size,
+              regressions: regressions, improvements: improvements,
+              regression_count: regressions.size, improvement_count: improvements.size }
+          end
+
+          private
+
+          def build_summary(results)
+            {
+              total:          results.size,
+              passed:         results.count { |r| r[:passed] },
+              failed:         results.count { |r| !r[:passed] },
+              avg_latency_ms: results.empty? ? 0 : (results.sum { |r| r[:latency_ms] } / results.size).round
+            }
+          end
+
+          def load_experiment_results(name)
+            exp = db[:experiments].where(name: name).first
+            return nil unless exp
+
+            db[:experiment_results].where(experiment_id: exp[:id]).order(:row_index).all
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/dataset/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Dataset
+      VERSION = '0.1.0'
+    end
+  end
+end

data/lib/legion/extensions/dataset.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative 'dataset/version'
+require_relative 'dataset/helpers/import_export'
+require_relative 'dataset/runners/dataset'
+require_relative 'dataset/runners/experiment'
+require_relative 'dataset/client'
+
+module Legion
+  module Extensions
+    module Dataset
+      extend Legion::Extensions::Core if defined?(Legion::Extensions::Core)
+    end
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: lex-dataset
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matthew Iverson
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Provides versioned dataset storage with import/export (CSV/JSON/JSONL),
+  experiment runner with evaluator integration, and regression detection.
+email:
+- matt@iverson.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/legion/extensions/dataset.rb
+- lib/legion/extensions/dataset/client.rb
+- lib/legion/extensions/dataset/helpers/import_export.rb
+- lib/legion/extensions/dataset/runners/dataset.rb
+- lib/legion/extensions/dataset/runners/experiment.rb
+- lib/legion/extensions/dataset/version.rb
+homepage: https://github.com/LegionIO/lex-dataset
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Versioned dataset management for LegionIO
+test_files: []